Add option to exclude an enum possible value's fields from docs

CE-773 Fixing globs for local filesystem by using Files.walkFileTree.…

pom.xml

@@ -33,6 +33,7 @@
       <module>qqq-backend-module-api</module>
       <module>qqq-backend-module-filesystem</module>
       <module>qqq-backend-module-rdbms</module>
+      <module>qqq-backend-module-mongodb</module>
       <module>qqq-language-support-javascript</module>
       <module>qqq-middleware-picocli</module>
       <module>qqq-middleware-javalin</module>
@@ -108,6 +109,16 @@
    </dependencyManagement>
 
    <build>
+      <resources>
+         <resource>
+            <directory>src/main/java</directory>
+            <filtering>false</filtering>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+         </resource>
+      </resources>
       <plugins>
          <!-- plugins specifically for this module -->
          <!-- none at this time -->

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/QBackendTransaction.java

@@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.actions;
 
 
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
 /*******************************************************************************
@@ -30,12 +33,26 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
  ** part of a transaction.
  **
  ** Most obvious use-case would be a JDBC Connection.  See subclass in rdbms module.
+ ** Ditto MongoDB.
  **
  ** Note:  One would imagine that this class shouldn't ever implement Serializable...
  *******************************************************************************/
 public class QBackendTransaction
 {
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QBackendTransaction openFor(AbstractTableActionInput input) throws QException
+   {
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(input.getBackend());
+      QBackendTransaction      transaction              = qModule.openTransaction(input);
+      return (transaction);
+   }
+
+
+
    /*******************************************************************************
     ** Commit the transaction.
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -30,10 +30,12 @@ import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
@@ -73,6 +75,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
 
    public static final String AUDIT_CONTEXT_FIELD_NAME = "auditContext";
 
+   private static Set<String> loggedUnauditableTableNames = new HashSet<>();
 
 
    /*******************************************************************************
@@ -88,6 +91,20 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
       long                     start            = System.currentTimeMillis();
       DMLType                  dmlType          = getDMLType(tableActionInput);
 
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // currently, the table's primary key must be id... so, log (once) and return early if not that //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      QFieldMetaData field = table.getField(table.getPrimaryKeyField());
+      if(!QFieldType.INTEGER.equals(field.getType()))
+      {
+         if(!loggedUnauditableTableNames.contains(table.getName()))
+         {
+            LOG.info("Cannot audit table without integer as its primary key", logPair("tableName", table.getName()));
+            loggedUnauditableTableNames.add(table.getName());
+         }
+         return (output);
+      }
+
       try
       {
          List<QRecord> recordList = CollectionUtils.nonNullList(input.getRecordList()).stream()

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -43,6 +43,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
@@ -87,8 +88,9 @@ public class PollingAutomationPerTableRunner implements Runnable
 {
    private static final QLogger LOG = QLogger.getLogger(PollingAutomationPerTableRunner.class);
 
-   private final TableActions tableActions;
-   private final String       name;
+   private final TableActionsInterface tableActions;
+
+   private String name;
 
    private QInstance          instance;
    private Supplier<QSession> sessionSupplier;
@@ -116,10 +118,51 @@ public class PollingAutomationPerTableRunner implements Runnable
 
 
    /*******************************************************************************
-    **
+    ** Interface to be used by 2 records in this class - normal TableActions, and
+    ** ShardedTableActions.
     *******************************************************************************/
-   public record TableActions(String tableName, AutomationStatus status)
+   public interface TableActionsInterface
    {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      String tableName();
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      AutomationStatus status();
+   }
+
+
+
+   /*******************************************************************************
+    ** Wrapper for a pair of (tableName, automationStatus)
+    *******************************************************************************/
+   public record TableActions(String tableName, AutomationStatus status) implements TableActionsInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public void noopToFakeTestCoverage()
+      {
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** extended version of TableAction, for sharding use-case - adds the shard
+    ** details.
+    *******************************************************************************/
+   public record ShardedTableActions(String tableName, AutomationStatus status, String shardByFieldName, Serializable shardValue, String shardLabel) implements TableActionsInterface
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public void noopToFakeTestCoverage()
+      {
+      }
    }
 
 
@@ -128,16 +171,46 @@ public class PollingAutomationPerTableRunner implements Runnable
     ** basically just get a list of tables which at least *could* have automations
     ** run - either meta-data automations, or table-triggers (data/user defined).
     *******************************************************************************/
-   public static List<TableActions> getTableActions(QInstance instance, String providerName)
+   public static List<TableActionsInterface> getTableActions(QInstance instance, String providerName)
    {
-      List<TableActions> tableActionList = new ArrayList<>();
+      List<TableActionsInterface> tableActionList = new ArrayList<>();
 
       for(QTableMetaData table : instance.getTables().values())
       {
-         if(table.getAutomationDetails() != null && providerName.equals(table.getAutomationDetails().getProviderName()))
+         QTableAutomationDetails automationDetails = table.getAutomationDetails();
+         if(automationDetails != null && providerName.equals(automationDetails.getProviderName()))
          {
-            tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
-            tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+            if(StringUtils.hasContent(automationDetails.getShardByFieldName()))
+            {
+               //////////////////////////////////////////////////////////////////////////////////////////////
+               // for sharded automations, add a tableAction (of the sharded subtype) for each shard-value //
+               //////////////////////////////////////////////////////////////////////////////////////////////
+               try
+               {
+                  QueryInput queryInput = new QueryInput();
+                  queryInput.setTableName(automationDetails.getShardSourceTableName());
+                  QueryOutput queryOutput = new QueryAction().execute(queryInput);
+                  for(QRecord record : queryOutput.getRecords())
+                  {
+                     Serializable shardId = record.getValue(automationDetails.getShardIdFieldName());
+                     String       label   = record.getValueString(automationDetails.getShardLabelFieldName());
+                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                     tableActionList.add(new ShardedTableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS, automationDetails.getShardByFieldName(), shardId, label));
+                  }
+               }
+               catch(Exception e)
+               {
+                  LOG.error("Error getting sharded table automation actions for a table", e, new LogPair("tableName", table.getName()));
+               }
+            }
+            else
+            {
+               ///////////////////////////////////////////////////////////////////
+               // for non-sharded, we just need tabler name & automation status //
+               ///////////////////////////////////////////////////////////////////
+               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_INSERT_AUTOMATIONS));
+               tableActionList.add(new TableActions(table.getName(), AutomationStatus.PENDING_UPDATE_AUTOMATIONS));
+            }
          }
       }
 
@@ -149,12 +222,17 @@ public class PollingAutomationPerTableRunner implements Runnable
    /*******************************************************************************
     **
     *******************************************************************************/
-   public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
+   public PollingAutomationPerTableRunner(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActionsInterface tableActions)
    {
       this.instance = instance;
       this.sessionSupplier = sessionSupplier;
       this.tableActions = tableActions;
       this.name = providerName + ">" + tableActions.tableName() + ">" + tableActions.status().getInsertOrUpdate();
+
+      if(tableActions instanceof ShardedTableActions shardedTableActions)
+      {
+         this.name += ":" + shardedTableActions.shardLabel();
+      }
    }
 
 
@@ -229,6 +307,15 @@ public class PollingAutomationPerTableRunner implements Runnable
                throw (new NotImplementedException("Automation Status Tracking type [" + statusTrackingType + "] is not yet implemented in here."));
             }
 
+            if(tableActions instanceof ShardedTableActions shardedTableActions)
+            {
+               //////////////////////////////////////////////////////////////
+               // for sharded actions, add the shardBy field as a criteria //
+               //////////////////////////////////////////////////////////////
+               QQueryFilter filter = queryInput.getFilter();
+               filter.addCriteria(new QFilterCriteria(shardedTableActions.shardByFieldName(), QCriteriaOperator.EQUALS, shardedTableActions.shardValue()));
+            }
+
             queryInput.setRecordPipe(recordPipe);
             return (new QueryAction().execute(queryInput));
          }, () ->
@@ -258,7 +345,23 @@ public class PollingAutomationPerTableRunner implements Runnable
       {
          if(action.getTriggerEvent().equals(triggerEvent))
          {
-            rs.add(action);
+            ///////////////////////////////////////////////////////////
+            // for sharded configs, only run if the shard id matches //
+            ///////////////////////////////////////////////////////////
+            if(tableActions instanceof ShardedTableActions shardedTableActions)
+            {
+               if(shardedTableActions.shardValue().equals(action.getShardId()))
+               {
+                  rs.add(action);
+               }
+            }
+            else
+            {
+               ////////////////////////////////////////////
+               // for non-sharded, always add the action //
+               ////////////////////////////////////////////
+               rs.add(action);
+            }
          }
       }
 
@@ -471,7 +574,7 @@ public class PollingAutomationPerTableRunner implements Runnable
             @Override
             public QQueryFilter getQueryFilter()
             {
-               List<Serializable> recordIds = records.stream().map(r -> r.getValueInteger(table.getPrimaryKeyField())).collect(Collectors.toList());
+               List<Serializable> recordIds = records.stream().map(r -> r.getValue(table.getPrimaryKeyField())).collect(Collectors.toList());
                return (new QQueryFilter().withCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds)));
             }
          });

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/InsertInterface.java

@@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
  ** Interface for the Insert action.
  **
  *******************************************************************************/
-public interface InsertInterface extends QActionInterface
+public interface InsertInterface
 {
    /*******************************************************************************
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/metadata/MetaDataAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.metadata;
 
 
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -150,14 +151,21 @@ public class MetaDataAction
       }
       metaDataOutput.setWidgets(widgets);
 
+      ///////////////////////////////////////////////////////
+      // sort apps - by sortOrder (integer), then by label //
+      ///////////////////////////////////////////////////////
+      List<QAppMetaData> sortedApps = metaDataInput.getInstance().getApps().values().stream()
+         .sorted(Comparator.comparing((QAppMetaData a) -> a.getSortOrder())
+            .thenComparing((QAppMetaData a) -> a.getLabel()))
+         .toList();
+
       ///////////////////////////////////
       // map apps to frontend metadata //
       ///////////////////////////////////
       Map<String, QFrontendAppMetaData> apps = new LinkedHashMap<>();
-      for(Map.Entry<String, QAppMetaData> entry : metaDataInput.getInstance().getApps().entrySet())
+      for(QAppMetaData app : sortedApps)
       {
-         String       appName = entry.getKey();
-         QAppMetaData app     = entry.getValue();
+         String appName = app.getName();
 
          PermissionCheckResult permissionResult = PermissionsHelper.getPermissionCheckResult(metaDataInput, app);
          if(permissionResult.equals(PermissionCheckResult.DENY_HIDE))
@@ -191,7 +199,7 @@ public class MetaDataAction
       // organize app tree nodes by their hierarchy //
       ////////////////////////////////////////////////
       List<AppTreeNode> appTree = new ArrayList<>();
-      for(QAppMetaData appMetaData : metaDataInput.getInstance().getApps().values())
+      for(QAppMetaData appMetaData : sortedApps)
       {
          if(appMetaData.getParentAppName() == null)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactory.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.processes;
+
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+
+
+/*******************************************************************************
+ ** Constructor for commonly used QProcessCallback's
+ *******************************************************************************/
+public class QProcessCallbackFactory
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QProcessCallback forFilter(QQueryFilter filter)
+   {
+      return new QProcessCallback()
+      {
+         /*******************************************************************************
+          **
+          *******************************************************************************/
+         @Override
+         public QQueryFilter getQueryFilter()
+         {
+            return (filter);
+         }
+
+
+
+         /*******************************************************************************
+          **
+          *******************************************************************************/
+         @Override
+         public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
+         {
+            return (Collections.emptyMap());
+         }
+      };
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -72,7 +72,7 @@ import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
- ** Action handler for running q-processes (which are a sequence of q-functions).
+ ** Action handler for running q-processes (which are a sequence of q-steps).
  *
  *******************************************************************************/
 public class RunProcessAction
@@ -82,6 +82,7 @@ public class RunProcessAction
    public static final String BASEPULL_THIS_RUNTIME_KEY = "basepullThisRuntimeKey";
    public static final String BASEPULL_LAST_RUNTIME_KEY = "basepullLastRuntimeKey";
    public static final String BASEPULL_TIMESTAMP_FIELD  = "basepullTimestampField";
+   public static final String BASEPULL_CONFIGURATION = "basepullConfiguration";
 
    ////////////////////////////////////////////////////////////////////////////////////////////////
    // indicator that the timestamp field should be updated - e.g., the execute step is finished. //
@@ -633,5 +634,6 @@ public class RunProcessAction
 
       runProcessInput.getValues().put(BASEPULL_LAST_RUNTIME_KEY, lastRunTime);
       runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
+      runProcessInput.getValues().put(BASEPULL_CONFIGURATION, basepullConfiguration);
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import java.io.Serializable;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -52,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.actions.audits.DMLAuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -116,21 +118,24 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
       setAutomationStatusField(insertInput);
 
-      //////////////////////////////////////////////////////
-      // load the backend module and its insert interface //
-      //////////////////////////////////////////////////////
-      QBackendModuleInterface qModule         = getBackendModuleInterface(insertInput);
-      InsertInterface         insertInterface = qModule.getInsertInterface();
-
       /////////////////////////////
       // run standard validators //
       /////////////////////////////
       performValidations(insertInput, false);
 
-      ////////////////////////////////////
-      // have the backend do the insert //
-      ////////////////////////////////////
-      InsertOutput insertOutput = insertInterface.execute(insertInput);
+      //////////////////////////////////////////////////////
+      // use the backend module to actually do the insert //
+      //////////////////////////////////////////////////////
+      InsertOutput insertOutput = runInsertInBackend(insertInput);
+
+      if(insertOutput.getRecords() == null)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////
+         // in case the module failed to set record in the output, put an empty list there //
+         // to avoid so many downstream NPE's                                              //
+         ////////////////////////////////////////////////////////////////////////////////////
+         insertOutput.setRecords(new ArrayList<>());
+      }
 
       //////////////////////////////
       // log if there were errors //
@@ -186,6 +191,71 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private InsertOutput runInsertInBackend(InsertInput insertInput) throws QException
+   {
+      ///////////////////////////////////
+      // exit early if 0 input records //
+      ///////////////////////////////////
+      if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
+      {
+         LOG.debug("Insert request called with 0 records.  Returning with no-op", logPair("tableName", insertInput.getTableName()));
+         InsertOutput rs = new InsertOutput();
+         rs.setRecords(new ArrayList<>());
+         return (rs);
+      }
+
+      /////////////////////////////////////////////
+      // set values in create date & modify date //
+      // todo .. better (not hard-coded names)   //
+      /////////////////////////////////////////////
+      Instant now = Instant.now();
+      for(QRecord record : insertInput.getRecords())
+      {
+         setValueIfTableHasField(record, insertInput.getTable(), "createDate", now);
+         setValueIfTableHasField(record, insertInput.getTable(), "modifyDate", now);
+      }
+
+      //////////////////////////////////////////////////////
+      // load the backend module and its insert interface //
+      //////////////////////////////////////////////////////
+      QBackendModuleInterface qModule         = getBackendModuleInterface(insertInput.getBackend());
+      InsertInterface         insertInterface = qModule.getInsertInterface();
+
+      ////////////////////////////////////
+      // have the backend do the insert //
+      ////////////////////////////////////
+      InsertOutput insertOutput = insertInterface.execute(insertInput);
+      return insertOutput;
+   }
+
+
+
+   /*******************************************************************************
+    ** If the table has a field with the given name, then set the given value in the
+    ** given record.
+    *******************************************************************************/
+   private static void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value)
+   {
+      try
+      {
+         if(table.getFields().containsKey(fieldName))
+         {
+            record.setValue(fieldName, value);
+         }
+      }
+      catch(Exception e)
+      {
+         /////////////////////////////////////////////////
+         // this means field doesn't exist, so, ignore. //
+         /////////////////////////////////////////////////
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -417,23 +487,11 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QBackendModuleInterface getBackendModuleInterface(InsertInput insertInput) throws QException
+   private QBackendModuleInterface getBackendModuleInterface(QBackendMetaData backend) throws QException
    {
-      ActionHelper.validateSession(insertInput);
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
-      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(insertInput.getBackend());
+      QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
       return (qModule);
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public QBackendTransaction openTransaction(InsertInput insertInput) throws QException
-   {
-      QBackendModuleInterface qModule = getBackendModuleInterface(insertInput);
-      return (qModule.getInsertInterface().openTransaction(insertInput));
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -84,9 +84,7 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
          String         primaryKeyField = table.getPrimaryKeyField();
          if(transaction == null)
          {
-            InsertInput insertInput = new InsertInput();
-            insertInput.setTableName(input.getTableName());
-            transaction = new InsertAction().openTransaction(insertInput);
+            transaction = QBackendTransaction.openFor(new InsertInput(input.getTableName()));
             weOwnTheTransaction = true;
          }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -135,7 +135,16 @@ public class UpdateAction
       ////////////////////////////////////
       // have the backend do the update //
       ////////////////////////////////////
-      UpdateOutput updateOutput = updateInterface.execute(updateInput);
+      UpdateOutput updateOutput = runUpdateInBackend(updateInput, updateInterface);
+
+      if(updateOutput.getRecords() == null)
+      {
+         ////////////////////////////////////////////////////////////////////////////////////
+         // in case the module failed to set record in the output, put an empty list there //
+         // to avoid so many downstream NPE's                                              //
+         ////////////////////////////////////////////////////////////////////////////////////
+         updateOutput.setRecords(new ArrayList<>());
+      }
 
       //////////////////////////////
       // log if there were errors //
@@ -194,6 +203,28 @@ public class UpdateAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private UpdateOutput runUpdateInBackend(UpdateInput updateInput, UpdateInterface updateInterface) throws QException
+   {
+      ///////////////////////////////////
+      // exit early if 0 input records //
+      ///////////////////////////////////
+      if(CollectionUtils.nullSafeIsEmpty(updateInput.getRecords()))
+      {
+         LOG.debug("Update request called with 0 records.  Returning with no-op", logPair("tableName", updateInput.getTableName()));
+         UpdateOutput rs = new UpdateOutput();
+         rs.setRecords(new ArrayList<>());
+         return (rs);
+      }
+
+      UpdateOutput updateOutput = updateInterface.execute(updateInput);
+      return updateOutput;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UpdateActionRecordSplitHelper.java

@@ -0,0 +1,203 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
+
+
+/*******************************************************************************
+ ** Helper for backends that want to do their updates on records grouped by the
+ ** set of fields that are being changed, and/or by the values those fields are
+ ** being set to.
+ **
+ ** e.g., RDBMS, for n records where some sub-set of fields are all having values
+ ** set the same (say, a status=x), we can do that as 1 query where id in (?,?,...,?).
+ *******************************************************************************/
+public class UpdateActionRecordSplitHelper
+{
+   private ListingHash<List<String>, QRecord> recordsByFieldBeingUpdated = new ListingHash<>();
+   private boolean                            haveAnyWithoutErrors       = false;
+   private List<QRecord>                      outputRecords              = new ArrayList<>();
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void init(UpdateInput updateInput)
+   {
+      QTableMetaData table = updateInput.getTable();
+      Instant        now   = Instant.now();
+
+      for(QRecord record : updateInput.getRecords())
+      {
+         ////////////////////////////////////////////
+         // todo .. better (not a hard-coded name) //
+         ////////////////////////////////////////////
+         setValueIfTableHasField(record, table, "modifyDate", now);
+
+         List<String> updatableFields = table.getFields().values().stream()
+            .map(QFieldMetaData::getName)
+            // todo - intent here is to avoid non-updateable fields - but this
+            //  should be like based on field.isUpdatable once that attribute exists
+            .filter(name -> !name.equals("id"))
+            .filter(name -> record.getValues().containsKey(name))
+            .toList();
+         recordsByFieldBeingUpdated.add(updatableFields, record);
+
+         if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+         {
+            haveAnyWithoutErrors = true;
+         }
+
+         //////////////////////////////////////////////////////////////////////////////
+         // go ahead and put the record into the output list at this point in time,  //
+         // so that the output list's order matches the input list order             //
+         // note that if we want to capture updated values (like modify dates), then //
+         // we may want a map of primary key to output record, for easy updating.    //
+         //////////////////////////////////////////////////////////////////////////////
+         QRecord outputRecord = new QRecord(record);
+         outputRecords.add(outputRecord);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean areAllValuesBeingUpdatedTheSame(UpdateInput updateInput, List<QRecord> recordList, List<String> fieldsBeingUpdated)
+   {
+      if(updateInput.getAreAllValuesBeingUpdatedTheSame() != null)
+      {
+         ////////////////////////////////////////////////////////////
+         // if input told us what value to use here, then trust it //
+         ////////////////////////////////////////////////////////////
+         return (updateInput.getAreAllValuesBeingUpdatedTheSame());
+      }
+      else
+      {
+         if(recordList.size() == 1)
+         {
+            //////////////////////////////////////////////////////
+            // if a single record, then yes, that always counts //
+            //////////////////////////////////////////////////////
+            return (true);
+         }
+
+         ///////////////////////////////////////////////////////////////////////
+         // else iterate over the records, comparing them to the first record //
+         // return a false if any diffs are found. if no diffs, return true.  //
+         ///////////////////////////////////////////////////////////////////////
+         QRecord firstRecord = recordList.get(0);
+         for(int i = 1; i < recordList.size(); i++)
+         {
+            QRecord record = recordList.get(i);
+
+            if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+            {
+               ///////////////////////////////////////////////////////
+               // skip records w/ errors (that we won't be updating //
+               ///////////////////////////////////////////////////////
+               continue;
+            }
+
+            for(String fieldName : fieldsBeingUpdated)
+            {
+               if(!Objects.equals(firstRecord.getValue(fieldName), record.getValue(fieldName)))
+               {
+                  return (false);
+               }
+            }
+         }
+
+         return (true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** If the table has a field with the given name, then set the given value in the
+    ** given record.
+    *******************************************************************************/
+   protected void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value)
+   {
+      try
+      {
+         if(table.getFields().containsKey(fieldName))
+         {
+            record.setValue(fieldName, value);
+         }
+      }
+      catch(Exception e)
+      {
+         /////////////////////////////////////////////////
+         // this means field doesn't exist, so, ignore. //
+         /////////////////////////////////////////////////
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for haveAnyWithoutErrors
+    **
+    *******************************************************************************/
+   public boolean getHaveAnyWithoutErrors()
+   {
+      return haveAnyWithoutErrors;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for recordsByFieldBeingUpdated
+    **
+    *******************************************************************************/
+   public ListingHash<List<String>, QRecord> getRecordsByFieldBeingUpdated()
+   {
+      return recordsByFieldBeingUpdated;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for outputRecords
+    **
+    *******************************************************************************/
+   public List<QRecord> getOutputRecords()
+   {
+      return outputRecords;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QPossibleValueTranslator.java

@@ -34,7 +34,6 @@ import java.util.Objects;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
@@ -104,7 +103,7 @@ public class QPossibleValueTranslator
          {
             if(!transactionsPerTable.containsKey(tableName))
             {
-               transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
+               transactionsPerTable.put(tableName, QBackendTransaction.openFor(new InsertInput(tableName)));
             }
 
             return (transactionsPerTable.get(tableName));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -334,24 +334,34 @@ public class QValueFormatter
                         if(exposedJoin.getJoinTable().equals(nameParts[0]))
                         {
                            QTableMetaData joinTable = QContext.getQInstance().getTable(nameParts[0]);
-                           fieldMap.put(fieldName, joinTable.getField(nameParts[1]));
+                           if(joinTable.getFields().containsKey(nameParts[1]))
+                           {
+                              fieldMap.put(fieldName, joinTable.getField(nameParts[1]));
+                           }
                         }
                      }
                   }
                   else
                   {
-                     fieldMap.put(fieldName, table.getField(fieldName));
+                     if(table.getFields().containsKey(fieldName))
+                     {
+                        fieldMap.put(fieldName, table.getField(fieldName));
+                     }
                   }
                }
                catch(Exception e)
                {
-                  ///////////////////////////////////////////////////////////
-                  // put an empty field in - so no formatting will be done //
-                  ///////////////////////////////////////////////////////////
-                  LOG.info("Error getting field for setting display value", e, logPair("fieldName", fieldName), logPair("tableName", table.getName()));
-                  fieldMap.put(fieldName, new QFieldMetaData());
+                  LOG.warn("Error getting field for setting display value", e, logPair("fieldName", fieldName), logPair("tableName", table.getName()));
                }
             }
+
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if we didn't find the field definition, put an empty field in the map, so no formatting will be done //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if(!fieldMap.containsKey(fieldName))
+            {
+               fieldMap.put(fieldName, new QFieldMetaData());
+            }
          }
 
          setDisplayValuesInRecord(fieldMap, record);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -133,11 +134,17 @@ public class CsvToQRecordAdapter
                CSVFormat.DEFAULT
                   .withFirstRecordAsHeader()
                   .withIgnoreHeaderCase()
+                  .withIgnoreEmptyLines()
                   .withTrim());
 
             List<String> headers = csvParser.getHeaderNames();
             headers = makeHeadersUnique(headers);
 
+            ////////////////////////////////////////
+            // used by csv-headers-as-field-names //
+            ////////////////////////////////////////
+            Map<String, QFieldMetaData> csvHeaderFieldMapping = buildCsvHeaderFieldMappingIfNeeded(inputWrapper, headers);
+
             Iterator<CSVRecord> csvIterator = csvParser.iterator();
             int                 recordCount = 0;
             while(csvIterator.hasNext())
@@ -160,11 +167,27 @@ public class CsvToQRecordAdapter
                QRecord qRecord = new QRecord();
                try
                {
-                  for(QFieldMetaData field : table.getFields().values())
+                  if(inputWrapper.getCsvHeadersAsFieldNames())
                   {
-                     String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
-                     fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
-                     setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
+                     /////////////////////////////////////////////////////////////////////////////////////////
+                     // in csv-headers-as-field-names mode, don't mess with table, and don't do any mapping //
+                     /////////////////////////////////////////////////////////////////////////////////////////
+                     for(Map.Entry<String, String> entry : csvValues.entrySet())
+                     {
+                        setValue(inputWrapper, qRecord, csvHeaderFieldMapping.get(entry.getKey()), entry.getValue());
+                     }
+                  }
+                  else
+                  {
+                     ///////////////////////////////////////
+                     // otherwise, fields come from table //
+                     ///////////////////////////////////////
+                     for(QFieldMetaData field : table.getFields().values())
+                     {
+                        String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
+                        fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
+                        setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
+                     }
                   }
 
                   runRecordCustomizer(recordCustomizer, qRecord);
@@ -247,6 +270,26 @@ public class CsvToQRecordAdapter
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Map<String, QFieldMetaData> buildCsvHeaderFieldMappingIfNeeded(InputWrapper inputWrapper, List<String> headers)
+   {
+      Map<String, QFieldMetaData> csvHeaderFieldMapping = null;
+      if(inputWrapper.getCsvHeadersAsFieldNames())
+      {
+         csvHeaderFieldMapping = new HashMap<>();
+         for(String header : headers)
+         {
+            header = adjustHeaderCase(header, inputWrapper);
+            csvHeaderFieldMapping.put(header, new QFieldMetaData(header, QFieldType.STRING));
+         }
+      }
+      return csvHeaderFieldMapping;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -376,7 +419,8 @@ public class CsvToQRecordAdapter
       private Integer                  limit;
       private boolean                  doCorrectValueTypes = false;
 
-      private boolean caseSensitiveHeaders = false;
+      private boolean caseSensitiveHeaders   = false;
+      private boolean csvHeadersAsFieldNames = false;
 
 
 
@@ -618,6 +662,40 @@ public class CsvToQRecordAdapter
 
 
 
+      /*******************************************************************************
+       ** Getter for csvHeadersAsFieldNames
+       **
+       *******************************************************************************/
+      public boolean getCsvHeadersAsFieldNames()
+      {
+         return csvHeadersAsFieldNames;
+      }
+
+
+
+      /*******************************************************************************
+       ** Setter for csvHeadersAsFieldNames
+       **
+       *******************************************************************************/
+      public void setCsvHeadersAsFieldNames(boolean csvHeadersAsFieldNames)
+      {
+         this.csvHeadersAsFieldNames = csvHeadersAsFieldNames;
+      }
+
+
+
+      /*******************************************************************************
+       ** Fluent setter for csvHeadersAsFieldNames
+       **
+       *******************************************************************************/
+      public InputWrapper withCsvHeadersAsFieldNames(boolean csvHeadersAsFieldNames)
+      {
+         this.csvHeadersAsFieldNames = csvHeadersAsFieldNames;
+         return (this);
+      }
+
+
+
       /*******************************************************************************
        ** Getter for doCorrectValueTypes
        **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapter.java

@@ -103,13 +103,23 @@ public class JsonToQRecordAdapter
    {
       QRecord record = new QRecord();
 
-      for(QFieldMetaData field : table.getFields().values())
+      if(table == null)
       {
-         String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
-         // todo - so if the mapping didn't say how to map this field, does that mean we should use the default name for the field?
-         if(jsonObject.has(fieldSource))
+         jsonObject.keys().forEachRemaining(key ->
          {
-            record.setValue(field.getName(), (Serializable) jsonObject.get(fieldSource));
+            record.setValue(key, jsonObject.optString(key));
+         });
+      }
+      else
+      {
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
+            // todo - so if the mapping didn't say how to map this field, does that mean we should use the default name for the field?
+            if(jsonObject.has(fieldSource))
+            {
+               record.setValue(field.getName(), (Serializable) jsonObject.get(fieldSource));
+            }
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -480,6 +480,11 @@ public class QInstanceValidator
                validateTableCustomizer(tableName, entry.getKey(), entry.getValue());
             }
 
+            if(table.getBackendDetails() != null)
+            {
+               table.getBackendDetails().validate(qInstance, table, this);
+            }
+
             validateTableAutomationDetails(qInstance, table);
             validateTableUniqueKeys(table);
             validateAssociatedScripts(table);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QFilterCriteria.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -42,7 +43,7 @@ public class QFilterCriteria implements Serializable, Cloneable
 {
    private static final QLogger LOG = QLogger.getLogger(QFilterCriteria.class);
 
-   private String             fieldName;
+   private String fieldName;
    private QCriteriaOperator  operator;
    private List<Serializable> values;
 
@@ -93,11 +94,37 @@ public class QFilterCriteria implements Serializable, Cloneable
    /*******************************************************************************
     **
     *******************************************************************************/
-   public QFilterCriteria(String fieldName, QCriteriaOperator operator, List<Serializable> values)
+   @SuppressWarnings("unchecked")
+   public QFilterCriteria(String fieldName, QCriteriaOperator operator, List<? extends Serializable> values)
    {
       this.fieldName = fieldName;
       this.operator = operator;
-      this.values = values == null ? new ArrayList<>() : values;
+      this.values = values == null ? new ArrayList<>() : (List<Serializable>) values;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings({ "rawtypes", "unchecked" })
+   public QFilterCriteria(String fieldName, QCriteriaOperator operator, Collection<? extends Serializable> values)
+   {
+      this.fieldName = fieldName;
+      this.operator = operator;
+
+      if(values == null)
+      {
+         this.values = new ArrayList<>();
+      }
+      else if(values instanceof List list)
+      {
+         this.values = list;
+      }
+      else
+      {
+         this.values = new ArrayList<>(values);
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -108,7 +108,7 @@ public class QRecord implements Serializable
 
 
    /*******************************************************************************
-    ** Copy constructor.
+    ** Copy constructor.  Makes a deep clone.
     **
     *******************************************************************************/
    public QRecord(QRecord record)
@@ -120,10 +120,10 @@ public class QRecord implements Serializable
       this.displayValues = deepCopySimpleMap(record.displayValues);
       this.backendDetails = deepCopySimpleMap(record.backendDetails);
 
-      this.associatedRecords = deepCopyAssociatedRecords(record.associatedRecords);
-
       this.errors = record.errors == null ? null : new ArrayList<>(record.errors);
       this.warnings = record.warnings == null ? null : new ArrayList<>(record.warnings);
+
+      this.associatedRecords = deepCopyAssociatedRecords(record.associatedRecords);
    }
 
 
@@ -143,17 +143,17 @@ public class QRecord implements Serializable
     ** todo - move to a cloning utils maybe?
     *******************************************************************************/
    @SuppressWarnings({ "unchecked" })
-   private <K, V> Map<K, V> deepCopySimpleMap(Map<K, V> map)
+   private <V extends Serializable> Map<String, V> deepCopySimpleMap(Map<String, V> map)
    {
       if(map == null)
       {
          return (null);
       }
 
-      Map<K, V> clone = new LinkedHashMap<>();
-      for(Map.Entry<K, V> entry : map.entrySet())
+      Map<String, V> clone = new LinkedHashMap<>();
+      for(Map.Entry<String, V> entry : map.entrySet())
       {
-         V value = entry.getValue();
+         Serializable value = entry.getValue();
 
          //////////////////////////////////////////////////////////////////////////
          // not sure from where/how java.sql.Date objects are getting in here... //
@@ -167,15 +167,27 @@ public class QRecord implements Serializable
             ArrayList<?> cloneList = new ArrayList<>(arrayList);
             clone.put(entry.getKey(), (V) cloneList);
          }
-         else if(entry.getValue() instanceof Serializable serializableValue)
+         else if(entry.getValue() instanceof LinkedHashMap<?, ?> linkedHashMap)
          {
-            LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
-            clone.put(entry.getKey(), (V) SerializationUtils.clone(serializableValue));
+            LinkedHashMap<?, ?> cloneMap = new LinkedHashMap<>(linkedHashMap);
+            clone.put(entry.getKey(), (V) cloneMap);
+         }
+         else if(entry.getValue() instanceof HashMap<?, ?> hashMap)
+         {
+            HashMap<?, ?> cloneMap = new HashMap<>(hashMap);
+            clone.put(entry.getKey(), (V) cloneMap);
+         }
+         else if(entry.getValue() instanceof QRecord otherQRecord)
+         {
+            clone.put(entry.getKey(), (V) new QRecord(otherQRecord));
          }
          else
          {
-            LOG.warn("Non-serializable value in QRecord...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
-            clone.put(entry.getKey(), entry.getValue());
+            //////////////////////////////////////////////////////////////////////////////
+            // we know entry is serializable at this point, based on type param's bound //
+            //////////////////////////////////////////////////////////////////////////////
+            LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
+            clone.put(entry.getKey(), (V) SerializationUtils.clone(entry.getValue()));
          }
       }
       return (clone);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -95,6 +95,7 @@ public class MetaDataProducerHelper
       ////////////////////////////////////////////////////////////////////////////////////////////
       // sort them by sort order, then by the type that they return - specifically - doing apps //
       // after all other types (as apps often try to get other types from the instance)         //
+      // also - do backends earlier than others (e.g., tables may expect backends to exist)     //
       ////////////////////////////////////////////////////////////////////////////////////////////
       producers.sort(Comparator
          .comparing((MetaDataProducerInterface<?> p) -> p.getSortOrder())
@@ -105,11 +106,15 @@ public class MetaDataProducerHelper
                Class<?> outputType = p.getClass().getMethod("produce", QInstance.class).getReturnType();
                if(outputType.equals(QAppMetaData.class))
                {
-                  return (1);
+                  return (2);
+               }
+               else if(outputType.equals(QBackendMetaData.class))
+               {
+                  return (0);
                }
                else
                {
-                  return (0);
+                  return (1);
                }
             }
             catch(Exception e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -1198,4 +1198,14 @@ public class QInstance
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void add(TopLevelMetaDataInterface metaData)
+   {
+      metaData.addSelfToInstance(this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java

@@ -43,6 +43,8 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu
    private String name;
    private String label;
 
+   private Integer sortOrder = 500;
+
    private QPermissionRules permissionRules;
 
    private List<QAppChildMetaData> children;
@@ -426,4 +428,36 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu
    {
       qInstance.addApp(this);
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for sortOrder
+    *******************************************************************************/
+   public Integer getSortOrder()
+   {
+      return (this.sortOrder);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sortOrder
+    *******************************************************************************/
+   public void setSortOrder(Integer sortOrder)
+   {
+      this.sortOrder = sortOrder;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sortOrder
+    *******************************************************************************/
+   public QAppMetaData withSortOrder(Integer sortOrder)
+   {
+      this.sortOrder = sortOrder;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.processes;
 
 
 import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 
@@ -58,6 +59,54 @@ public class AbstractProcessMetaDataBuilder
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for name
+    **
+    *******************************************************************************/
+   public AbstractProcessMetaDataBuilder withName(String name)
+   {
+      processMetaData.setName(name);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for label
+    **
+    *******************************************************************************/
+   public AbstractProcessMetaDataBuilder withLabel(String name)
+   {
+      processMetaData.setLabel(name);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    **
+    *******************************************************************************/
+   public AbstractProcessMetaDataBuilder withTableName(String tableName)
+   {
+      processMetaData.setTableName(tableName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for icon
+    **
+    *******************************************************************************/
+   public AbstractProcessMetaDataBuilder withIcon(QIcon icon)
+   {
+      processMetaData.setIcon(icon);
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java

@@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.serialization.QTableBackendDetailsDeserializer;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
@@ -100,4 +103,16 @@ public abstract class QTableBackendDetails
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
+   {
+      ////////////////////////
+      // noop in base class //
+      ////////////////////////
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java

@@ -34,6 +34,7 @@ import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntityField;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -49,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -57,6 +59,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
  *******************************************************************************/
 public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface
 {
+   private static final QLogger LOG = QLogger.getLogger(QTableMetaData.class);
+
    private String name;
    private String label;
 
@@ -813,6 +817,15 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
     *******************************************************************************/
    public QTableMetaData withUniqueKey(UniqueKey uniqueKey)
    {
+      ////////////////////////////////////////////////////////////////////////////////////
+      // you can't add a null key, so, if someone tried, just gracefully return w/ noop //
+      ////////////////////////////////////////////////////////////////////////////////////
+      if(uniqueKey == null)
+      {
+         LOG.debug("Skipping request to add null uniqueKey", logPair("tableName", name));
+         return (this);
+      }
+
       if(this.uniqueKeys == null)
       {
          this.uniqueKeys = new ArrayList<>();
@@ -1130,6 +1143,15 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
     *******************************************************************************/
    public QTableMetaData withRecordSecurityLock(RecordSecurityLock recordSecurityLock)
    {
+      /////////////////////////////////////////////////////////////////////////////////////
+      // you can't add a null lock, so, if someone tried, just gracefully return w/ noop //
+      /////////////////////////////////////////////////////////////////////////////////////
+      if(recordSecurityLock == null)
+      {
+         LOG.debug("Skipping request to add null recordSecurityLock", logPair("tableName", name));
+         return (this);
+      }
+
       if(this.recordSecurityLocks == null)
       {
          this.recordSecurityLocks = new ArrayList<>();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/QTableAutomationDetails.java

@@ -37,6 +37,11 @@ public class QTableAutomationDetails
 
    private Integer overrideBatchSize;
 
+   private String shardByFieldName; // field in "this" table, to use for sharding
+   private String shardSourceTableName; // name of the table where the shards are defined as rows
+   private String shardLabelFieldName; // field in shard-source-table to use for labeling shards
+   private String shardIdFieldName; // field in shard-source-table to identify shards (e.g., joins to this table's shardByFieldName)
+
 
 
    /*******************************************************************************
@@ -188,4 +193,128 @@ public class QTableAutomationDetails
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shardByFieldName
+    *******************************************************************************/
+   public String getShardByFieldName()
+   {
+      return (this.shardByFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shardByFieldName
+    *******************************************************************************/
+   public void setShardByFieldName(String shardByFieldName)
+   {
+      this.shardByFieldName = shardByFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shardByFieldName
+    *******************************************************************************/
+   public QTableAutomationDetails withShardByFieldName(String shardByFieldName)
+   {
+      this.shardByFieldName = shardByFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shardSourceTableName
+    *******************************************************************************/
+   public String getShardSourceTableName()
+   {
+      return (this.shardSourceTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shardSourceTableName
+    *******************************************************************************/
+   public void setShardSourceTableName(String shardSourceTableName)
+   {
+      this.shardSourceTableName = shardSourceTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shardSourceTableName
+    *******************************************************************************/
+   public QTableAutomationDetails withShardSourceTableName(String shardSourceTableName)
+   {
+      this.shardSourceTableName = shardSourceTableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shardLabelFieldName
+    *******************************************************************************/
+   public String getShardLabelFieldName()
+   {
+      return (this.shardLabelFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shardLabelFieldName
+    *******************************************************************************/
+   public void setShardLabelFieldName(String shardLabelFieldName)
+   {
+      this.shardLabelFieldName = shardLabelFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shardLabelFieldName
+    *******************************************************************************/
+   public QTableAutomationDetails withShardLabelFieldName(String shardLabelFieldName)
+   {
+      this.shardLabelFieldName = shardLabelFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shardIdFieldName
+    *******************************************************************************/
+   public String getShardIdFieldName()
+   {
+      return (this.shardIdFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shardIdFieldName
+    *******************************************************************************/
+   public void setShardIdFieldName(String shardIdFieldName)
+   {
+      this.shardIdFieldName = shardIdFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shardIdFieldName
+    *******************************************************************************/
+   public QTableAutomationDetails withShardIdFieldName(String shardIdFieldName)
+   {
+      this.shardIdFieldName = shardIdFieldName;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/automation/TableAutomationAction.java

@@ -37,6 +37,7 @@ public class TableAutomationAction
    private TriggerEvent triggerEvent;
    private Integer      priority = 500;
    private QQueryFilter filter;
+   private Serializable shardId;
 
    ////////////////////////////////////////////////////////////////////////
    // flag that will cause the records to cause their associations to be //
@@ -329,4 +330,35 @@ public class TableAutomationAction
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shardId
+    *******************************************************************************/
+   public Serializable getShardId()
+   {
+      return (this.shardId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shardId
+    *******************************************************************************/
+   public void setShardId(Serializable shardId)
+   {
+      this.shardId = shardId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shardId
+    *******************************************************************************/
+   public TableAutomationAction withShardId(Serializable shardId)
+   {
+      this.shardId = shardId;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.modules.backend;
 
 
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
@@ -29,6 +30,8 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
 
@@ -126,6 +129,14 @@ public interface QBackendModuleInterface
       return null;
    }
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
+   {
+      return (new QBackendTransaction());
+   }
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/AbstractMemoryAction.java

@@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
 import java.io.Serializable;
-import com.kingsrook.qqq.backend.core.actions.interfaces.QActionInterface;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -32,7 +31,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 /*******************************************************************************
  ** Base class for all core actions in the Memory backend module.
  *******************************************************************************/
-public abstract class AbstractMemoryAction implements QActionInterface
+public abstract class AbstractMemoryAction
 {
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryInsertAction.java

@@ -22,13 +22,10 @@
 package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
 
 
-import java.time.Instant;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 
 /*******************************************************************************
@@ -45,18 +42,6 @@ public class MemoryInsertAction extends AbstractMemoryAction implements InsertIn
    {
       try
       {
-         QTableMetaData table = insertInput.getTable();
-         Instant        now   = Instant.now();
-
-         for(QRecord record : insertInput.getRecords())
-         {
-            ///////////////////////////////////////////
-            // todo .. better (not hard-coded names) //
-            ///////////////////////////////////////////
-            setValueIfTableHasField(record, table, "createDate", now, false);
-            setValueIfTableHasField(record, table, "modifyDate", now, false);
-         }
-
          InsertOutput insertOutput = new InsertOutput();
          insertOutput.setRecords(MemoryRecordStore.getInstance().insert(insertInput, true));
          return (insertOutput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/BasepullConfiguration.java

@@ -40,6 +40,9 @@ public class BasepullConfiguration implements Serializable
 
    private String timestampField; // the name of the field in the table being queried against the last-run timestamp.
 
+   private Integer secondsToSubtractFromLastRunTimeForTimestampQuery; // option to adjust the query's start-time (based on last run time) by a number of seconds.
+   private Integer secondsToSubtractFromThisRunTimeForTimestampQuery; // option to adjust the query's end-time (based on this run time) by a number of seconds.
+
 
 
    /*******************************************************************************
@@ -244,4 +247,66 @@ public class BasepullConfiguration implements Serializable
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for secondsToSubtractFromLastRunTimeForTimestampQuery
+    *******************************************************************************/
+   public Integer getSecondsToSubtractFromLastRunTimeForTimestampQuery()
+   {
+      return (this.secondsToSubtractFromLastRunTimeForTimestampQuery);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for secondsToSubtractFromLastRunTimeForTimestampQuery
+    *******************************************************************************/
+   public void setSecondsToSubtractFromLastRunTimeForTimestampQuery(Integer secondsToSubtractFromLastRunTimeForTimestampQuery)
+   {
+      this.secondsToSubtractFromLastRunTimeForTimestampQuery = secondsToSubtractFromLastRunTimeForTimestampQuery;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for secondsToSubtractFromLastRunTimeForTimestampQuery
+    *******************************************************************************/
+   public BasepullConfiguration withSecondsToSubtractFromLastRunTimeForTimestampQuery(Integer secondsToSubtractFromLastRunTimeForTimestampQuery)
+   {
+      this.secondsToSubtractFromLastRunTimeForTimestampQuery = secondsToSubtractFromLastRunTimeForTimestampQuery;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for secondsToSubtractFromThisRunTimeForTimestampQuery
+    *******************************************************************************/
+   public Integer getSecondsToSubtractFromThisRunTimeForTimestampQuery()
+   {
+      return (this.secondsToSubtractFromThisRunTimeForTimestampQuery);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for secondsToSubtractFromThisRunTimeForTimestampQuery
+    *******************************************************************************/
+   public void setSecondsToSubtractFromThisRunTimeForTimestampQuery(Integer secondsToSubtractFromThisRunTimeForTimestampQuery)
+   {
+      this.secondsToSubtractFromThisRunTimeForTimestampQuery = secondsToSubtractFromThisRunTimeForTimestampQuery;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for secondsToSubtractFromThisRunTimeForTimestampQuery
+    *******************************************************************************/
+   public BasepullConfiguration withSecondsToSubtractFromThisRunTimeForTimestampQuery(Integer secondsToSubtractFromThisRunTimeForTimestampQuery)
+   {
+      this.secondsToSubtractFromThisRunTimeForTimestampQuery = secondsToSubtractFromThisRunTimeForTimestampQuery;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStep.java

@@ -22,6 +22,8 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
 
 
+import java.io.Serializable;
+import java.time.Instant;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -122,7 +124,21 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
     *******************************************************************************/
    protected String getLastRunTimeString(RunBackendStepInput runBackendStepInput) throws QException
    {
-      return (runBackendStepInput.getBasepullLastRunTime().toString());
+      Instant lastRunTime = runBackendStepInput.getBasepullLastRunTime();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // allow the timestamps to be adjusted by the specified number of seconds.                  //
+      // normally this would be a positive value, to move to an earlier time - but it could also  //
+      // be a negative value, if you wanted (for some reason) to move forward in time             //
+      // this is useful to provide overlapping windows of time, in case records are being missed. //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      Serializable basepullConfigurationValue = runBackendStepInput.getValue(RunProcessAction.BASEPULL_CONFIGURATION);
+      if(basepullConfigurationValue instanceof BasepullConfiguration basepullConfiguration && basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery() != null)
+      {
+         lastRunTime = lastRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromLastRunTimeForTimestampQuery());
+      }
+
+      return (lastRunTime.toString());
    }
 
 
@@ -132,6 +148,14 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
     *******************************************************************************/
    protected String getThisRunTimeString(RunBackendStepInput runBackendStepInput) throws QException
    {
-      return (runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY).toString());
+      Instant thisRunTime = runBackendStepInput.getValueInstant(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY);
+
+      Serializable basepullConfigurationValue = runBackendStepInput.getValue(RunProcessAction.BASEPULL_CONFIGURATION);
+      if(basepullConfigurationValue instanceof BasepullConfiguration basepullConfiguration && basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery() != null)
+      {
+         thisRunTime = thisRunTime.minusSeconds(basepullConfiguration.getSecondsToSubtractFromThisRunTimeForTimestampQuery());
+      }
+
+      return (thisRunTime.toString());
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java

@@ -27,7 +27,6 @@ import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.async.AsyncRecordPipeLoop;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
@@ -116,7 +115,7 @@ public class StreamedETLBackendStep implements BackendStep
 
       insertInput.setTableName(runBackendStepInput.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE));
 
-      return new InsertAction().openTransaction(insertInput);
+      return QBackendTransaction.openFor(insertInput);
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaDeleteStep.java

@@ -26,7 +26,6 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@@ -34,7 +33,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 
 
@@ -87,9 +85,8 @@ public class LoadViaDeleteStep extends AbstractLoadStep
    @Override
    public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
    {
-      InsertInput insertInput = new InsertInput();
-      insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
-
-      return (Optional.of(new InsertAction().openTransaction(insertInput)));
+      DeleteInput deleteInput = new DeleteInput();
+      deleteInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+      return (Optional.of(QBackendTransaction.openFor(deleteInput)));
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStep.java

@@ -129,8 +129,7 @@ public class LoadViaInsertOrUpdateStep extends AbstractLoadStep
    {
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
-
-      return (Optional.of(new InsertAction().openTransaction(insertInput)));
+      return (Optional.of(QBackendTransaction.openFor(insertInput)));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertStep.java

@@ -88,7 +88,6 @@ public class LoadViaInsertStep extends AbstractLoadStep
    {
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
-
-      return (Optional.of(new InsertAction().openTransaction(insertInput)));
+      return (Optional.of(QBackendTransaction.openFor(insertInput)));
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaUpdateStep.java

@@ -24,14 +24,12 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit
 
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 
@@ -81,9 +79,8 @@ public class LoadViaUpdateStep extends AbstractLoadStep
    @Override
    public Optional<QBackendTransaction> openTransaction(RunBackendStepInput runBackendStepInput) throws QException
    {
-      InsertInput insertInput = new InsertInput();
-      insertInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
-
-      return (Optional.of(new InsertAction().openTransaction(insertInput)));
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(runBackendStepInput.getValueString(FIELD_DESTINATION_TABLE));
+      return (Optional.of(QBackendTransaction.openFor(updateInput)));
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java

@@ -75,7 +75,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
       InsertAction insertAction = new InsertAction();
       InsertInput  insertInput  = new InsertInput();
       insertInput.setTableName(ScriptRevision.TABLE_NAME);
-      QBackendTransaction transaction = insertAction.openTransaction(insertInput);
+      QBackendTransaction transaction = QBackendTransaction.openFor(insertInput);
       insertInput.setTransaction(transaction);
 
       try

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -182,6 +182,14 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
    public record SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey, boolean performInserts, boolean performUpdates)
    {
 
+      /*******************************************************************************
+       ** Overloaded constructor - defaults both performInserts & performUpdates to true.
+       *******************************************************************************/
+      public SyncProcessConfig(String sourceTable, String sourceTableKeyField, String destinationTable, String destinationTableForeignKey)
+      {
+         this(sourceTable, sourceTableKeyField, destinationTable, destinationTableForeignKey, true, true);
+      }
+
       /*******************************************************************************
        ** artificial method, here to make jacoco see that this class is indeed
        ** included in test coverage...

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java

@@ -120,52 +120,68 @@ public class ScheduleManager
          return;
       }
 
-      for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
+      boolean needToClearContext = false;
+      try
       {
-         startQueueProvider(queueProvider);
-      }
-
-      for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
-      {
-         startAutomationProviderPerTable(automationProvider);
-      }
-
-      for(QProcessMetaData process : qInstance.getProcesses().values())
-      {
-         if(process.getSchedule() != null && allowedToStart(process.getName()))
+         if(QContext.getQInstance() == null)
          {
-            QScheduleMetaData scheduleMetaData = process.getSchedule();
-            if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
+            needToClearContext = true;
+            QContext.init(qInstance, sessionSupplier.get());
+         }
+
+         for(QQueueProviderMetaData queueProvider : qInstance.getQueueProviders().values())
+         {
+            startQueueProvider(queueProvider);
+         }
+
+         for(QAutomationProviderMetaData automationProvider : qInstance.getAutomationProviders().values())
+         {
+            startAutomationProviderPerTable(automationProvider);
+         }
+
+         for(QProcessMetaData process : qInstance.getProcesses().values())
+         {
+            if(process.getSchedule() != null && allowedToStart(process.getName()))
             {
-               ///////////////////////////////////////////////
-               // if no variants, or variant is serial mode //
-               ///////////////////////////////////////////////
-               startProcess(process, null);
-            }
-            else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy()))
-            {
-               /////////////////////////////////////////////////////////////////////////////////////////////////////
-               // if this a "parallel", which for example means we want to have a thread for each backend variant //
-               // running at the same time, get the variant records and schedule each separately                  //
-               /////////////////////////////////////////////////////////////////////////////////////////////////////
-               QContext.init(qInstance, sessionSupplier.get());
-               QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend());
-               for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process)))
+               QScheduleMetaData scheduleMetaData = process.getSchedule();
+               if(process.getSchedule().getVariantBackend() == null || QScheduleMetaData.RunStrategy.SERIAL.equals(process.getSchedule().getVariantRunStrategy()))
                {
-                  try
+                  ///////////////////////////////////////////////
+                  // if no variants, or variant is serial mode //
+                  ///////////////////////////////////////////////
+                  startProcess(process, null);
+               }
+               else if(QScheduleMetaData.RunStrategy.PARALLEL.equals(process.getSchedule().getVariantRunStrategy()))
+               {
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if this a "parallel", which for example means we want to have a thread for each backend variant //
+                  // running at the same time, get the variant records and schedule each separately                  //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////
+                  QBackendMetaData backendMetaData = qInstance.getBackend(scheduleMetaData.getVariantBackend());
+                  for(QRecord qRecord : CollectionUtils.nonNullList(getBackendVariantFilteredRecords(process)))
                   {
-                     startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
-                  }
-                  catch(Exception e)
-                  {
-                     LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord));
+                     try
+                     {
+                        startProcess(process, MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
+                     }
+                     catch(Exception e)
+                     {
+                        LOG.error("An error starting process [" + process.getLabel() + "], with backend variant data.", e, new LogPair("variantQRecord", qRecord));
+                     }
                   }
                }
+               else
+               {
+                  LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided.");
+               }
             }
-            else
-            {
-               LOG.error("Unsupported Schedule Run Strategy [" + process.getSchedule().getVariantRunStrategy() + "] was provided.");
-            }
+         }
+      }
+      finally
+      {
+         if(needToClearContext)
+         {
+            QContext.clear();
          }
       }
    }
@@ -210,8 +226,8 @@ public class ScheduleManager
       // ask the PollingAutomationPerTableRunner how many threads of itself need setup //
       // then start a scheduled executor foreach one                                   //
       ///////////////////////////////////////////////////////////////////////////////////
-      List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName());
-      for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
+      List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, automationProvider.getName());
+      for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
       {
          if(allowedToStart(tableAction.tableName()))
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -574,7 +574,7 @@ public class ValueUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static LocalTime getValueAsLocalTime(Serializable value)
+   public static LocalTime getValueAsLocalTime(Object value)
    {
       try
       {
@@ -615,7 +615,7 @@ public class ValueUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static byte[] getValueAsByteArray(Serializable value)
+   public static byte[] getValueAsByteArray(Object value)
    {
       if(value == null)
       {
@@ -641,7 +641,7 @@ public class ValueUtils
     **
     *******************************************************************************/
    @SuppressWarnings("unchecked")
-   public static <T extends Serializable> T getValueAsType(Class<T> type, Serializable value)
+   public static <T extends Serializable> T getValueAsType(Class<T> type, Object value)
    {
       if(type.equals(Integer.class))
       {
@@ -687,7 +687,7 @@ public class ValueUtils
     **
     *******************************************************************************/
    @SuppressWarnings("checkstyle:indentation")
-   public static Serializable getValueAsFieldType(QFieldType type, Serializable value)
+   public static Serializable getValueAsFieldType(QFieldType type, Object value)
    {
       return switch(type)
       {

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunnerTest.java

@@ -202,8 +202,8 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
     *******************************************************************************/
    private void runAllTableActions(QInstance qInstance) throws QException
    {
-      List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
-      for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
+      List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
+      for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
       {
          PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunner(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction);
 
@@ -504,8 +504,8 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       assertThatThrownBy(() ->
       {
-         List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
-         for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
+         List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
+         for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
          {
             PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(qInstance, TestUtils.POLLING_AUTOMATION, QSession::new, tableAction);
 
@@ -564,7 +564,7 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
       /*******************************************************************************
        **
        *******************************************************************************/
-      public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActions tableActions)
+      public PollingAutomationPerTableRunnerThatShouldSimulateServerShutdownMidRun(QInstance instance, String providerName, Supplier<QSession> sessionSupplier, TableActionsInterface tableActions)
       {
          super(instance, providerName, sessionSupplier, tableActions);
       }
@@ -581,4 +581,16 @@ class PollingAutomationPerTableRunnerTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testLoadingRecordTypesToEnsureClassCoverage()
+   {
+      new PollingAutomationPerTableRunner.TableActions(null, null).noopToFakeTestCoverage();
+      new PollingAutomationPerTableRunner.ShardedTableActions(null, null, null, null, null).noopToFakeTestCoverage();
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/automation/polling/StandardScheduledExecutorTest.java

@@ -186,9 +186,9 @@ class StandardScheduledExecutorTest extends BaseTest
     *******************************************************************************/
    private void runPollingAutomationExecutorForAwhile(QInstance qInstance, Supplier<QSession> sessionSupplier)
    {
-      List<PollingAutomationPerTableRunner.TableActions> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
+      List<PollingAutomationPerTableRunner.TableActionsInterface> tableActions = PollingAutomationPerTableRunner.getTableActions(qInstance, TestUtils.POLLING_AUTOMATION);
       List<StandardScheduledExecutor>                    executors    = new ArrayList<>();
-      for(PollingAutomationPerTableRunner.TableActions tableAction : tableActions)
+      for(PollingAutomationPerTableRunner.TableActionsInterface tableAction : tableActions)
       {
          PollingAutomationPerTableRunner pollingAutomationPerTableRunner = new PollingAutomationPerTableRunner(qInstance, TestUtils.POLLING_AUTOMATION, sessionSupplier, tableAction);
          StandardScheduledExecutor       pollingAutomationExecutor       = new StandardScheduledExecutor(pollingAutomationPerTableRunner);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/QProcessCallbackFactoryTest.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.processes;
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for QProcessCallbackFactory 
+ *******************************************************************************/
+class QProcessCallbackFactoryTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      QProcessCallback qProcessCallback = QProcessCallbackFactory.forFilter(new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.EQUALS, "bar")));
+
+      QQueryFilter queryFilter = qProcessCallback.getQueryFilter();
+      assertEquals(1, queryFilter.getCriteria().size());
+      assertEquals("foo", queryFilter.getCriteria().get(0).getFieldName());
+      assertEquals(QCriteriaOperator.EQUALS, queryFilter.getCriteria().get(0).getOperator());
+      assertEquals("bar", queryFilter.getCriteria().get(0).getValues().get(0));
+
+      assertEquals(Collections.emptyMap(), qProcessCallback.getFieldValues(new ArrayList<>()));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/scripts/RunAssociatedScriptActionTest.java

@@ -156,10 +156,10 @@ class RunAssociatedScriptActionTest extends BaseTest
       /////////////////////////////////////
       assertEquals(N, TestUtils.queryTable(ScriptLog.TABLE_NAME).size());
 
-      ////////////////////////////////////////////////////////////////////////////////////////
-      // and we should have just ran 2 inserts - for the log & logLines (even though empty) //
-      ////////////////////////////////////////////////////////////////////////////////////////
-      assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_INSERTS_RAN));
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // and we should have just ran 1 inserts - for the log (no longer run one for empty insert of 0 log-lines) //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_INSERTS_RAN));
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // and we shouldn't have run N queries (which we would have (at least), if we would have built a new Action object inside the loop) //

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/UpdateActionRecordSplitHelperTest.java

@@ -0,0 +1,147 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for UpdateActionRecordSplitHelper 
+ *******************************************************************************/
+class UpdateActionRecordSplitHelperTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      String tableName = getClass().getSimpleName();
+      QContext.getQInstance().addTable(new QTableMetaData()
+         .withName(tableName)
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("A", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("B", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
+
+      UpdateInput updateInput = new UpdateInput(tableName)
+         .withRecord(new QRecord().withValue("id", 1).withValue("A", 1))
+         .withRecord(new QRecord().withValue("id", 2).withValue("A", 2))
+         .withRecord(new QRecord().withValue("id", 3).withValue("B", 3))
+         .withRecord(new QRecord().withValue("id", 4).withValue("B", 3))
+         .withRecord(new QRecord().withValue("id", 5).withValue("B", 3))
+         .withRecord(new QRecord().withValue("id", 6).withValue("A", 4).withValue("B", 5));
+      UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper();
+      updateActionRecordSplitHelper.init(updateInput);
+      ListingHash<List<String>, QRecord> recordsByFieldBeingUpdated = updateActionRecordSplitHelper.getRecordsByFieldBeingUpdated();
+
+      Function<Collection<QRecord>, Set<Integer>> extractIds = (records) ->
+         records.stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet());
+
+      ////////////////////////////////////////
+      // validate that modify dates got set //
+      ////////////////////////////////////////
+      updateInput.getRecords().forEach(r ->
+         assertThat(r.getValue("modifyDate")).isInstanceOf(Instant.class));
+
+      //////////////////////////////////////////////////////////////
+      // validate the grouping of records by fields-being-updated //
+      //////////////////////////////////////////////////////////////
+      assertEquals(3, recordsByFieldBeingUpdated.size());
+      assertEquals(Set.of(1, 2), extractIds.apply(recordsByFieldBeingUpdated.get(List.of("A", "modifyDate"))));
+      assertEquals(Set.of(3, 4, 5), extractIds.apply(recordsByFieldBeingUpdated.get(List.of("B", "modifyDate"))));
+      assertEquals(Set.of(6), extractIds.apply(recordsByFieldBeingUpdated.get(List.of("A", "B", "modifyDate"))));
+
+      ///////////////////////////////////////////////////////////////////
+      // validate the output records were built, in the order expected //
+      ///////////////////////////////////////////////////////////////////
+      List<QRecord> outputRecords = updateActionRecordSplitHelper.getOutputRecords();
+      for(int i = 0; i < outputRecords.size(); i++)
+      {
+         assertEquals(i + 1, outputRecords.get(i).getValueInteger("id"));
+      }
+
+      /////////////////////////////////////////////////////
+      // test the areAllValuesBeingUpdatedTheSame method //
+      /////////////////////////////////////////////////////
+      Function<List<String>, Boolean> runAreAllValuesBeingUpdatedTheSame = (fields) ->
+         UpdateActionRecordSplitHelper.areAllValuesBeingUpdatedTheSame(updateInput, recordsByFieldBeingUpdated.get(fields), fields);
+
+      assertFalse(runAreAllValuesBeingUpdatedTheSame.apply(List.of("A", "modifyDate")));
+      assertTrue(runAreAllValuesBeingUpdatedTheSame.apply(List.of("B", "modifyDate")));
+      assertTrue(runAreAllValuesBeingUpdatedTheSame.apply(List.of("A", "B", "modifyDate")));
+
+      ////////////////////////////////////////////////////////////////////
+      // make sure that the override of the logic for this method works //
+      ////////////////////////////////////////////////////////////////////
+      updateInput.setAreAllValuesBeingUpdatedTheSame(true);
+      assertTrue(runAreAllValuesBeingUpdatedTheSame.apply(List.of("A", "modifyDate")));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRecordsWithErrors()
+   {
+      String tableName = getClass().getSimpleName() + "WithErrors";
+      QContext.getQInstance().addTable(new QTableMetaData()
+         .withName(tableName)
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("A", QFieldType.INTEGER)));
+
+      {
+         UpdateInput updateInput = new UpdateInput(tableName)
+            .withRecord(new QRecord().withValue("id", 1).withValue("A", 1).withError(new SystemErrorStatusMessage("error")))
+            .withRecord(new QRecord().withValue("id", 2).withValue("A", 2).withError(new SystemErrorStatusMessage("error")))
+            .withRecord(new QRecord().withValue("id", 2).withValue("A", 3).withError(new SystemErrorStatusMessage("error")));
+         UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper();
+         updateActionRecordSplitHelper.init(updateInput);
+         assertFalse(updateActionRecordSplitHelper.getHaveAnyWithoutErrors());
+      }
+
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java

@@ -465,4 +465,60 @@ class CsvToQRecordAdapterTest extends BaseTest
       assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer.");
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCsvHeadersAsFields() throws QException
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+         .withCsvHeadersAsFieldNames(true)
+         .withCaseSensitiveHeaders(true)
+         .withCsv("""
+            firstName,birthDate,favoriteShapeId
+            John,1980,1
+            Paul,1970-06-15,green
+            """));
+
+      List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
+
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("John", qRecord.getValue("firstName"));
+      assertEquals("1980", qRecord.getValue("birthDate"));
+      assertEquals("1", qRecord.getValue("favoriteShapeId"));
+
+      qRecord = qRecords.get(1);
+      assertEquals("Paul", qRecord.getValue("firstName"));
+      assertEquals("1970-06-15", qRecord.getValue("birthDate"));
+      assertEquals("green", qRecord.getValue("favoriteShapeId"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCsvHeadersAsFieldsDuplicatedNames() throws QException
+   {
+      CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+      csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+         .withCsvHeadersAsFieldNames(true)
+         .withCaseSensitiveHeaders(true)
+         .withCsv("""
+            orderId,sku,sku
+            10001,BASIC1,BASIC2
+            """));
+
+      List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
+
+      QRecord qRecord = qRecords.get(0);
+      assertEquals("10001", qRecord.getValue("orderId"));
+      assertEquals("BASIC1", qRecord.getValue("sku"));
+      assertEquals("BASIC2", qRecord.getValue("sku 2"));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/JsonToQRecordAdapterTest.java

@@ -164,6 +164,29 @@ class JsonToQRecordAdapterTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_buildRecordsFromJsonWithoutTable_inputList()
+   {
+      JsonToQRecordAdapter jsonToQRecordAdapter = new JsonToQRecordAdapter();
+      List<QRecord> qRecords = jsonToQRecordAdapter.buildRecordsFromJson("""
+         [
+            { "firstName":"Tyler", "last":"Samples" },
+            { "firstName":"Tim", "lastName":"Chamberlain" }
+         ]
+         """, null, null);
+      assertNotNull(qRecords);
+      assertEquals(2, qRecords.size());
+      assertEquals("Tyler", qRecords.get(0).getValue("firstName"));
+      assertEquals("Samples", qRecords.get(0).getValue("last"));
+      assertEquals("Tim", qRecords.get(1).getValue("firstName"));
+      assertEquals("Chamberlain", qRecords.get(1).getValue("lastName"));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.data;
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -33,6 +34,7 @@ import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.Test;
 import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS;
 import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -147,6 +149,34 @@ class QRecordTest extends BaseTest
       QRecord byteArrayValue = new QRecord().withValue("myBytes", new byte[] { 65, 66, 67, 68 });
       assertArrayEquals(new byte[] { 65, 66, 67, 68 }, new QRecord(byteArrayValue).getValueByteArray("myBytes"));
 
+      ////////////////////////////////////////////
+      // qrecord as a value inside another (!?) //
+      ////////////////////////////////////////////
+      QRecord nestedQRecordValue     = new QRecord().withValue("myRecord", new QRecord().withValue("A", 1));
+      QRecord cloneWithNestedQRecord = new QRecord(nestedQRecordValue);
+      assertEquals(1, ((QRecord) cloneWithNestedQRecord.getValue("myRecord")).getValueInteger("A"));
+      assertNotSame(cloneWithNestedQRecord.getValue("myRecord"), nestedQRecordValue.getValue("myRecord"));
+
+      QRecord emptyRecord = new QRecord();
+      QRecord emptyClone  = new QRecord(emptyRecord);
+      assertNull(emptyClone.getTableName());
+      assertNull(emptyClone.getRecordLabel());
+      assertEquals(0, emptyClone.getValues().size());
+      assertEquals(0, emptyClone.getDisplayValues().size());
+      assertEquals(0, emptyClone.getBackendDetails().size());
+      assertEquals(0, emptyClone.getErrors().size());
+      assertEquals(0, emptyClone.getWarnings().size());
+      assertEquals(0, emptyClone.getAssociatedRecords().size());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testListAsValue()
+   {
       ArrayList<Integer> originalArrayList        = new ArrayList<>(List.of(1, 2, 3));
       QRecord            recordWithArrayListValue = new QRecord().withValue("myList", originalArrayList);
       QRecord            cloneWithArrayListValue  = new QRecord(recordWithArrayListValue);
@@ -162,17 +192,36 @@ class QRecordTest extends BaseTest
       //////////////////////////////////////////////////////////////////////////////////////////////////////
       originalArrayList.add(4);
       assertNotEquals(originalArrayList, cloneWithArrayListValue.getValue("myList"));
+   }
 
-      QRecord emptyRecord = new QRecord();
-      QRecord emptyClone  = new QRecord(emptyRecord);
-      assertNull(emptyClone.getTableName());
-      assertNull(emptyClone.getRecordLabel());
-      assertEquals(0, emptyClone.getValues().size());
-      assertEquals(0, emptyClone.getDisplayValues().size());
-      assertEquals(0, emptyClone.getBackendDetails().size());
-      assertEquals(0, emptyClone.getErrors().size());
-      assertEquals(0, emptyClone.getWarnings().size());
-      assertEquals(0, emptyClone.getAssociatedRecords().size());
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMapAsValue()
+   {
+      LinkedHashMap<String, Integer> originalMap        = new LinkedHashMap<>(Map.of("one", 1, "two", 2, "three", 3));
+      QRecord                        recordWithMapValue = new QRecord().withValue("myMap", originalMap);
+      QRecord                        cloneWithMapValue  = new QRecord(recordWithMapValue);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // the clone map and original map should be equals (have contents that are equals), but not be the same (reference) //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(originalMap, cloneWithMapValue.getValue("myMap"));
+      assertNotSame(originalMap, cloneWithMapValue.getValue("myMap"));
+
+      //////////////////////////////////////////////////////////
+      // make sure we re-created it as the same subtype (LHM) //
+      //////////////////////////////////////////////////////////
+      assertThat(cloneWithMapValue.getValue("myMap")).isInstanceOf(LinkedHashMap.class);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure a change to the original list doesn't change the cloned list (as it was cloned deeply) //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      originalMap.put("four", 4);
+      assertNotEquals(originalMap, cloneWithMapValue.getValue("myMap"));
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/basepull/ExtractViaBasepullQueryStepTest.java

@@ -97,4 +97,87 @@ class ExtractViaBasepullQueryStepTest extends BaseTest
          .withValues(Map.of("queryFilterJson", "{}"))));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSubtractingSeconds() throws QException
+   {
+      String originalLastRunTime       = "2023-12-28T15:00:00Z";
+      String lastRunTimeMinusOneMinute = "2023-12-28T14:59:00Z";
+
+      String originalThisRunTime        = "2023-12-28T15:05:00Z";
+      String thisRunTimePlusFiveSeconds = "2023-12-28T15:05:05Z";
+
+      ///////////////////////////
+      // cases for lastRunTime //
+      ///////////////////////////
+      {
+         ///////////////////////////////////////////////////////////////////////////////
+         // confirm we don't fail (and don't subtract) if config is absent from input //
+         ///////////////////////////////////////////////////////////////////////////////
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.setBasepullLastRunTime(Instant.parse(originalLastRunTime));
+         String lastRunTimeString = new ExtractViaBasepullQueryStep().getLastRunTimeString(input);
+         assertEquals(originalLastRunTime, lastRunTimeString);
+      }
+      {
+         //////////////////////////////////////////////////////////////////////////////////
+         // confirm we don't fail or subtract if secondsToSubtract isn't given in config //
+         //////////////////////////////////////////////////////////////////////////////////
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.setBasepullLastRunTime(Instant.parse(originalLastRunTime));
+         input.addValue(RunProcessAction.BASEPULL_CONFIGURATION, new BasepullConfiguration());
+         String lastRunTimeString = new ExtractViaBasepullQueryStep().getLastRunTimeString(input);
+         assertEquals(originalLastRunTime, lastRunTimeString);
+      }
+      {
+         ///////////////////////////////////////////////////////////////////////
+         // confirm we do subtract if a subtract value is given in the config //
+         ///////////////////////////////////////////////////////////////////////
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.setBasepullLastRunTime(Instant.parse(originalLastRunTime));
+         input.addValue(RunProcessAction.BASEPULL_CONFIGURATION, new BasepullConfiguration()
+            .withSecondsToSubtractFromLastRunTimeForTimestampQuery(60));
+         String lastRunTimeString = new ExtractViaBasepullQueryStep().getLastRunTimeString(input);
+         assertEquals(lastRunTimeMinusOneMinute, lastRunTimeString);
+      }
+
+      ///////////////////////////
+      // cases for thisRunTime //
+      ///////////////////////////
+      {
+         ///////////////////////////////////////////////////////////////////////////////
+         // confirm we don't fail (and don't subtract) if config is absent from input //
+         ///////////////////////////////////////////////////////////////////////////////
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.addValue(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY, originalThisRunTime);
+         String thisRunTimeString = new ExtractViaBasepullQueryStep().getThisRunTimeString(input);
+         assertEquals(originalThisRunTime, thisRunTimeString);
+      }
+      {
+         //////////////////////////////////////////////////////////////////////////////////
+         // confirm we don't fail or subtract if secondsToSubtract isn't given in config //
+         //////////////////////////////////////////////////////////////////////////////////
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.addValue(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY, originalThisRunTime);
+         input.addValue(RunProcessAction.BASEPULL_CONFIGURATION, new BasepullConfiguration());
+         String thisRunTimeString = new ExtractViaBasepullQueryStep().getThisRunTimeString(input);
+         assertEquals(originalThisRunTime, thisRunTimeString);
+      }
+      {
+         ///////////////////////////////////////////////////////////////////////
+         // confirm we do subtract if a subtract value is given in the config //
+         ///////////////////////////////////////////////////////////////////////
+         RunBackendStepInput input = new RunBackendStepInput();
+         input.addValue(RunProcessAction.BASEPULL_THIS_RUNTIME_KEY, originalThisRunTime);
+         input.addValue(RunProcessAction.BASEPULL_CONFIGURATION, new BasepullConfiguration()
+            .withSecondsToSubtractFromThisRunTimeForTimestampQuery(-5));
+         String thisRunTimeString = new ExtractViaBasepullQueryStep().getThisRunTimeString(input);
+         assertEquals(thisRunTimePlusFiveSeconds, thisRunTimeString);
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GarbageCollectorTest.java

@@ -74,7 +74,7 @@ class GarbageCollectorTest extends BaseTest
    @Test
    void testBasic() throws QException
    {
-      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
+      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "timestamp", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
       QContext.getQInstance().addProcess(process);
 
       new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
@@ -97,11 +97,11 @@ class GarbageCollectorTest extends BaseTest
    private static List<QRecord> getPersonRecords()
    {
       List<QRecord> records = List.of(
-         new QRecord().withValue("id", 1).withValue("createDate", Instant.now().minus(90, ChronoUnit.DAYS)),
-         new QRecord().withValue("id", 2).withValue("createDate", Instant.now().minus(31, ChronoUnit.DAYS)),
-         new QRecord().withValue("id", 3).withValue("createDate", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
-         new QRecord().withValue("id", 4).withValue("createDate", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
-         new QRecord().withValue("id", 5).withValue("createDate", Instant.now().minus(5, ChronoUnit.DAYS)));
+         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
+         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
+         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
       return records;
    }
 
@@ -113,7 +113,7 @@ class GarbageCollectorTest extends BaseTest
    @Test
    void testOverrideDate() throws QException
    {
-      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
+      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_PERSON_MEMORY, "timestamp", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
       QContext.getQInstance().addProcess(process);
 
       new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
@@ -157,7 +157,7 @@ class GarbageCollectorTest extends BaseTest
    @Test
    void testWithDeleteAllJoins() throws QException
    {
-      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), "*");
+      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "timestamp", NowWithOffset.minus(30, ChronoUnit.DAYS), "*");
       QContext.getQInstance().addProcess(process);
 
       QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE_ALL_ACCESS, true);
@@ -192,7 +192,7 @@ class GarbageCollectorTest extends BaseTest
    @Test
    void testWithDeleteSomeJoins() throws QException
    {
-      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), TestUtils.TABLE_NAME_LINE_ITEM);
+      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "timestamp", NowWithOffset.minus(30, ChronoUnit.DAYS), TestUtils.TABLE_NAME_LINE_ITEM);
       QContext.getQInstance().addProcess(process);
 
       //////////////////////////////////////////////////////////////////////////
@@ -232,7 +232,7 @@ class GarbageCollectorTest extends BaseTest
    @Test
    void testWithDeleteNoJoins() throws QException
    {
-      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "createDate", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
+      QProcessMetaData process = GarbageCollectorProcessMetaDataProducer.createProcess(TestUtils.TABLE_NAME_ORDER, "timestamp", NowWithOffset.minus(30, ChronoUnit.DAYS), null);
       QContext.getQInstance().addProcess(process);
 
       ////////////////////////////////////////////////////////////////////////////////
@@ -270,11 +270,11 @@ class GarbageCollectorTest extends BaseTest
    private static List<QRecord> getOrderRecords()
    {
       List<QRecord> records = List.of(
-         new QRecord().withValue("id", 1).withValue("createDate", Instant.now().minus(90, ChronoUnit.DAYS)),
-         new QRecord().withValue("id", 2).withValue("createDate", Instant.now().minus(31, ChronoUnit.DAYS)),
-         new QRecord().withValue("id", 3).withValue("createDate", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
-         new QRecord().withValue("id", 4).withValue("createDate", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
-         new QRecord().withValue("id", 5).withValue("createDate", Instant.now().minus(5, ChronoUnit.DAYS)));
+         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
+         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
+         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
+         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
       return records;
    }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -549,7 +549,9 @@ public class TestUtils
          .withField(new QFieldMetaData("cost", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
          .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
          .withField(new QFieldMetaData("ssn", QFieldType.STRING).withType(QFieldType.PASSWORD))
-         .withField(new QFieldMetaData("superSecret", QFieldType.STRING).withType(QFieldType.PASSWORD).withIsHidden(true));
+         .withField(new QFieldMetaData("superSecret", QFieldType.STRING).withType(QFieldType.PASSWORD).withIsHidden(true))
+         .withField(new QFieldMetaData("timestamp", QFieldType.DATE_TIME)) // adding this for GC tests, so we can set a date-time (since CD & MD are owned by system)
+         ;
    }
 
 
@@ -602,6 +604,7 @@ public class TestUtils
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("timestamp", QFieldType.DATE_TIME)) // adding this for GC tests, so we can set a date-time (since CD & MD are owned by system)
          .withField(new QFieldMetaData("orderId", QFieldType.INTEGER))
          .withField(new QFieldMetaData("lineNumber", QFieldType.STRING))
          .withField(new QFieldMetaData("sku", QFieldType.STRING).withLabel("SKU"))

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -189,12 +189,6 @@ public class BaseAPIActionUtil
       InsertOutput insertOutput = new InsertOutput();
       insertOutput.setRecords(new ArrayList<>());
 
-      if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
-      {
-         LOG.debug("Insert request called with 0 records.  Returning with no-op");
-         return (insertOutput);
-      }
-
       try
       {
          // todo - supports bulk post?

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
@@ -52,6 +53,7 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinali
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.NotImplementedException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -205,15 +207,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
          List<FILE>                            files        = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
 
+         int recordCount = 0;
+
+         FILE_LOOP:
          for(FILE file : files)
          {
-            LOG.info("Processing file: " + getFullPathForFile(file));
-
             InputStream inputStream = readFile(file);
             switch(tableDetails.getCardinality())
             {
                case MANY:
                {
+                  LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
                   switch(tableDetails.getRecordFormat())
                   {
                      case CSV:
@@ -260,14 +264,33 @@ public abstract class AbstractBaseFilesystemAction<FILE>
                }
                case ONE:
                {
+                  ////////////////////////////////////////////////////////////////////////////////
+                  // for one-record tables, put the entire file's contents into a single record //
+                  ////////////////////////////////////////////////////////////////////////////////
                   String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
+                  byte[] bytes               = inputStream.readAllBytes();
 
-                  byte[] bytes = inputStream.readAllBytes();
                   QRecord record = new QRecord()
                      .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
                      .withValue(tableDetails.getContentsFieldName(), bytes);
                   queryOutput.addRecord(record);
 
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  recordCount++;
+
+                  ////////////////////////////////////////////////////////////////////////////
+                  // break out of the file loop if we have hit the limit (if one was given) //
+                  ////////////////////////////////////////////////////////////////////////////
+                  if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null)
+                  {
+                     if(recordCount >= queryInput.getFilter().getLimit())
+                     {
+                        break FILE_LOOP;
+                     }
+                  }
+
                   break;
                }
                default:
@@ -374,6 +397,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          QTableMetaData   table   = insertInput.getTable();
          QBackendMetaData backend = insertInput.getBackend();
 
+         output.setRecords(new ArrayList<>());
+
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
          if(tableDetails.getCardinality().equals(Cardinality.ONE))
          {

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java

@@ -22,7 +22,11 @@
 package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
 
 
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
 /*******************************************************************************
@@ -35,11 +39,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
    private RecordFormat recordFormat;
    private Cardinality  cardinality;
 
-   ///////////////////////////////////////////////////////////////////////////////////////////////////
-   // todo default these to null, and give validation error if not set for a cardinality=ONE table? //
-   ///////////////////////////////////////////////////////////////////////////////////////////////////
-   private String contentsFieldName = "contents";
-   private String fileNameFieldName = "fileName";
+   private String contentsFieldName;
+   private String fileNameFieldName;
+
 
 
    /*******************************************************************************
@@ -243,4 +245,40 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
    }
 
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
+   {
+      super.validate(qInstance, table, qInstanceValidator);
+
+      String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
+      if(qInstanceValidator.assertCondition(cardinality != null, prefix + "missing cardinality"))
+      {
+         if(cardinality.equals(Cardinality.ONE))
+         {
+            if(qInstanceValidator.assertCondition(StringUtils.hasContent(contentsFieldName), prefix + "missing contentsFieldName, which is required for Cardinality ONE"))
+            {
+               qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(contentsFieldName), prefix + "contentsFieldName [" + contentsFieldName + "] is not a field on this table.");
+            }
+
+            if(qInstanceValidator.assertCondition(StringUtils.hasContent(fileNameFieldName), prefix + "missing fileNameFieldName, which is required for Cardinality ONE"))
+            {
+               qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(fileNameFieldName), prefix + "fileNameFieldName [" + fileNameFieldName + "] is not a field on this table.");
+            }
+
+            qInstanceValidator.assertCondition(recordFormat == null, prefix + "has a recordFormat, which is not allowed for Cardinality ONE");
+         }
+
+         if(cardinality.equals(Cardinality.MANY))
+         {
+            qInstanceValidator.assertCondition(!StringUtils.hasContent(contentsFieldName), prefix + "has a contentsFieldName, which is not allowed for Cardinality MANY");
+            qInstanceValidator.assertCondition(!StringUtils.hasContent(fileNameFieldName), prefix + "has a fileNameFieldName, which is not allowed for Cardinality MANY");
+            qInstanceValidator.assertCondition(recordFormat != null, prefix + "missing recordFormat, which is required for Cardinality MANY");
+         }
+      }
+
+   }
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java

@@ -0,0 +1,210 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+
+
+/*******************************************************************************
+ ** Builder class to create standard style QTableMetaData for tables in filesystem
+ ** modules (avoid some boilerplate).
+ **
+ ** e.g., lets us create a file-based table like so:
+ <pre>
+ QTableMetaData table = new FilesystemTableMetaDataBuilder()
+ .withName("myTableName")
+ .withBackend(qInstance.getBackend("myBackendName"))
+ .withGlob("*.csv")
+ .withBasePath("/")
+ .buildStandardCardinalityOneTable();
+ </pre>
+ *******************************************************************************/
+public class FilesystemTableMetaDataBuilder
+{
+   private String           name;
+   private QBackendMetaData backend;
+   private String           basePath;
+   private String           glob;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   public QTableMetaData buildStandardCardinalityOneTable()
+   {
+      AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType())
+      {
+         case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails();
+         case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails();
+         default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType());
+      };
+
+      return new QTableMetaData()
+         .withName(name)
+         .withIsHidden(true)
+         .withBackendName(backend.getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("contents", QFieldType.STRING))
+         .withBackendDetails(tableBackendDetails
+            .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
+            .withBasePath(basePath)
+            .withGlob(glob));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backend
+    *******************************************************************************/
+   public QBackendMetaData getBackend()
+   {
+      return (this.backend);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backend
+    *******************************************************************************/
+   public void setBackend(QBackendMetaData backend)
+   {
+      this.backend = backend;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backend
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withBackend(QBackendMetaData backend)
+   {
+      this.backend = backend;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getName()
+   {
+      return (this.name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    *******************************************************************************/
+   public void setName(String name)
+   {
+      this.name = name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for name
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withName(String name)
+   {
+      this.name = name;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for basePath
+    *******************************************************************************/
+   public String getBasePath()
+   {
+      return (this.basePath);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for basePath
+    *******************************************************************************/
+   public void setBasePath(String basePath)
+   {
+      this.basePath = basePath;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for basePath
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withBasePath(String basePath)
+   {
+      this.basePath = basePath;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for glob
+    *******************************************************************************/
+   public String getGlob()
+   {
+      return (this.glob);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for glob
+    *******************************************************************************/
+   public void setGlob(String glob)
+   {
+      this.glob = glob;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for glob
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withGlob(String glob)
+   {
+      this.glob = glob;
+      return (this);
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java

@@ -0,0 +1,137 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.utils;
+
+
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+
+
+/*******************************************************************************
+ ** utility methods shared by s3 & local-filesystem utils classes
+ *******************************************************************************/
+public class SharedFilesystemBackendModuleUtils
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static boolean doesFilePathMatchFilter(String filePath, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
+   {
+      if(filter == null || !filter.hasAnyCriteria())
+      {
+         return (true);
+      }
+
+      if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
+      {
+         ///////////////////////////////
+         // todo - well, we could ... //
+         ///////////////////////////////
+         throw (new QException("Filters with sub-filters are not supported for querying filesystems at this time."));
+      }
+
+      Path path = Path.of(URI.create("file:///" + filePath));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QFilterCriteria criteria : filter.getCriteria())
+      {
+         boolean matches = doesFilePathMatchOneCriteria(criteria, tableDetails, path);
+
+         if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
+         {
+            ////////////////////////////////////////////////////////////////////////////////
+            // if it's not a match, and it's an AND filter, then the whole thing is false //
+            ////////////////////////////////////////////////////////////////////////////////
+            return (false);
+         }
+
+         if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+         {
+            ////////////////////////////////////////////////////////////
+            // if it's an OR filter, and we've a match, return a true //
+            ////////////////////////////////////////////////////////////
+            return (true);
+         }
+      }
+
+      //////////////////////////////////////////////////////////////////////
+      // if we didn't return above, return now                            //
+      // for an OR - if we didn't find something true, then return false. //
+      // else, an AND - if we didn't find a false, we can return true.    //
+      //////////////////////////////////////////////////////////////////////
+      if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
+      {
+         return (false);
+      }
+
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesFilePathMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
+   {
+      if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
+      {
+         if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
+         {
+            return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
+         }
+         else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
+         {
+            boolean anyMatch = false;
+            for(int i = 0; i < criteria.getValues().size(); i++)
+            {
+               if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
+               {
+                  anyMatch = true;
+                  break;
+               }
+            }
+
+            return (anyMatch);
+         }
+         else
+         {
+            throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
+         }
+      }
+      else
+      {
+         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+      }
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModule.java

@@ -51,6 +51,7 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys
 {
    private static final QLogger LOG = QLogger.getLogger(FilesystemBackendModule.class);
 
+   public static final String BACKEND_TYPE = "filesystem";
 
 
    /*******************************************************************************
@@ -71,7 +72,7 @@ public class FilesystemBackendModule implements QBackendModuleInterface, Filesys
    @Override
    public String getBackendType()
    {
-      return ("filesystem");
+      return (BACKEND_TYPE);
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -23,36 +23,32 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
 import java.io.File;
-import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
-import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOCase;
-import org.apache.commons.io.filefilter.AndFileFilter;
-import org.apache.commons.io.filefilter.NameFileFilter;
-import org.apache.commons.io.filefilter.OrFileFilter;
-import org.apache.commons.io.filefilter.TrueFileFilter;
-import org.apache.commons.io.filefilter.WildcardFileFilter;
 
 
 /*******************************************************************************
@@ -70,71 +66,69 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
    @Override
    public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
    {
-      String fullPath  = getFullBasePath(table, backendBase);
-      File   directory = new File(fullPath);
-      File[] files     = null;
-
-      AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
-
-      FileFilter fileFilter = TrueFileFilter.INSTANCE;
-
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // if each file is its own record (ONE), then we may need to do filtering of the directory listing based on the input filter //
-      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(Cardinality.ONE.equals(tableBackendDetails.getCardinality()))
+      try
       {
-         if(filter != null && filter.hasAnyCriteria())
+         String fullPath  = getFullBasePath(table, backendBase);
+         File   directory = new File(fullPath);
+
+         AbstractFilesystemTableBackendDetails tableBackendDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+
+         String pattern = "regex:.*";
+         if(StringUtils.hasContent(tableBackendDetails.getGlob()))
          {
-            List<FileFilter> fileFilterList = new ArrayList<>();
-            for(QFilterCriteria criteria : filter.getCriteria())
-            {
-               if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
-               {
-                  if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
-                  {
-                     fileFilterList.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(0))));
-                  }
-                  else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
-                  {
-                     List<NameFileFilter> nameInFilters = new ArrayList<>();
-                     for(int i = 0; i < criteria.getValues().size(); i++)
-                     {
-                        nameInFilters.add(new NameFileFilter(ValueUtils.getValueAsString(criteria.getValues().get(i))));
-                     }
-                     fileFilterList.add(new OrFileFilter(nameInFilters));
-                  }
-                  else
-                  {
-                     throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
-                  }
-               }
-               else
-               {
-                  throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
-               }
-            }
-
-            fileFilter = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? new AndFileFilter(fileFilterList) : new OrFileFilter(fileFilterList);
+            pattern = "glob:" + tableBackendDetails.getGlob();
          }
-      }
+         List<String> matchedFiles = recursivelyListFilesMatchingPattern(directory.toPath(), pattern, backendBase, table);
+         List<File>   rs           = new ArrayList<>();
 
-      ///////////////////////////////////////////////////////////////////////////////////////////
-      // if the table has a glob specified, add it as an AND to the filter built to this point //
-      ///////////////////////////////////////////////////////////////////////////////////////////
-      if(StringUtils.hasContent(tableBackendDetails.getGlob()))
+         for(String matchedFile : matchedFiles)
+         {
+            if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
+            {
+               rs.add(new File(fullPath + File.separatorChar + matchedFile));
+            }
+         }
+
+         return (rs);
+      }
+      catch(Exception e)
       {
-         WildcardFileFilter globFilenameFilter = new WildcardFileFilter(tableBackendDetails.getGlob(), IOCase.INSENSITIVE);
-         fileFilter = new AndFileFilter(List.of(globFilenameFilter, fileFilter));
+         throw (new QException("Error searching files", e));
       }
+   }
 
-      files = directory.listFiles(fileFilter);
 
-      if(files == null)
+
+   /*******************************************************************************
+    ** Credit: https://www.baeldung.com/java-files-match-wildcard-strings
+    *******************************************************************************/
+   List<String> recursivelyListFilesMatchingPattern(Path rootDir, String pattern, QBackendMetaData backend, QTableMetaData table) throws IOException
+   {
+      List<String> matchesList = new ArrayList<>();
+
+      FileVisitor<Path> matcherVisitor = new SimpleFileVisitor<>()
       {
-         return Collections.emptyList();
+         @Override
+         public FileVisitResult visitFile(Path file, BasicFileAttributes attribs)
+         {
+            FileSystem  fs      = FileSystems.getDefault();
+            PathMatcher matcher = fs.getPathMatcher(pattern);
+            Path        path    = Path.of(stripBackendAndTableBasePathsFromFileName(file.toAbsolutePath().toString(), backend, table));
+
+            if(matcher.matches(path))
+            {
+               matchesList.add(path.toString());
+            }
+            return FileVisitResult.CONTINUE;
+         }
+      };
+
+      if(rootDir.toFile().exists())
+      {
+         Files.walkFileTree(rootDir, matcherVisitor);
       }
 
-      return (Arrays.stream(files).filter(File::isFile).toList());
+      return matchesList;
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterMetaDataTemplate.java

@@ -0,0 +1,504 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.util.List;
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
+
+
+/*******************************************************************************
+ ** Class to serve as a template for producing an instance of a process & tables
+ ** that provide the QQQ service to manage importing files (e.g., partner feeds on S3).
+ **
+ ** The template contains the following components:
+ ** - A process that loads files from a source-table (e.g., of filesystem, cardinality=ONE)
+ **   and stores them in the following tables:
+ ** - {baseName}importFile table - simple header for imported files.
+ ** - {baseName}importRecord table - a record foreach record in an imported file.
+ ** - PVS for the importFile table
+ ** - Join & Widget (to show importRecords on importFile view screen)
+ **
+ ** Most likely one would add all the meta-data objects in an instance of this
+ ** template, then either use tableAutomations or a basepull process against records
+ ** in the importRecord table, to run through a process (e.g., an AbstractTableSync)
+ ** to result in final values for your business case.
+ **
+ ** A typical usage may look like:
+ **
+ ** <pre>
+ // set up the process that'll be used to import the files.
+ FilesystemImporterProcessMetaDataBuilder importerProcessBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
+ .withFileFormat("csv")
+ .withSourceTableName(MyFeedSourceTableMetaDataProducer.NAME)
+ .withRemoveFileAfterImport(true)
+ .withUpdateFileIfNameExists(false)
+ .withName("myFeedImporter")
+ .withSchedule(new QScheduleMetaData().withRepeatSeconds(300));
+
+ FilesystemImporterMetaDataTemplate template = new FilesystemImporterMetaDataTemplate(qInstance, "myFeed", MongoDBMetaDataProducer.NAME, importerProcessBuilder, table ->
+ {
+ // whatever customizations you may need on the tables
+ table.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED));
+ });
+
+ // set up automations on the table
+ template.addAutomationStatusField(template.getImportRecordTable(), getStandardAutomationStatusField().withBackendName("metaData.automationStatus"));
+ template.addStandardPostInsertAutomation(template.getImportRecordTable(), getBasicTableAutomationDetails(), "myFeedTableSyncProcess");
+
+ // finally, add all the meta-data from the template to a QInstance
+ template.addToInstance(qInstance);
+ </pre>
+ **
+ *******************************************************************************/
+public class FilesystemImporterMetaDataTemplate
+{
+   public static final String IMPORT_FILE_TABLE_SUFFIX       = "ImportFile";
+   public static final String IMPORT_RECORD_TABLE_SUFFIX     = "ImportRecord";
+   public static final String IMPORT_FILE_RECORD_JOIN_SUFFIX = "ImportFileImportRecordJoin";
+
+   private QTableMetaData                           importFileTable;
+   private QTableMetaData                           importRecordTable;
+   private QPossibleValueSource                     importFilePVS;
+   private QJoinMetaData                            importFileImportRecordJoin;
+   private QWidgetMetaDataInterface                 importFileImportRecordJoinWidget;
+   private FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate(QInstance qInstance, String importBaseName, String backendName, FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder, Consumer<QTableMetaData> tableEnricher)
+   {
+      QBackendMetaData backend = qInstance.getBackend(backendName);
+
+      this.importFileTable = defineTableImportFile(backend, importBaseName);
+      this.importRecordTable = defineTableImportRecord(backend, importBaseName);
+
+      for(QTableMetaData table : List.of(this.importFileTable, this.importRecordTable))
+      {
+         table.setBackendName(backendName);
+         if(tableEnricher != null)
+         {
+            tableEnricher.accept(table);
+         }
+      }
+
+      this.importFilePVS = QPossibleValueSource.newForTable(this.importFileTable.getName());
+
+      this.importFileImportRecordJoin = defineImportFileImportRecordJoin(importBaseName);
+      this.importFileImportRecordJoinWidget = defineImportFileImportRecordChildWidget(this.importFileImportRecordJoin);
+
+      this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder
+         .withImportFileTable(this.importFileTable.getName())
+         .withImportRecordTable(this.importRecordTable.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAutomationStatusField(QTableMetaData table, QFieldMetaData automationStatusField)
+   {
+      table.addField(automationStatusField);
+      table.getSections().get(1).getFieldNames().add(0, automationStatusField.getName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public TableAutomationAction addStandardPostInsertAutomation(QTableMetaData table, QTableAutomationDetails automationDetails, String processName)
+   {
+      TableAutomationAction action = new TableAutomationAction()
+         .withName(table.getName() + "PostInsert")
+         .withTriggerEvent(TriggerEvent.POST_INSERT)
+         .withProcessName(processName);
+
+      table.withAutomationDetails(automationDetails
+         .withAction(action));
+
+      return (action);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QWidgetMetaDataInterface defineImportFileImportRecordChildWidget(QJoinMetaData join)
+   {
+      return ChildRecordListRenderer.widgetMetaDataBuilder(join)
+         .withName(join.getName())
+         .withLabel("Import Records")
+         .withMaxRows(100)
+         .withCanAddChildRecord(false)
+         .getWidgetMetaData();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QJoinMetaData defineImportFileImportRecordJoin(String importBaseName)
+   {
+      return new QJoinMetaData()
+         .withLeftTable(importBaseName + IMPORT_FILE_TABLE_SUFFIX)
+         .withRightTable(importBaseName + IMPORT_RECORD_TABLE_SUFFIX)
+         .withName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX)
+         .withType(JoinType.ONE_TO_MANY)
+         .withJoinOn(new JoinOn("id", "importFileId"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData defineTableImportFile(QBackendMetaData backend, String importBaseName)
+   {
+      QFieldType idType = getIdFieldType(backend);
+
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName(importBaseName + IMPORT_FILE_TABLE_SUFFIX)
+         .withIcon(new QIcon().withName("upload_file"))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("sourceFileName")
+         .withPrimaryKeyField("id")
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
+
+         .withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
+         .withField(new QFieldMetaData("sourceFileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("archivedPath", QFieldType.STRING))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
+
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "sourceFileName", "archivedPath")))
+         .withSection(new QFieldSection("records", new QIcon().withName("power_input"), Tier.T2).withWidgetName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
+
+         .withAssociation(new Association().withName("importRecords").withJoinName(importBaseName + IMPORT_FILE_RECORD_JOIN_SUFFIX).withAssociatedTableName(importBaseName + IMPORT_RECORD_TABLE_SUFFIX));
+
+      return (qTableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QFieldType getIdFieldType(QBackendMetaData backend)
+   {
+      QFieldType idType = QFieldType.INTEGER;
+      if("mongodb".equals(backend.getBackendType()))
+      {
+         idType = QFieldType.STRING;
+      }
+      return idType;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String getIdFieldBackendName(QBackendMetaData backend)
+   {
+      if("mongodb".equals(backend.getBackendType()))
+      {
+         return ("_id");
+      }
+      return (null);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QTableMetaData defineTableImportRecord(QBackendMetaData backend, String importBaseName)
+   {
+      QFieldType idType = getIdFieldType(backend);
+
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName(importBaseName + IMPORT_RECORD_TABLE_SUFFIX)
+         .withIcon(new QIcon().withName("power_input"))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("importFileId", "recordNo")
+         .withPrimaryKeyField("id")
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD))
+         .withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(ImportRecordPostQueryCustomizer.class))
+
+         .withField(new QFieldMetaData("id", idType).withIsEditable(false).withBackendName(getIdFieldBackendName(backend)))
+
+         .withField(new QFieldMetaData("importFileId", idType).withBackendName("metaData.importFileId")
+            .withPossibleValueSourceName(importBaseName + IMPORT_FILE_TABLE_SUFFIX))
+         .withField(new QFieldMetaData("recordNo", QFieldType.INTEGER).withBackendName("metaData.recordNo"))
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // so, we'll use this field as a "virtual" field, e.g., populated with JSON in table post-query customizer, with all un-structured values //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         .withField(new QFieldMetaData("values", QFieldType.TEXT)
+            .withIsEditable(false)
+            .withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR)
+               .withValue(AdornmentType.CodeEditorValues.languageMode("json"))))
+
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("metaData.createDate").withIsEditable(false))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("metaData.modifyDate").withIsEditable(false))
+
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "importFileId", "recordNo")))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("values")))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+
+      return (qTableMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addToInstance(QInstance instance)
+   {
+      instance.add(importFileTable);
+      instance.add(importRecordTable);
+      instance.add(importFilePVS);
+      instance.add(importFileImportRecordJoin);
+      instance.add(importFileImportRecordJoinWidget);
+      instance.add(importerProcessMetaDataBuilder.getProcessMetaData());
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFileTable
+    *******************************************************************************/
+   public QTableMetaData getImportFileTable()
+   {
+      return (this.importFileTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFileTable
+    *******************************************************************************/
+   public void setImportFileTable(QTableMetaData importFileTable)
+   {
+      this.importFileTable = importFileTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFileTable
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFileTable(QTableMetaData importFileTable)
+   {
+      this.importFileTable = importFileTable;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importRecordTable
+    *******************************************************************************/
+   public QTableMetaData getImportRecordTable()
+   {
+      return (this.importRecordTable);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importRecordTable
+    *******************************************************************************/
+   public void setImportRecordTable(QTableMetaData importRecordTable)
+   {
+      this.importRecordTable = importRecordTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importRecordTable
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportRecordTable(QTableMetaData importRecordTable)
+   {
+      this.importRecordTable = importRecordTable;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFilePVS
+    *******************************************************************************/
+   public QPossibleValueSource getImportFilePVS()
+   {
+      return (this.importFilePVS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFilePVS
+    *******************************************************************************/
+   public void setImportFilePVS(QPossibleValueSource importFilePVS)
+   {
+      this.importFilePVS = importFilePVS;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFilePVS
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFilePVS(QPossibleValueSource importFilePVS)
+   {
+      this.importFilePVS = importFilePVS;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFileImportRecordJoin
+    *******************************************************************************/
+   public QJoinMetaData getImportFileImportRecordJoin()
+   {
+      return (this.importFileImportRecordJoin);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFileImportRecordJoin
+    *******************************************************************************/
+   public void setImportFileImportRecordJoin(QJoinMetaData importFileImportRecordJoin)
+   {
+      this.importFileImportRecordJoin = importFileImportRecordJoin;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFileImportRecordJoin
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFileImportRecordJoin(QJoinMetaData importFileImportRecordJoin)
+   {
+      this.importFileImportRecordJoin = importFileImportRecordJoin;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importFileImportRecordJoinWidget
+    *******************************************************************************/
+   public QWidgetMetaDataInterface getImportFileImportRecordJoinWidget()
+   {
+      return (this.importFileImportRecordJoinWidget);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importFileImportRecordJoinWidget
+    *******************************************************************************/
+   public void setImportFileImportRecordJoinWidget(QWidgetMetaDataInterface importFileImportRecordJoinWidget)
+   {
+      this.importFileImportRecordJoinWidget = importFileImportRecordJoinWidget;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importFileImportRecordJoinWidget
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImportFileImportRecordJoinWidget(QWidgetMetaDataInterface importFileImportRecordJoinWidget)
+   {
+      this.importFileImportRecordJoinWidget = importFileImportRecordJoinWidget;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for importerProcessMetaDataBuilder
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder getImporterProcessMetaDataBuilder()
+   {
+      return (this.importerProcessMetaDataBuilder);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for importerProcessMetaDataBuilder
+    *******************************************************************************/
+   public void setImporterProcessMetaDataBuilder(FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder)
+   {
+      this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for importerProcessMetaDataBuilder
+    *******************************************************************************/
+   public FilesystemImporterMetaDataTemplate withImporterProcessMetaDataBuilder(FilesystemImporterProcessMetaDataBuilder importerProcessMetaDataBuilder)
+   {
+      this.importerProcessMetaDataBuilder = importerProcessMetaDataBuilder;
+      return (this);
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterProcessMetaDataBuilder.java

@@ -0,0 +1,189 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.AbstractProcessMetaDataBuilder;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+
+
+/*******************************************************************************
+ ** Process MetaData Builder for FilesystemImporter process.
+ ** Meant to be used with (and actually is a parameter to the constructor of)
+ ** {@link FilesystemImporterMetaDataTemplate}
+ *******************************************************************************/
+public class FilesystemImporterProcessMetaDataBuilder extends AbstractProcessMetaDataBuilder
+{
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder()
+   {
+      super(new QProcessMetaData()
+         .addStep(new QBackendStepMetaData()
+            .withName("sync")
+            .withCode(new QCodeReference(FilesystemImporterStep.class))
+            .withInputData(new QFunctionInputMetaData()
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_SOURCE_TABLE, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_FILE_FORMAT, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_FILE_TABLE, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_RECORD_TABLE, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_REMOVE_FILE_AFTER_IMPORT, QFieldType.BOOLEAN).withDefaultValue(true))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_UPDATE_FILE_IF_NAME_EXISTS, QFieldType.BOOLEAN).withDefaultValue(false))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, QFieldType.BOOLEAN).withDefaultValue(false))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_ARCHIVE_PATH, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, QFieldType.STRING))
+               .withField(new QFieldMetaData(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, QFieldType.STRING))
+            )));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withSourceTableName(String sourceTableName)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_SOURCE_TABLE, sourceTableName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withFileFormat(String fileFormat)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_FILE_FORMAT, fileFormat);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportFileTable(String importFileTable)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_FILE_TABLE, importFileTable);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportRecordTable(String importRecordTable)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_RECORD_TABLE, importRecordTable);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withRemoveFileAfterImport(boolean removeFileAfterImport)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_REMOVE_FILE_AFTER_IMPORT, removeFileAfterImport);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withUpdateFileIfNameExists(boolean updateFileIfNameExists)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_UPDATE_FILE_IF_NAME_EXISTS, updateFileIfNameExists);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withArchiveFileEnabled(boolean archiveFileEnabled)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_FILE_ENABLED, archiveFileEnabled);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withArchiveTableName(String archiveTableName)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_TABLE_NAME, archiveTableName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withArchivePath(String archivePath)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_ARCHIVE_PATH, archivePath);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldName(String securityFieldName)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME, securityFieldName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public FilesystemImporterProcessMetaDataBuilder withImportSecurityFieldValue(Serializable securityFieldValue)
+   {
+      setInputFieldDefaultValue(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE, securityFieldValue);
+      return (this);
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -0,0 +1,403 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
+import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** BackendStep for FilesystemImporter process
+ **
+ ** Job is to:
+ ** - foreach file in the `source` table (e.g., a ONE-type filesystem table):
+ **   - optionally create an archive/backup copy of the file
+ **   - create a record in the `importFile` table
+ **   - parse the file, creating many records in the `importRecord` table
+ **   - remove the file from the `source` (if so configured (e.g., may turn off for Read-only FS))
+ *******************************************************************************/
+@SuppressWarnings("unchecked")
+public class FilesystemImporterStep implements BackendStep
+{
+   private static final QLogger LOG = QLogger.getLogger(FilesystemImporterStep.class);
+
+   public static final String FIELD_SOURCE_TABLE        = "sourceTable";
+   public static final String FIELD_FILE_FORMAT         = "fileFormat";
+   public static final String FIELD_IMPORT_FILE_TABLE   = "importFileTable";
+   public static final String FIELD_IMPORT_RECORD_TABLE = "importRecordTable";
+
+   public static final String FIELD_IMPORT_SECURITY_FIELD_NAME  = "importSecurityFieldName";
+   public static final String FIELD_IMPORT_SECURITY_FIELD_VALUE = "importSecurityFieldValue";
+
+   public static final String FIELD_ARCHIVE_FILE_ENABLED     = "archiveFileEnabled";
+   public static final String FIELD_ARCHIVE_TABLE_NAME       = "archiveTableName";
+   public static final String FIELD_ARCHIVE_PATH             = "archivePath";
+   public static final String FIELD_REMOVE_FILE_AFTER_IMPORT = "removeFileAfterImport";
+
+   public static final String FIELD_UPDATE_FILE_IF_NAME_EXISTS = "updateFileIfNameExists";
+
+
+
+   /*******************************************************************************
+    ** Execute the step - using the request as input, and the result as output.
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // defer to a private method here, so we can add a type-parameter for that method to use              //
+      // would think we could do that here, but get compiler error, since this method comes from base class //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////
+      doRun(runBackendStepInput, runBackendStepOutput);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <F> void doRun(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      String  fileFormat             = runBackendStepInput.getValueString(FIELD_FILE_FORMAT);
+      Boolean removeFileAfterImport  = runBackendStepInput.getValueBoolean(FIELD_REMOVE_FILE_AFTER_IMPORT);
+      Boolean updateFileIfNameExists = runBackendStepInput.getValueBoolean(FIELD_UPDATE_FILE_IF_NAME_EXISTS);
+      Boolean archiveFileEnabled     = runBackendStepInput.getValueBoolean(FIELD_ARCHIVE_FILE_ENABLED);
+
+      QTableMetaData sourceTable     = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_SOURCE_TABLE));
+      QTableMetaData importFileTable = runBackendStepInput.getInstance().getTable(runBackendStepInput.getValueString(FIELD_IMPORT_FILE_TABLE));
+
+      String missingFieldErrorPrefix = "Process " + runBackendStepInput.getProcessName() + " was misconfigured - missing value in field: ";
+      Objects.requireNonNull(fileFormat, missingFieldErrorPrefix + FIELD_FILE_FORMAT);
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // list files in the backend system                                              //
+      // todo - can we do this using query action, with this being a "ONE" type table? //
+      ///////////////////////////////////////////////////////////////////////////////////
+      QBackendMetaData                    sourceBackend    = runBackendStepInput.getInstance().getBackendForTable(sourceTable.getName());
+      FilesystemBackendModuleInterface<F> sourceModule     = (FilesystemBackendModuleInterface<F>) new QBackendModuleDispatcher().getQBackendModule(sourceBackend);
+      AbstractBaseFilesystemAction<F>     sourceActionBase = sourceModule.getActionBase();
+      sourceActionBase.preAction(sourceBackend);
+      Map<String, F> sourceFiles = getFileNames(sourceActionBase, sourceTable, sourceBackend);
+
+      if(CollectionUtils.nullSafeIsEmpty(sourceFiles))
+      {
+         LOG.debug("No files found in import filesystem", logPair("sourceTable", sourceTable));
+         return;
+      }
+
+      ////////////////////////////////////////////////////////
+      // look up any existing file records with those names //
+      ////////////////////////////////////////////////////////
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(importFileTable.getName());
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("sourceFileName", QCriteriaOperator.IN, sourceFiles.keySet())));
+
+      QueryOutput               queryOutput           = new QueryAction().execute(queryInput);
+      Map<String, Serializable> existingImportedFiles = CollectionUtils.listToMap(queryOutput.getRecords(), r -> r.getValueString("sourceFileName"), r -> r.getValue("id"));
+
+      for(Map.Entry<String, F> sourceEntry : sourceFiles.entrySet())
+      {
+         QBackendTransaction transaction = null;
+         try
+         {
+            String sourceFileName = sourceEntry.getKey();
+
+            /////////////////////////////////////////////////////////
+            // if filename was already imported, decide what to do //
+            /////////////////////////////////////////////////////////
+            boolean      alreadyImported = existingImportedFiles.containsKey(sourceFileName);
+            Serializable idToUpdate      = null;
+            if(alreadyImported)
+            {
+               //////////////////////////////////////////////////////////////////////////////////
+               // todo - would we want to support importing multiple-times the same file name? //
+               // possibly - if so, add it here, presumably w/ another boolean field           //
+               //////////////////////////////////////////////////////////////////////////////////
+               if(updateFileIfNameExists)
+               {
+                  LOG.info("Updating already-imported file", logPair("fileName", sourceFileName), logPair("id", idToUpdate));
+                  idToUpdate = existingImportedFiles.get(sourceFileName);
+               }
+               else
+               {
+                  LOG.debug("Skipping already-imported file", logPair("fileName", sourceFileName));
+                  removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
+                  continue;
+               }
+            }
+
+            ///////////////////////////////////
+            // read the file as input stream //
+            ///////////////////////////////////
+            try(InputStream inputStream = sourceActionBase.readFile(sourceEntry.getValue()))
+            {
+               byte[] bytes = inputStream.readAllBytes();
+
+               //////////////////////////////////////
+               // archive the file, if so directed //
+               //////////////////////////////////////
+               String archivedPath = null;
+               if(archiveFileEnabled)
+               {
+                  archivedPath = archiveFile(runBackendStepInput, sourceFileName, bytes);
+               }
+
+               /////////////////////////////////
+               // build record for importFile //
+               /////////////////////////////////
+               LOG.info("Syncing file [" + sourceFileName + "]");
+               QRecord importFileRecord = new QRecord()
+                  .withValue("id", idToUpdate)
+                  .withValue("sourceFileName", sourceFileName)
+                  .withValue("archivedPath", archivedPath);
+
+               addSecurityValue(runBackendStepInput, importFileRecord);
+
+               //////////////////////////////////////
+               // build child importRecord records //
+               //////////////////////////////////////
+               String content = new String(bytes);
+               importFileRecord.withAssociatedRecords("importRecords", parseFileIntoRecords(runBackendStepInput, content));
+
+               ///////////////////////////////////////////////////////////////////
+               // insert the file & records (records as association under file) //
+               ///////////////////////////////////////////////////////////////////
+               InsertAction insertAction = new InsertAction();
+               InsertInput  insertInput  = new InsertInput();
+               insertInput.setTableName(importFileTable.getName());
+               insertInput.setRecords(List.of(importFileRecord));
+
+               transaction = QBackendTransaction.openFor(insertInput);
+               insertInput.setTransaction(transaction);
+
+               InsertOutput insertOutput = insertAction.execute(insertInput);
+
+               LOG.info("Inserted insertFile & records", logPair("id", insertOutput.getRecords().get(0).getValue("id")));
+
+               transaction.commit();
+            }
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            // after the records are built, we can delete the file                                       //
+            // if we are interrupted between the commit & the delete, then the file will be found again, //
+            // and we'll either skip it or do an update, based on FIELD_UPDATE_FILE_IF_NAME_EXISTS flag  //
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            removeSourceFileIfSoConfigured(removeFileAfterImport, sourceActionBase, sourceTable, sourceBackend, sourceFileName);
+         }
+         catch(Exception e)
+         {
+            LOG.error("Error processing file: " + sourceEntry, e);
+            if(transaction != null)
+            {
+               transaction.rollback();
+            }
+         }
+         finally
+         {
+            if(transaction != null)
+            {
+               transaction.close();
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** if the process is configured w/ a security field & value, set it on the import
+    ** File & Record records.
+    *******************************************************************************/
+   private void addSecurityValue(RunBackendStepInput runBackendStepInput, QRecord record)
+   {
+      String       securityField = runBackendStepInput.getValueString(FIELD_IMPORT_SECURITY_FIELD_NAME);
+      Serializable securityValue = runBackendStepInput.getValue(FIELD_IMPORT_SECURITY_FIELD_VALUE);
+
+      if(StringUtils.hasContent(securityField) && securityValue != null)
+      {
+         record.setValue(securityField, securityValue);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
+   {
+      if(removeFileAfterImport)
+      {
+         String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
+         sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String archiveFile(RunBackendStepInput runBackendStepInput, String sourceFileName, byte[] bytes) throws QException, IOException
+   {
+      String         archiveTableName = runBackendStepInput.getValueString(FIELD_ARCHIVE_TABLE_NAME);
+      QTableMetaData archiveTable;
+      try
+      {
+         archiveTable = runBackendStepInput.getInstance().getTable(archiveTableName);
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error getting archive table [" + archiveTableName + "]", e));
+      }
+
+      String archivePath = Objects.requireNonNullElse(runBackendStepInput.getValueString(FIELD_ARCHIVE_PATH), "");
+
+      QBackendMetaData                    archiveBackend    = runBackendStepInput.getInstance().getBackendForTable(archiveTable.getName());
+      FilesystemBackendModuleInterface<?> archiveModule     = (FilesystemBackendModuleInterface<?>) new QBackendModuleDispatcher().getQBackendModule(archiveBackend);
+      AbstractBaseFilesystemAction<?>     archiveActionBase = archiveModule.getActionBase();
+      archiveActionBase.preAction(archiveBackend);
+
+      LocalDateTime now = LocalDateTime.now();
+      String path = archiveActionBase.getFullBasePath(archiveTable, archiveBackend)
+         + File.separator + archivePath
+         + File.separator + now.getYear()
+         + File.separator + now.getMonth()
+         + File.separator + UUID.randomUUID()
+         + "-" + sourceFileName.replaceAll(".*" + File.separator, "");
+      path = AbstractBaseFilesystemAction.stripDuplicatedSlashes(path);
+
+      LOG.info("Archiving file", logPair("path", path));
+      archiveActionBase.writeFile(archiveBackend, path, bytes);
+
+      return (path);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   List<QRecord> parseFileIntoRecords(RunBackendStepInput runBackendStepInput, String content) throws QException
+   {
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // first, parse the content into records, w/ unknown field names - just whatever is in the CSV or JSON //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////
+      String fileFormat = runBackendStepInput.getValueString(FIELD_FILE_FORMAT);
+
+      List<QRecord> contentRecords = switch(fileFormat.toLowerCase())
+      {
+         case "csv" ->
+         {
+            CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
+            csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
+               .withCsv(content)
+               .withCaseSensitiveHeaders(true)
+               .withCsvHeadersAsFieldNames(true)
+            );
+            yield (csvToQRecordAdapter.getRecordList());
+         }
+
+         case "json" -> new JsonToQRecordAdapter().buildRecordsFromJson(content, null, null);
+
+         default -> throw (new QException("Unexpected file format: " + fileFormat));
+      };
+
+      /////////////////////////////////////////////////////////////////////////////
+      // now, add some fields that we know about to those records, for returning //
+      /////////////////////////////////////////////////////////////////////////////
+      List<QRecord> importRecordList = new ArrayList<>();
+      int           recordNo         = 1;
+      for(QRecord record : contentRecords)
+      {
+         record.setValue("recordNo", recordNo++);
+         addSecurityValue(runBackendStepInput, record);
+
+         importRecordList.add(record);
+      }
+
+      return (importRecordList);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private <F> Map<String, F> getFileNames(AbstractBaseFilesystemAction<F> actionBase, QTableMetaData table, QBackendMetaData backend) throws QException
+   {
+      List<F> files = actionBase.listFiles(table, backend);
+
+      /////////////////////////////////////////////////////
+      // use a tree map, so files will be sorted by name //
+      /////////////////////////////////////////////////////
+      Map<String, F> rs = new TreeMap<>();
+
+      for(F file : files)
+      {
+         String fileName = actionBase.stripBackendAndTableBasePathsFromFileName(actionBase.getFullPathForFile(file), backend, table);
+         rs.put(fileName, file);
+      }
+
+      return (rs);
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizer.java

@@ -0,0 +1,82 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPostQueryCustomizer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** combine all unstructured fields of the record into a JSON blob in the "values" field.
+ *******************************************************************************/
+public class ImportRecordPostQueryCustomizer extends AbstractPostQueryCustomizer
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public List<QRecord> apply(List<QRecord> records)
+   {
+      if(CollectionUtils.nullSafeHasContents(records))
+      {
+         QTableMetaData table = null;
+         if(StringUtils.hasContent(records.get(0).getTableName()))
+         {
+            table = QContext.getQInstance().getTable(records.get(0).getTableName());
+         }
+
+         for(QRecord record : records)
+         {
+            Map<String, Serializable> values = record.getValues();
+
+            if(table != null)
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // remove known values from a clone of the values map - then only put the un-structured values in a JSON document in the values field //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               values = new HashMap<>(values);
+               for(String fieldName : table.getFields().keySet())
+               {
+                  values.remove(fieldName);
+               }
+            }
+
+            String valuesJson = JsonUtils.toJson(values);
+            record.setValue("values", valuesJson);
+         }
+      }
+
+      return (records);
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBack
  *******************************************************************************/
 public class S3BackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface
 {
+   public static final String BACKEND_TYPE = "s3";
 
 
    /*******************************************************************************
@@ -66,7 +67,7 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke
    @Override
    public String getBackendType()
    {
-      return ("s3");
+      return (BACKEND_TYPE);
    }
 
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3Utils.java

@@ -42,9 +42,9 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.utils.SharedFilesystemBackendModuleUtils;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.local.actions.AbstractFilesystemAction;
 
@@ -106,6 +106,10 @@ public class S3Utils
          useQQueryFilter = true;
       }
 
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. //
+      // as this will be a common case.                                                                                             //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       if(filter != null && useQQueryFilter)
       {
          if(filter.getCriteria() != null && filter.getCriteria().size() == 1)
@@ -113,7 +117,15 @@ public class S3Utils
             QFilterCriteria criteria = filter.getCriteria().get(0);
             if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS))
             {
-               prefix += "/" + criteria.getValues().get(0);
+               if(!prefix.isEmpty())
+               {
+                  ///////////////////////////////////////////////////////
+                  // remember, a prefix starting with / finds nothing! //
+                  ///////////////////////////////////////////////////////
+                  prefix += "/";
+               }
+
+               prefix += criteria.getValues().get(0);
             }
          }
       }
@@ -194,12 +206,24 @@ public class S3Utils
             ///////////////////////////////////////////////////////////////////////////////////
             // if we're a file-per-record table, and we have a filter, compare the key to it //
             ///////////////////////////////////////////////////////////////////////////////////
-            if(!doesObjectKeyMatchFilter(key, filter, tableDetails))
+            if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails))
             {
                continue;
             }
 
             rs.add(objectSummary);
+
+            /////////////////////////////////////////////////////////////////
+            // if we have a limit, and we've hit it, break out of the loop //
+            /////////////////////////////////////////////////////////////////
+            if(filter != null && useQQueryFilter && filter.getLimit() != null)
+            {
+               if(rs.size() >= filter.getLimit())
+               {
+                  break;
+               }
+            }
+
          }
       }
       while(listObjectsV2Result.isTruncated());
@@ -209,95 +233,6 @@ public class S3Utils
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private boolean doesObjectKeyMatchFilter(String key, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException
-   {
-      if(filter == null || !filter.hasAnyCriteria())
-      {
-         return (true);
-      }
-
-      Path path = Path.of(URI.create("file:///" + key));
-
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      // foreach criteria, build a pathmatcher (or many, for an in-list), and check if the file matches //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////
-      for(QFilterCriteria criteria : filter.getCriteria())
-      {
-         boolean matches = doesObjectKeyMatchOneCriteria(criteria, tableDetails, path);
-
-         if(!matches && QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()))
-         {
-            ////////////////////////////////////////////////////////////////////////////////
-            // if it's not a match, and it's an AND filter, then the whole thing is false //
-            ////////////////////////////////////////////////////////////////////////////////
-            return (false);
-         }
-
-         if(matches && QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
-         {
-            ////////////////////////////////////////////////////////////
-            // if it's an OR filter, and we've a match, return a true //
-            ////////////////////////////////////////////////////////////
-            return (true);
-         }
-      }
-
-      //////////////////////////////////////////////////////////////////////
-      // if we didn't return above, return now                            //
-      // for an OR - if we didn't find something true, then return false. //
-      // else, an AND - if we didn't find a false, we can return true.    //
-      //////////////////////////////////////////////////////////////////////
-      if(QQueryFilter.BooleanOperator.OR.equals(filter.getBooleanOperator()))
-      {
-         return (false);
-      }
-
-      return (true);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static boolean doesObjectKeyMatchOneCriteria(QFilterCriteria criteria, AbstractFilesystemTableBackendDetails tableBackendDetails, Path path) throws QException
-   {
-      if(tableBackendDetails.getFileNameFieldName().equals(criteria.getFieldName()))
-      {
-         if(QCriteriaOperator.EQUALS.equals(criteria.getOperator()) && CollectionUtils.nonNullList(criteria.getValues()).size() == 1)
-         {
-            return (FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(0)).matches(path));
-         }
-         else if(QCriteriaOperator.IN.equals(criteria.getOperator()) && !CollectionUtils.nonNullList(criteria.getValues()).isEmpty())
-         {
-            boolean anyMatch = false;
-            for(int i = 0; i < criteria.getValues().size(); i++)
-            {
-               if(FileSystems.getDefault().getPathMatcher("glob:**/" + criteria.getValues().get(i)).matches(path))
-               {
-                  anyMatch = true;
-                  break;
-               }
-            }
-
-            return (anyMatch);
-         }
-         else
-         {
-            throw (new QException("Unable to query filename field using operator: " + criteria.getOperator()));
-         }
-      }
-      else
-      {
-         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
-      }
-   }
-
-
-
    /*******************************************************************************
     ** Get the contents (as an InputStream) for an object in s3
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java

@@ -26,10 +26,10 @@ import java.io.File;
 import java.io.IOException;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -39,12 +39,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.streamed.StreamedETLFilesystemBackendStep;
+import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterMetaDataTemplate;
+import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer.FilesystemImporterProcessMetaDataBuilder;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
@@ -56,18 +59,23 @@ import org.apache.commons.io.FileUtils;
  *******************************************************************************/
 public class TestUtils
 {
-   public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
-   public static final String BACKEND_NAME_S3       = "s3";
-   public static final String BACKEND_NAME_MOCK     = "mock";
+   public static final String BACKEND_NAME_LOCAL_FS       = "local-filesystem";
+   public static final String BACKEND_NAME_S3             = "s3";
+   public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix";
+   public static final String BACKEND_NAME_MOCK           = "mock";
+   public static final String BACKEND_NAME_MEMORY = "memory";
 
    public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
    public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV  = "person-local-csv";
    public static final String TABLE_NAME_BLOB_LOCAL_FS        = "local-blob";
+   public static final String TABLE_NAME_ARCHIVE_LOCAL_FS    = "local-archive";
    public static final String TABLE_NAME_PERSON_S3            = "person-s3";
    public static final String TABLE_NAME_BLOB_S3              = "s3-blob";
    public static final String TABLE_NAME_PERSON_MOCK          = "person-mock";
+   public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix";
 
-   public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
+   public static final String PROCESS_NAME_STREAMED_ETL                   = "etl.streamed";
+   public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter";
 
    ///////////////////////////////////////////////////////////////////
    // shouldn't be accessed directly, as we append a counter to it. //
@@ -135,20 +143,44 @@ public class TestUtils
       qInstance.addTable(defineLocalFilesystemJSONPersonTable());
       qInstance.addTable(defineLocalFilesystemCSVPersonTable());
       qInstance.addTable(defineLocalFilesystemBlobTable());
+      qInstance.addTable(defineLocalFilesystemArchiveTable());
       qInstance.addBackend(defineS3Backend());
+      qInstance.addBackend(defineS3BackendSansPrefix());
       qInstance.addTable(defineS3CSVPersonTable());
       qInstance.addTable(defineS3BlobTable());
+      qInstance.addTable(defineS3BlobSansPrefixTable());
       qInstance.addBackend(defineMockBackend());
+      qInstance.addBackend(defineMemoryBackend());
       qInstance.addTable(defineMockPersonTable());
       qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
 
-      new QInstanceValidator().validate(qInstance);
+      definePersonCsvImporter(qInstance);
 
       return (qInstance);
    }
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void definePersonCsvImporter(QInstance qInstance)
+   {
+      String importBaseName = "personImporter";
+      FilesystemImporterProcessMetaDataBuilder filesystemImporterProcessMetaDataBuilder = (FilesystemImporterProcessMetaDataBuilder) new FilesystemImporterProcessMetaDataBuilder()
+         .withSourceTableName(TABLE_NAME_PERSON_LOCAL_FS_CSV)
+         .withFileFormat("csv")
+         .withArchiveFileEnabled(true)
+         .withArchiveTableName(TABLE_NAME_ARCHIVE_LOCAL_FS)
+         .withArchivePath("archive-of/personImporterFiles")
+         .withName(LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+
+      FilesystemImporterMetaDataTemplate filesystemImporterMetaDataTemplate = new FilesystemImporterMetaDataTemplate(qInstance, importBaseName, BACKEND_NAME_MEMORY, filesystemImporterProcessMetaDataBuilder, table -> table.withAuditRules(QAuditRules.defaultInstanceLevelNone()));
+      filesystemImporterMetaDataTemplate.addToInstance(qInstance);
+   }
+
+
+
    /*******************************************************************************
     ** Define the authentication used in standard tests - using 'mock' type.
     **
@@ -249,6 +281,30 @@ public class TestUtils
          .withBackendDetails(new FilesystemTableBackendDetails()
             .withBasePath("blobs")
             .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
+         );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineLocalFilesystemArchiveTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_ARCHIVE_LOCAL_FS)
+         .withLabel("Archive")
+         .withBackendName(defineLocalFilesystemBackend().getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB))
+         .withBackendDetails(new FilesystemTableBackendDetails()
+            .withBasePath("archive")
+            .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
          );
    }
 
@@ -269,6 +325,29 @@ public class TestUtils
          .withBackendDetails(new S3TableBackendDetails()
             .withBasePath("blobs")
             .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
+         );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineS3BlobSansPrefixTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_BLOB_S3_SANS_PREFIX)
+         .withLabel("Blob S3")
+         .withBackendName(defineS3BackendSansPrefix().getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB))
+         .withBackendDetails(new S3TableBackendDetails()
+            .withCardinality(Cardinality.ONE)
+            .withFileNameFieldName("fileName")
+            .withContentsFieldName("contents")
          );
    }
 
@@ -287,6 +366,18 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static S3BackendMetaData defineS3BackendSansPrefix()
+   {
+      return (new S3BackendMetaData()
+         .withBucketName(BaseS3Test.BUCKET_NAME_FOR_SANS_PREFIX_BACKEND)
+         .withName(BACKEND_NAME_S3_SANS_PREFIX));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -318,6 +409,18 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QBackendMetaData defineMemoryBackend()
+   {
+      return (new QBackendMetaData()
+         .withBackendType(MemoryBackendModule.class)
+         .withName(BACKEND_NAME_MEMORY));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java

@@ -0,0 +1,195 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
+
+
+import java.util.function.Consumer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ ** Unit test for AbstractFilesystemTableBackendDetails 
+ *******************************************************************************/
+class AbstractFilesystemTableBackendDetailsTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidInstancePasses() throws QInstanceValidationException
+   {
+      new QInstanceValidator().validate(QContext.getQInstance());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMissingCardinality() throws QException
+   {
+      assertValidationFailureReasons((QInstance qInstance) ->
+      {
+         qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3).withBackendDetails(new FilesystemTableBackendDetails());
+      }, false, "missing cardinality");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCardinalityOneIssues() throws QException
+   {
+      assertValidationFailureReasons((QInstance qInstance) ->
+      {
+         qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
+            .withCardinality(Cardinality.ONE)
+         );
+      }, false, "missing contentsFieldName", "missing fileNameFieldName");
+
+      assertValidationFailureReasons((QInstance qInstance) ->
+      {
+         qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
+            .withCardinality(Cardinality.ONE)
+            .withContentsFieldName("foo")
+            .withFileNameFieldName("bar")
+         );
+      }, false, "contentsFieldName [foo] is not a field", "fileNameFieldName [bar] is not a field");
+
+      assertValidationFailureReasons((QInstance qInstance) ->
+      {
+         qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
+            .withCardinality(Cardinality.ONE)
+            .withContentsFieldName("contents")
+            .withFileNameFieldName("fileName")
+            .withRecordFormat(RecordFormat.CSV)
+         );
+      }, false, "has a recordFormat");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testCardinalityManyIssues() throws QException
+   {
+      assertValidationFailureReasons((QInstance qInstance) ->
+      {
+         qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails()
+            .withCardinality(Cardinality.MANY)
+         );
+      }, false, "missing recordFormat");
+
+      assertValidationFailureReasons((QInstance qInstance) ->
+      {
+         qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails()
+            .withCardinality(Cardinality.MANY)
+            .withRecordFormat(RecordFormat.CSV)
+            .withContentsFieldName("foo")
+            .withFileNameFieldName("bar")
+         );
+      }, false, "has a contentsFieldName", "has a fileNameFieldName");
+   }
+
+
+
+   /*******************************************************************************
+    ** Implementation for the overloads of this name.
+    *******************************************************************************/
+   private void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... reasons) throws QException
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         setup.accept(qInstance);
+         new QInstanceValidator().validate(qInstance);
+         fail("Should have thrown validationException");
+      }
+      catch(QInstanceValidationException e)
+      {
+         if(!allowExtraReasons)
+         {
+            int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size();
+            assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons)
+               + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--"));
+         }
+
+         for(String reason : reasons)
+         {
+            assertReason(reason, e);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Assert that an instance is valid!
+    *******************************************************************************/
+   private void assertValidationSuccess(Consumer<QInstance> setup) throws QException
+   {
+      try
+      {
+         QInstance qInstance = TestUtils.defineInstance();
+         setup.accept(qInstance);
+         new QInstanceValidator().validate(qInstance);
+      }
+      catch(QInstanceValidationException e)
+      {
+         fail("Expected no validation errors, but received: " + e.getMessage());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** utility method for asserting that a specific reason string is found within
+    ** the list of reasons in the QInstanceValidationException.
+    **
+    *******************************************************************************/
+   private void assertReason(String reason, QInstanceValidationException e)
+   {
+      assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)");
+      assertThat(e.getReasons())
+         .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason)
+         .anyMatch(s -> s.contains(reason));
+   }
+
+}

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java

@@ -141,8 +141,10 @@ public class FilesystemBackendModuleTest
       // ensure unsupported filters throw //
       //////////////////////////////////////
       assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42))))
+         .rootCause()
          .hasMessageContaining("Unable to query filesystem table by field");
       assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK))))
+         .rootCause()
          .hasMessageContaining("Unable to query filename field using operator");
    }
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java

@@ -49,7 +49,7 @@ public class FilesystemActionTest extends BaseTest
     **
     *******************************************************************************/
    @BeforeEach
-   public void beforeEach() throws Exception
+   public void filesystemBaseBeforeEach() throws Exception
    {
       primeFilesystem();
       QContext.init(TestUtils.defineInstance(), new QSession());
@@ -61,7 +61,7 @@ public class FilesystemActionTest extends BaseTest
     **
     *******************************************************************************/
    @AfterEach
-   public void afterEach() throws Exception
+   public void filesystemBaseAfterEach() throws Exception
    {
       cleanFilesystem();
    }
@@ -130,7 +130,7 @@ public class FilesystemActionTest extends BaseTest
    /*******************************************************************************
     ** Write some data files into the directory for the filesystem module.
     *******************************************************************************/
-   private void writePersonCSVFiles(File baseDirectory) throws IOException
+   protected void writePersonCSVFiles(File baseDirectory) throws IOException
    {
       String fullPath = baseDirectory.getAbsolutePath();
       if(TestUtils.defineLocalFilesystemCSVPersonTable().getBackendDetails() instanceof FilesystemTableBackendDetails details)

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -93,13 +93,15 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    @Test
    public void testQueryForCardinalityOne() throws QException
    {
+      FilesystemQueryAction filesystemQueryAction = new FilesystemQueryAction();
+
       QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
       queryInput.setFilter(new QQueryFilter());
-      QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
+      QueryOutput queryOutput = filesystemQueryAction.execute(queryInput);
       assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
 
       queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
-      queryOutput = new FilesystemQueryAction().execute(queryInput);
+      queryOutput = filesystemQueryAction.execute(queryInput);
       assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
       assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
 
@@ -112,8 +114,15 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
       reInitInstanceInContext(instance);
 
       queryInput.setFilter(new QQueryFilter());
-      queryOutput = new FilesystemQueryAction().execute(queryInput);
+      queryOutput = filesystemQueryAction.execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
+
+      //////////////////////////////
+      // add a limit to the query //
+      //////////////////////////////
+      queryInput.setFilter(new QQueryFilter().withLimit(1));
+      queryOutput = filesystemQueryAction.execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
    }
 
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStepTest.java

@@ -0,0 +1,264 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.io.File;
+import java.time.LocalDateTime;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.local.actions.FilesystemActionTest;
+import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
+import org.json.JSONObject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemImporterStep 
+ *******************************************************************************/
+class FilesystemImporterStepTest extends FilesystemActionTest
+{
+
+   //////////////////////////////////////////////////////////////////////////
+   // note - we take advantage of the @BeforeEach and @AfterEach to set up //
+   // and clean up files on disk for this test.                            //
+   //////////////////////////////////////////////////////////////////////////
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterEach
+   public void afterEach() throws Exception
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      /////////////////////////////////////////////////////
+      // make sure we see 2 source files before we begin //
+      /////////////////////////////////////////////////////
+      FilesystemBackendMetaData backend   = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
+      String                    basePath  = backend.getBasePath();
+      File                      sourceDir = new File(basePath + "/persons-csv/");
+      assertEquals(2, listOrFail(sourceDir).length);
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(1, record.getValue("importFileId"));
+      assertEquals("John", record.getValue("firstName"));
+      assertThat(record.getValue("values")).isInstanceOf(String.class);
+      JSONObject values = new JSONObject(record.getValueString("values"));
+      assertEquals("John", values.get("firstName"));
+
+      ///////////////////////////////////////////
+      // make sure 2 archive files got created //
+      ///////////////////////////////////////////
+      LocalDateTime now = LocalDateTime.now();
+      assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
+
+      ////////////////////////////////////////////
+      // make sure the source files got deleted //
+      ////////////////////////////////////////////
+      assertEquals(0, listOrFail(sourceDir).length);
+   }
+
+
+
+   /*******************************************************************************
+    ** do a listFiles, but fail properly if it returns null (so IJ won't warn all the time)
+    *******************************************************************************/
+   private static File[] listOrFail(File dir)
+   {
+      File[] files = dir.listFiles();
+      if(files == null)
+      {
+         fail("Null result when listing directory: " + dir);
+      }
+      return (files);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testJSON() throws QException
+   {
+      ////////////////////////////////////////////////////////////////////
+      // adjust the process to use the JSON file table, and JSON format //
+      ////////////////////////////////////////////////////////////////////
+      QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_SOURCE_TABLE)).findFirst().get().setDefaultValue(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_FILE_FORMAT)).findFirst().get().setDefaultValue("json");
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(3, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      QRecord record = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(1, record.getValue("importFileId"));
+      assertEquals("John", record.getValue("firstName"));
+      assertThat(record.getValue("values")).isInstanceOf(String.class);
+      JSONObject values = new JSONObject(record.getValueString("values"));
+      assertEquals("John", values.get("firstName"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNoFilesFound() throws Exception
+   {
+      cleanFilesystem();
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(0, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+   }
+
+   // todo - updates?
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDuplicateFileNameNonUpdate() throws Exception
+   {
+      FilesystemBackendMetaData backend   = (FilesystemBackendMetaData) QContext.getQInstance().getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
+      String                    basePath  = backend.getBasePath();
+      File                      sourceDir = new File(basePath + "/persons-csv/");
+
+      /////////////////////////////////////////////////////////////////
+      // run the process once - assert how many records got inserted //
+      /////////////////////////////////////////////////////////////////
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      String importBaseName = "personImporter";
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      ///////////////////////////////////////////////////////
+      // put the source files back - assert they are there //
+      ///////////////////////////////////////////////////////
+      writePersonCSVFiles(new File(basePath));
+      assertEquals(2, listOrFail(sourceDir).length);
+
+      ////////////////////////
+      // re-run the process //
+      ////////////////////////
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      ////////////////////////////////////////
+      // make sure no new records are built //
+      ////////////////////////////////////////
+      assertEquals(2, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX)).getCount());
+      assertEquals(5, new CountAction().execute(new CountInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX)).getCount());
+
+      /////////////////////////////////////////////////
+      // make sure no new archive files were created //
+      /////////////////////////////////////////////////
+      LocalDateTime now = LocalDateTime.now();
+      assertEquals(2, listOrFail(new File(basePath + "/archive/archive-of/personImporterFiles/" + now.getYear() + "/" + now.getMonth())).length);
+
+      ////////////////////////////////////////////
+      // make sure the source files got deleted //
+      ////////////////////////////////////////////
+      assertEquals(0, listOrFail(sourceDir).length);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityKey() throws QException
+   {
+      //////////////////////////////////////////////
+      // Add a security name/value to our process //
+      //////////////////////////////////////////////
+      QProcessMetaData process = QContext.getQInstance().getProcess(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_NAME)).findFirst().get().setDefaultValue("customerId");
+      process.getInputFields().stream().filter(f -> f.getName().equals(FilesystemImporterStep.FIELD_IMPORT_SECURITY_FIELD_VALUE)).findFirst().get().setDefaultValue(47);
+
+      RunProcessInput runProcessInput = new RunProcessInput();
+      runProcessInput.setProcessName(TestUtils.LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME);
+      new RunProcessAction().execute(runProcessInput);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      // assert the security field gets its value on both the importFile & importRecord records //
+      ////////////////////////////////////////////////////////////////////////////////////////////
+      String  importBaseName = "personImporter";
+      QRecord fileRecord     = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_FILE_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(47, fileRecord.getValue("customerId"));
+
+      QRecord recordRecord = new GetAction().executeForRecord(new GetInput(importBaseName + FilesystemImporterMetaDataTemplate.IMPORT_RECORD_TABLE_SUFFIX).withPrimaryKey(1));
+      assertEquals(47, recordRecord.getValue("customerId"));
+   }
+
+}

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/ImportRecordPostQueryCustomizerTest.java

@@ -0,0 +1,85 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.filesystem.importer;
+
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for ImportRecordPostQueryCustomizer 
+ *******************************************************************************/
+class ImportRecordPostQueryCustomizerTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      Instant createDate = Instant.parse("2024-01-08T20:07:21Z");
+
+      List<QRecord> output = new ImportRecordPostQueryCustomizer().apply(List.of(
+         new QRecord()
+            .withTableName("personImporterImportRecord")
+            .withValue("importFileId", 1)
+            .withValue("unmapped", 2)
+            .withValue("unstructured", 3)
+            .withValue("nosqlObject", MapBuilder.of(HashMap::new).with("foo", "bar").with("createDate", createDate).build())
+      ));
+
+      assertEquals(1, output.get(0).getValue("importFileId"));
+      assertEquals(2, output.get(0).getValue("unmapped"));
+      assertEquals(3, output.get(0).getValue("unstructured"));
+      assertEquals(Map.of("foo", "bar", "createDate", createDate), output.get(0).getValue("nosqlObject"));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // make sure all un-structured fields get put in the "values" field as a JSON string     //
+      // compare as maps, beacuse JSONObject seems to care about the ordering, which, we don't //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      Map<String, Object> expectedMap = new JSONObject("""
+         {
+            "unmapped": 2,
+            "unstructured": 3,
+            "nosqlObject":
+            {
+               "foo": "bar",
+               "createDate": "%s"
+            }
+         }
+         """.formatted(createDate)).toMap();
+      Map<String, Object> actualMap = new JSONObject(output.get(0).getValueString("values")).toMap();
+      assertThat(actualMap).isEqualTo(expectedMap);
+   }
+
+}

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessS3Test.java

@@ -38,6 +38,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
 import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
 import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassForTest;
@@ -197,8 +199,8 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
       for(String path : paths)
       {
          assertTrue(s3ObjectSummaries.stream().anyMatch(s3o -> s3o.getKey().equals(path)),
-            "Path [" + path + "] should be in the listing, but was not.  Full listing is: " +
-               s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
+            "Path [" + path + "] should be in the listing, but was not.  Full listing is: "
+               + s3ObjectSummaries.stream().map(S3ObjectSummary::getKey).collect(Collectors.joining(",")));
       }
    }
 
@@ -257,6 +259,8 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
          .withBackendName(backend.getName())
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
          .withBackendDetails(new S3TableBackendDetails()
+            .withCardinality(Cardinality.MANY)
+            .withRecordFormat(RecordFormat.CSV)
             .withBasePath(path)
             .withGlob(glob));
       qInstance.addTable(qTableMetaData);

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java

@@ -35,6 +35,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
 import org.apache.commons.io.FileUtils;
@@ -118,6 +120,8 @@ class FilesystemSyncProcessTest extends BaseTest
          .withBackendName(TestUtils.BACKEND_NAME_LOCAL_FS)
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
          .withBackendDetails(new FilesystemTableBackendDetails()
+            .withCardinality(Cardinality.MANY)
+            .withRecordFormat(RecordFormat.CSV)
             .withBasePath(subPath));
    }
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/BaseS3Test.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.module.filesystem.s3;
 
 
+import java.util.List;
 import cloud.localstack.ServiceName;
 import cloud.localstack.awssdkv1.TestUtils;
 import cloud.localstack.docker.LocalstackDockerExtension;
@@ -46,6 +47,7 @@ public class BaseS3Test extends BaseTest
    public static final String TEST_FOLDER = "test-files";
    public static final String SUB_FOLDER  = "sub-folder";
 
+   public static final String BUCKET_NAME_FOR_SANS_PREFIX_BACKEND = "localstack-test-bucket-sans-prefix";
 
 
    /*******************************************************************************
@@ -65,6 +67,11 @@ public class BaseS3Test extends BaseTest
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-1.txt", "Hello, Blob");
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-2.txt", "Hi, Bob");
       amazonS3.putObject(BUCKET_NAME, TEST_FOLDER + "/blobs/BLOB-3.md", "# Hi, MD");
+
+      amazonS3.createBucket(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND);
+      amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-1.txt", "Hello, Blob");
+      amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-2.txt", "Hi, Bob");
+      amazonS3.putObject(BUCKET_NAME_FOR_SANS_PREFIX_BACKEND, "BLOB-3.md", "# Hi, MD");
    }
 
 
@@ -77,16 +84,19 @@ public class BaseS3Test extends BaseTest
    {
       AmazonS3 amazonS3 = getAmazonS3();
 
-      if(amazonS3.doesBucketExistV2(BUCKET_NAME))
+      for(String bucketName : List.of(BUCKET_NAME, BUCKET_NAME_FOR_SANS_PREFIX_BACKEND))
       {
-         ////////////////////////
-         // todo - paginate... //
-         ////////////////////////
-         for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(BUCKET_NAME).getObjectSummaries())
+         if(amazonS3.doesBucketExistV2(bucketName))
          {
-            amazonS3.deleteObject(BUCKET_NAME, objectSummary.getKey());
+            ////////////////////////
+            // todo - paginate... //
+            ////////////////////////
+            for(S3ObjectSummary objectSummary : amazonS3.listObjectsV2(bucketName).getObjectSummaries())
+            {
+               amazonS3.deleteObject(bucketName, objectSummary.getKey());
+            }
+            amazonS3.deleteBucket(bucketName);
          }
-         amazonS3.deleteBucket(BUCKET_NAME);
       }
    }
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3QueryActionTest.java

@@ -105,6 +105,57 @@ public class S3QueryActionTest extends BaseS3Test
       queryInput.setFilter(new QQueryFilter());
       queryOutput = s3QueryAction.execute(queryInput);
       assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
+
+      //////////////////////////////
+      // add a limit to the query //
+      //////////////////////////////
+      queryInput.setFilter(new QQueryFilter().withLimit(1));
+      queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
+   }
+
+
+
+   /*******************************************************************************
+    ** We had a bug where, if both the backend and table have no basePath ("prefix"),
+    ** then our file-listing was doing a request with a prefix starting with /, which
+    ** causes no results, so, this test is to show that isn't happening.
+    *******************************************************************************/
+   @Test
+   public void testQueryForCardinalityOneInBackendWithoutPrefix() throws QException
+   {
+      QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_BLOB_S3_SANS_PREFIX);
+      queryInput.setFilter(new QQueryFilter());
+
+      S3QueryAction s3QueryAction = new S3QueryAction();
+      s3QueryAction.setS3Utils(getS3Utils());
+
+      QueryOutput queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt")));
+      queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Filtered query should find 1 row");
+      assertEquals("BLOB-1.txt", queryOutput.getRecords().get(0).getValueString("fileName"));
+
+      ////////////////////////////////////////////////////////////////
+      // put a glob on the table - now should only find 2 txt files //
+      ////////////////////////////////////////////////////////////////
+      QInstance instance = TestUtils.defineInstance();
+      ((S3TableBackendDetails) (instance.getTable(TestUtils.TABLE_NAME_BLOB_S3_SANS_PREFIX).getBackendDetails()))
+         .withGlob("*.txt");
+      reInitInstanceInContext(instance);
+
+      queryInput.setFilter(new QQueryFilter());
+      queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Query should use glob and find 2 rows");
+
+      //////////////////////////////
+      // add a limit to the query //
+      //////////////////////////////
+      queryInput.setFilter(new QQueryFilter().withLimit(1));
+      queryOutput = s3QueryAction.execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Query with limit should be respected");
    }
 
 }

qqq-backend-module-mongodb/pom.xml

@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ QQQ - Low-code Application Framework for Engineers.
+  ~ Copyright (C) 2021-2024.  Kingsrook, LLC
+  ~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+  ~ contact@kingsrook.com
+  ~ https://github.com/Kingsrook/
+  ~
+  ~ This program is free software: you can redistribute it and/or modify
+  ~ it under the terms of the GNU Affero General Public License as
+  ~ published by the Free Software Foundation, either version 3 of the
+  ~ License, or (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU Affero General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Affero General Public License
+  ~ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+   <modelVersion>4.0.0</modelVersion>
+
+   <artifactId>qqq-backend-module-mongodb</artifactId>
+
+   <parent>
+      <groupId>com.kingsrook.qqq</groupId>
+      <artifactId>qqq-parent-project</artifactId>
+      <version>${revision}</version>
+   </parent>
+
+   <properties>
+      <!-- props specifically to this module -->
+      <!-- none at this time -->
+   </properties>
+
+   <dependencies>
+      <!-- other qqq modules deps -->
+      <dependency>
+         <groupId>com.kingsrook.qqq</groupId>
+         <artifactId>qqq-backend-core</artifactId>
+         <version>${revision}</version>
+      </dependency>
+
+      <!-- 3rd party deps specifically for this module -->
+      <dependency>
+         <groupId>org.mongodb</groupId>
+         <artifactId>mongodb-driver-sync</artifactId>
+         <version>4.11.1</version>
+      </dependency>
+      <dependency>
+         <groupId>org.apache.logging.log4j</groupId>
+         <artifactId>log4j-slf4j-impl</artifactId>
+         <version>2.17.1</version>
+      </dependency>
+
+      <dependency>
+         <groupId>org.testcontainers</groupId>
+         <artifactId>mongodb</artifactId>
+         <version>1.19.3</version>
+         <scope>test</scope>
+      </dependency>
+
+      <!-- Common deps for all qqq modules -->
+      <dependency>
+         <groupId>org.apache.maven.plugins</groupId>
+         <artifactId>maven-checkstyle-plugin</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.apache.logging.log4j</groupId>
+         <artifactId>log4j-api</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.apache.logging.log4j</groupId>
+         <artifactId>log4j-core</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.junit.jupiter</groupId>
+         <artifactId>junit-jupiter-engine</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.assertj</groupId>
+         <artifactId>assertj-core</artifactId>
+         <scope>test</scope>
+      </dependency>
+   </dependencies>
+
+   <build>
+      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <version>2.4.3</version>
+            <configuration>
+               <createDependencyReducedPom>false</createDependencyReducedPom>
+               <filters>
+                  <filter>
+                     <artifact>*:*</artifact>
+                     <excludes>
+                        <exclude>META-INF/*</exclude>
+                     </excludes>
+                  </filter>
+               </filters>
+            </configuration>
+            <executions>
+               <execution>
+                  <phase>${plugin.shade.phase}</phase>
+                  <goals>
+                     <goal>shade</goal>
+                  </goals>
+               </execution>
+            </executions>
+         </plugin>
+      </plugins>
+   </build>
+
+</project>

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/MongoDBBackendModule.java

@@ -0,0 +1,168 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb;
+
+
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.mongodb.actions.AbstractMongoDBAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoClientContainer;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBAggregateAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBCountAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBDeleteAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBInsertAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBQueryAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBTransaction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoDBUpdateAction;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
+
+
+/*******************************************************************************
+ ** QQQ Backend module for working with MongoDB
+ *******************************************************************************/
+public class MongoDBBackendModule implements QBackendModuleInterface
+{
+   static
+   {
+      QBackendModuleDispatcher.registerBackendModule(new MongoDBBackendModule());
+   }
+
+   /*******************************************************************************
+    ** Method where a backend module must be able to provide its type (name).
+    *******************************************************************************/
+   public String getBackendType()
+   {
+      return ("mongodb");
+   }
+
+
+
+   /*******************************************************************************
+    ** Method to identify the class used for backend meta data for this module.
+    *******************************************************************************/
+   @Override
+   public Class<? extends QBackendMetaData> getBackendMetaDataClass()
+   {
+      return (MongoDBBackendMetaData.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** Method to identify the class used for table-backend details for this module.
+    *******************************************************************************/
+   @Override
+   public Class<? extends QTableBackendDetails> getTableBackendDetailsClass()
+   {
+      return (MongoDBTableBackendDetails.class);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public CountInterface getCountInterface()
+   {
+      return (new MongoDBCountAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QueryInterface getQueryInterface()
+   {
+      return (new MongoDBQueryAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public InsertInterface getInsertInterface()
+   {
+      return (new MongoDBInsertAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public UpdateInterface getUpdateInterface()
+   {
+      return (new MongoDBUpdateAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public DeleteInterface getDeleteInterface()
+   {
+      return (new MongoDBDeleteAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public AggregateInterface getAggregateInterface()
+   {
+      return (new MongoDBAggregateAction());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QBackendTransaction openTransaction(AbstractTableActionInput input)
+   {
+      MongoDBBackendMetaData backend              = (MongoDBBackendMetaData) input.getBackend();
+      MongoClientContainer   mongoClientContainer = new AbstractMongoDBAction().openClient(backend, null);
+      return (new MongoDBTransaction(backend, mongoClientContainer.getMongoClient()));
+   }
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/AbstractMongoDBAction.java

@@ -0,0 +1,789 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
+import com.mongodb.ConnectionString;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.MongoCredential;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.model.Filters;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.bson.types.ObjectId;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Base class for all mongoDB module actions.
+ *******************************************************************************/
+public class AbstractMongoDBAction
+{
+   private static final QLogger LOG = QLogger.getLogger(AbstractMongoDBAction.class);
+
+   protected QueryStat queryStat;
+
+
+
+   /*******************************************************************************
+    ** Open a MongoDB Client / session -- re-using the one in the input transaction
+    ** if it is present.
+    *******************************************************************************/
+   public MongoClientContainer openClient(MongoDBBackendMetaData backend, QBackendTransaction transaction)
+   {
+      if(transaction instanceof MongoDBTransaction mongoDBTransaction)
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////
+         // re-use the connection from the transaction (indicating false in last parameter here) //
+         //////////////////////////////////////////////////////////////////////////////////////////
+         return (new MongoClientContainer(mongoDBTransaction.getMongoClient(), mongoDBTransaction.getClientSession(), false));
+      }
+
+      String           suffix           = StringUtils.hasContent(backend.getUrlSuffix()) ? "?" + backend.getUrlSuffix() : "";
+      ConnectionString connectionString = new ConnectionString("mongodb://" + backend.getHost() + ":" + backend.getPort() + "/" + suffix);
+
+      MongoCredential credential = MongoCredential.createCredential(backend.getUsername(), backend.getAuthSourceDatabase(), backend.getPassword().toCharArray());
+
+      MongoClientSettings settings = MongoClientSettings.builder()
+
+         ////////////////////////////////////////////////
+         // is this needed, what, for a cluster maybe? //
+         ////////////////////////////////////////////////
+         // .applyToClusterSettings(builder -> builder.hosts(seeds))
+
+         .applyConnectionString(connectionString)
+         .credential(credential)
+         .build();
+
+      MongoClient mongoClient = MongoClients.create(settings);
+
+      ////////////////////////////////////////////////////////////////////////////
+      // indicate that this connection was newly opened via the true param here //
+      ////////////////////////////////////////////////////////////////////////////
+      return (new MongoClientContainer(mongoClient, mongoClient.startSession(), true));
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the name to use for a field in the mongoDB, from the fieldMetaData.
+    **
+    ** That is, field.backendName if set -- else, field.name
+    *******************************************************************************/
+   protected String getFieldBackendName(QFieldMetaData field)
+   {
+      if(field.getBackendName() != null)
+      {
+         return (field.getBackendName());
+      }
+      return (field.getName());
+   }
+
+
+
+   /*******************************************************************************
+    ** Get the name to use for a table in the mongoDB, from the table's backendDetails.
+    **
+    ** else, the table's name.
+    *******************************************************************************/
+   protected String getBackendTableName(QTableMetaData table)
+   {
+      if(table == null)
+      {
+         return (null);
+      }
+
+      if(table.getBackendDetails() != null)
+      {
+         String backendTableName = ((MongoDBTableBackendDetails) table.getBackendDetails()).getTableName();
+         if(StringUtils.hasContent(backendTableName))
+         {
+            return (backendTableName);
+         }
+      }
+      return table.getName();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected int getPageSize()
+   {
+      return (1000);
+   }
+
+
+
+   /*******************************************************************************
+    ** Convert a mongodb document to a QRecord.
+    *******************************************************************************/
+   protected QRecord documentToRecord(QTableMetaData table, Document document)
+   {
+      QRecord record = new QRecord();
+      record.setTableName(table.getName());
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // first iterate over the table's fields, looking for them (at their backend name (path,    //
+      // if it has dots) inside the document note that we'll remove values from the document      //
+      // as we go - then after this loop, will handle all remaining values as unstructured fields //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      Map<String, Serializable> values = record.getValues();
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         String fieldName = field.getName();
+         String fieldBackendName = getFieldBackendName(field);
+
+         if(fieldBackendName.contains("."))
+         {
+            /////////////////////////////////////////////////////////////
+            // process backend-names with dots as hierarchical objects //
+            /////////////////////////////////////////////////////////////
+            String[] parts       = fieldBackendName.split("\\.");
+            Document tmpDocument = document;
+            for(int i = 0; i < parts.length - 1; i++)
+            {
+               if(!tmpDocument.containsKey(parts[i]))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if we can't find the sub-document, break, and we won't have a value for this field (do we want null?) //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  setValue(values, fieldName, null);
+                  break;
+               }
+               else
+               {
+                  if(tmpDocument.get(parts[i]) instanceof Document subDocument)
+                  {
+                     tmpDocument = subDocument;
+                  }
+                  else
+                  {
+                     LOG.warn("Unexpected - In table [" + table.getName() + "] found a non-document at sub-key [" + parts[i] + "] for field [" + field.getName() + "]");
+                  }
+               }
+            }
+
+            Object value = tmpDocument.remove(parts[parts.length - 1]);
+            setValue(values, fieldName, value);
+         }
+         else
+         {
+            Object value = document.remove(fieldBackendName);
+            setValue(values, fieldName, value);
+         }
+      }
+
+      //////////////////////////////////////////////////////////////
+      // handle remaining values in the document as un-structured //
+      //////////////////////////////////////////////////////////////
+      for(String subFieldName : document.keySet())
+      {
+         Object subValue = document.get(subFieldName);
+         setValue(values, subFieldName, subValue);
+      }
+
+      return (record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Recursive helper method to put a value in a map - where mongodb documents
+    ** are recursively expanded, and types are mapped to QQQ expectations.
+    *******************************************************************************/
+   private void setValue(Map<String, Serializable> values, String fieldName, Object value)
+   {
+      if(value instanceof ObjectId objectId)
+      {
+         values.put(fieldName, objectId.toString());
+      }
+      else if(value instanceof java.util.Date date)
+      {
+         values.put(fieldName, date.toInstant());
+      }
+      else if(value instanceof Document document)
+      {
+         LinkedHashMap<String, Serializable> subValues = new LinkedHashMap<>();
+         values.put(fieldName, subValues);
+
+         for(String subFieldName : document.keySet())
+         {
+            Object subValue = document.get(subFieldName);
+            setValue(subValues, subFieldName, subValue);
+         }
+      }
+      else if(value instanceof Serializable s)
+      {
+         values.put(fieldName, s);
+      }
+      else if(value != null)
+      {
+         values.put(fieldName, String.valueOf(value));
+      }
+      else
+      {
+         values.put(fieldName, null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Convert a QRecord to a mongodb document.
+    *******************************************************************************/
+   protected Document recordToDocument(QTableMetaData table, QRecord record) throws QException
+   {
+      Document document = new Document();
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      // first iterate over fields defined in the table - put them in the document for mongo first. //
+      // track the names that we've processed in a set. then later we'll go over all values in the  //
+      // record and send them all to mongo (skipping ones we knew about from the table definition)  //
+      ////////////////////////////////////////////////////////////////////////////////////////////////
+      Set<String> processedFields = new HashSet<>();
+
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         Serializable value = record.getValue(field.getName());
+         processedFields.add(field.getName());
+
+         if(field.getName().equals(table.getPrimaryKeyField()) && value == null)
+         {
+            ////////////////////////////////////
+            // let mongodb client generate id //
+            ////////////////////////////////////
+            continue;
+         }
+
+         String fieldBackendName = getFieldBackendName(field);
+         if(fieldBackendName.contains("."))
+         {
+            /////////////////////////////////////////////////////////////
+            // process backend-names with dots as hierarchical objects //
+            /////////////////////////////////////////////////////////////
+            String[] parts       = fieldBackendName.split("\\.");
+            Document tmpDocument = document;
+            for(int i = 0; i < parts.length - 1; i++)
+            {
+               if(!tmpDocument.containsKey(parts[i]))
+               {
+                  Document subDocument = new Document();
+                  tmpDocument.put(parts[i], subDocument);
+                  tmpDocument = subDocument;
+               }
+               else
+               {
+                  if(tmpDocument.get(parts[i]) instanceof Document subDocument)
+                  {
+                     tmpDocument = subDocument;
+                  }
+                  else
+                  {
+                     throw (new QException("Fields in table [" + table.getName() + "] specify both a sub-object and a field at the key: " + parts[i]));
+                  }
+               }
+            }
+            tmpDocument.append(parts[parts.length - 1], value);
+         }
+         else
+         {
+            document.append(fieldBackendName, value);
+         }
+      }
+
+      /////////////////////////
+      // do remaining values //
+      /////////////////////////
+      // for(Map.Entry<String, Serializable> entry : clone.getValues().entrySet())
+      for(Map.Entry<String, Serializable> entry : record.getValues().entrySet())
+      {
+         if(!processedFields.contains(entry.getKey()))
+         {
+            document.append(entry.getKey(), entry.getValue());
+         }
+      }
+
+      return (document);
+   }
+
+
+
+   /*******************************************************************************
+    ** Convert QQueryFilter to Bson search query document - including security
+    ** for the table if needed.
+    *******************************************************************************/
+   protected Bson makeSearchQueryDocument(QTableMetaData table, QQueryFilter filter) throws QException
+   {
+      Bson         searchQueryWithoutSecurity = makeSearchQueryDocumentWithoutSecurity(table, filter);
+      QQueryFilter securityFilter             = makeSecurityQueryFilter(table);
+      if(!securityFilter.hasAnyCriteria())
+      {
+         return (searchQueryWithoutSecurity);
+      }
+
+      Bson searchQueryForSecurity = makeSearchQueryDocumentWithoutSecurity(table, securityFilter);
+
+      if(searchQueryWithoutSecurity.toBsonDocument().isEmpty())
+      {
+         return (searchQueryForSecurity);
+      }
+      else
+      {
+         return (Filters.and(searchQueryWithoutSecurity, searchQueryForSecurity));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Build a QQueryFilter to apply record-level security to the query.
+    ** Note, it may be empty, if there are no lock fields, or all are all-access.
+    **
+    ** Originally copied from RDBMS module... should this be shared?
+    ** and/or, how big of a re-write did that get in the joins-enhancements branch...
+    *******************************************************************************/
+   private QQueryFilter makeSecurityQueryFilter(QTableMetaData table) throws QException
+   {
+      QQueryFilter securityFilter = new QQueryFilter();
+      securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND);
+
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
+      {
+         addSubFilterForRecordSecurityLock(QContext.getQInstance(), QContext.getQSession(), table, securityFilter, recordSecurityLock, null, table.getName(), false);
+      }
+
+      return (securityFilter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Helper for makeSecuritySearchQuery.
+    **
+    ** Originally copied from RDBMS module... should this be shared?
+    ** and/or, how big of a re-write did that get in the joins-enhancements branch...
+    *******************************************************************************/
+   private static void addSubFilterForRecordSecurityLock(QInstance instance, QSession session, QTableMetaData table, QQueryFilter securityFilter, RecordSecurityLock recordSecurityLock, JoinsContext joinsContext, String tableNameOrAlias, boolean isOuter) throws QException
+   {
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
+      if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
+      {
+         if(session.hasSecurityKeyValue(securityKeyType.getAllAccessKeyName(), true, QFieldType.BOOLEAN))
+         {
+            ///////////////////////////////////////////////////////////////////////////////
+            // if we have all-access on this key, then we don't need a criterion for it. //
+            ///////////////////////////////////////////////////////////////////////////////
+            return;
+         }
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // some differences from RDBMS here, due to not yet having joins support in mongo... //
+      ///////////////////////////////////////////////////////////////////////////////////////
+      // String fieldName = tableNameOrAlias + "." + recordSecurityLock.getFieldName();
+      String fieldName = recordSecurityLock.getFieldName();
+      if(CollectionUtils.nullSafeHasContents(recordSecurityLock.getJoinNameChain()))
+      {
+         throw (new QException("Security locks in mongodb with joinNameChain is not yet supported"));
+         // fieldName = recordSecurityLock.getFieldName();
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // else - get the key values from the session and decide what kind of criterion to build //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      QQueryFilter          lockFilter   = new QQueryFilter();
+      List<QFilterCriteria> lockCriteria = new ArrayList<>();
+      lockFilter.setCriteria(lockCriteria);
+
+      QFieldType type = QFieldType.INTEGER;
+      try
+      {
+         if(joinsContext == null)
+         {
+            type = table.getField(fieldName).getType();
+         }
+         else
+         {
+            JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(fieldName);
+            type = fieldAndTableNameOrAlias.field().getType();
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error getting field type...  Trying Integer", e);
+      }
+
+      List<Serializable> securityKeyValues = session.getSecurityKeyValues(recordSecurityLock.getSecurityKeyType(), type);
+      if(CollectionUtils.nullSafeIsEmpty(securityKeyValues))
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // handle user with no values -- they can only see null values, and only iff the lock's null-value behavior is ALLOW //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior()))
+         {
+            lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_BLANK));
+         }
+         else
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // else, if no user/session values, and null-value behavior is deny, then setup a FALSE condition, to allow no rows.           //
+            // todo - make some explicit contradiction here - maybe even avoid running the whole query - as you're not allowed ANY records //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, Collections.emptyList()));
+         }
+      }
+      else
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // else, if user/session has some values, build an IN rule -                                                //
+         // noting that if the lock's null-value behavior is ALLOW, then we actually want IS_NULL_OR_IN, not just IN //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(RecordSecurityLock.NullValueBehavior.ALLOW.equals(recordSecurityLock.getNullValueBehavior()))
+         {
+            lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IS_NULL_OR_IN, securityKeyValues));
+         }
+         else
+         {
+            lockCriteria.add(new QFilterCriteria(fieldName, QCriteriaOperator.IN, securityKeyValues));
+         }
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if this field is on the outer side of an outer join, then if we do a straight filter on it, then we're basically      //
+      // nullifying the outer join... so for an outer join use-case, OR the security field criteria with a primary-key IS NULL //
+      // which will make missing rows from the join be found.                                                                  //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      if(isOuter)
+      {
+         lockFilter.setBooleanOperator(QQueryFilter.BooleanOperator.OR);
+         lockFilter.addCriteria(new QFilterCriteria(tableNameOrAlias + "." + table.getPrimaryKeyField(), QCriteriaOperator.IS_BLANK));
+      }
+
+      securityFilter.addSubFilter(lockFilter);
+   }
+
+
+
+   /*******************************************************************************
+    ** w/o considering security, just map a QQueryFilter to a Bson searchQuery.
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:Indentation")
+   private Bson makeSearchQueryDocumentWithoutSecurity(QTableMetaData table, QQueryFilter filter)
+   {
+      if(filter == null || !filter.hasAnyCriteria())
+      {
+         return (new Document());
+      }
+
+      List<Bson> criteriaFilters = new ArrayList<>();
+
+      for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
+      {
+         List<Serializable> values           = criteria.getValues() == null ? new ArrayList<>() : new ArrayList<>(criteria.getValues());
+         QFieldMetaData     field            = table.getField(criteria.getFieldName());
+         String             fieldBackendName = getFieldBackendName(field);
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // replace any expression-type values with their evaluation                                                                         //
+         // also, "scrub" non-expression values, which type-converts them (e.g., strings in various supported date formats become LocalDate) //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         ListIterator<Serializable> valueListIterator = values.listIterator();
+         while(valueListIterator.hasNext())
+         {
+            Serializable value = valueListIterator.next();
+            if(value instanceof AbstractFilterExpression<?> expression)
+            {
+               valueListIterator.set(expression.evaluate());
+            }
+            /*
+            todo - is this needed??
+            else
+            {
+               Serializable scrubbedValue = scrubValue(field, value);
+               valueListIterator.set(scrubbedValue);
+            }
+            */
+         }
+
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // make sure any values we're going to run against the primary key (_id) are ObjectIds //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         if(field.getName().equals(table.getPrimaryKeyField()))
+         {
+            ListIterator<Serializable> iterator = values.listIterator();
+            while(iterator.hasNext())
+            {
+               Serializable value = iterator.next();
+               iterator.set(new ObjectId(String.valueOf(value)));
+            }
+         }
+
+         ////////
+         // :( //
+         ////////
+         if(StringUtils.hasContent(criteria.getOtherFieldName()))
+         {
+            throw (new IllegalArgumentException("A mongodb query with an 'otherFieldName' specified is not currently supported."));
+         }
+
+         criteriaFilters.add(switch(criteria.getOperator())
+         {
+            case EQUALS -> Filters.eq(fieldBackendName, getValue(values, 0));
+
+            case NOT_EQUALS -> Filters.and(
+               Filters.ne(fieldBackendName, getValue(values, 0)),
+
+               ////////////////////////////////////////////////////////////////////////////////////////////
+               // to match RDBMS and other QQQ backends, consider a null to not match a not-equals query //
+               ////////////////////////////////////////////////////////////////////////////////////////////
+               Filters.not(Filters.eq(fieldBackendName, null))
+            );
+
+            case NOT_EQUALS_OR_IS_NULL -> Filters.or(
+               Filters.eq(fieldBackendName, null),
+               Filters.ne(fieldBackendName, getValue(values, 0))
+            );
+            case IN -> filterIn(fieldBackendName, values);
+            case NOT_IN -> Filters.nor(filterIn(fieldBackendName, values));
+            case IS_NULL_OR_IN -> Filters.or(
+               Filters.eq(fieldBackendName, null),
+               filterIn(fieldBackendName, values)
+            );
+            case LIKE -> filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(getValue(values, 0)).replaceAll("%", ".*"), null);
+            case NOT_LIKE -> Filters.nor(filterRegex(fieldBackendName, null, ValueUtils.getValueAsString(getValue(values, 0)).replaceAll("%", ".*"), null));
+            case STARTS_WITH -> filterRegex(fieldBackendName, null, getValue(values, 0), ".*");
+            case ENDS_WITH -> filterRegex(fieldBackendName, ".*", getValue(values, 0), null);
+            case CONTAINS -> filterRegex(fieldBackendName, ".*", getValue(values, 0), ".*");
+            case NOT_STARTS_WITH -> Filters.nor(filterRegex(fieldBackendName, null, getValue(values, 0), ".*"));
+            case NOT_ENDS_WITH -> Filters.nor(filterRegex(fieldBackendName, ".*", getValue(values, 0), null));
+            case NOT_CONTAINS -> Filters.nor(filterRegex(fieldBackendName, ".*", getValue(values, 0), ".*"));
+            case LESS_THAN -> Filters.lt(fieldBackendName, getValue(values, 0));
+            case LESS_THAN_OR_EQUALS -> Filters.lte(fieldBackendName, getValue(values, 0));
+            case GREATER_THAN -> Filters.gt(fieldBackendName, getValue(values, 0));
+            case GREATER_THAN_OR_EQUALS -> Filters.gte(fieldBackendName, getValue(values, 0));
+            case IS_BLANK -> filterIsBlank(fieldBackendName);
+            case IS_NOT_BLANK -> Filters.nor(filterIsBlank(fieldBackendName));
+            case BETWEEN -> filterBetween(fieldBackendName, values);
+            case NOT_BETWEEN -> Filters.nor(filterBetween(fieldBackendName, values));
+         });
+      }
+
+      /////////////////////////////////////
+      // recursively process sub-filters //
+      /////////////////////////////////////
+      if(CollectionUtils.nullSafeHasContents(filter.getSubFilters()))
+      {
+         for(QQueryFilter subFilter : filter.getSubFilters())
+         {
+            criteriaFilters.add(makeSearchQueryDocumentWithoutSecurity(table, subFilter));
+         }
+      }
+
+      Bson bson = QQueryFilter.BooleanOperator.AND.equals(filter.getBooleanOperator()) ? Filters.and(criteriaFilters) : Filters.or(criteriaFilters);
+      return bson;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static Serializable getValue(List<Serializable> values, int i)
+   {
+      if(values == null || values.size() <= i)
+      {
+         throw new IllegalArgumentException("Incorrect number of values given for criteria");
+      }
+
+      return (values.get(i));
+   }
+
+
+
+   /*******************************************************************************
+    ** build a bson filter doing a regex (e.g., for LIKE, STARTS_WITH, etc)
+    *******************************************************************************/
+   private Bson filterRegex(String fieldBackendName, String prefix, Serializable mainRegex, String suffix)
+   {
+      if(prefix == null)
+      {
+         prefix = "";
+      }
+
+      if(suffix == null)
+      {
+         suffix = "";
+      }
+
+      String fullRegex = prefix + ValueUtils.getValueAsString(mainRegex + suffix);
+      return (Filters.regex(fieldBackendName, Pattern.compile(fullRegex)));
+   }
+
+
+
+   /*******************************************************************************
+    ** build a bson filter doing IN
+    *******************************************************************************/
+   private static Bson filterIn(String fieldBackendName, List<Serializable> values)
+   {
+      return Filters.in(fieldBackendName, values);
+   }
+
+
+
+   /*******************************************************************************
+    ** build a bson filter doing BETWEEN
+    *******************************************************************************/
+   private static Bson filterBetween(String fieldBackendName, List<Serializable> values)
+   {
+      return Filters.and(
+         Filters.gte(fieldBackendName, getValue(values, 0)),
+         Filters.lte(fieldBackendName, getValue(values, 1))
+      );
+   }
+
+
+
+   /*******************************************************************************
+    ** build a bson filter doing BLANK (null or == "")
+    *******************************************************************************/
+   private static Bson filterIsBlank(String fieldBackendName)
+   {
+      return Filters.or(
+         Filters.eq(fieldBackendName, null),
+         Filters.eq(fieldBackendName, "")
+      );
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for queryStat
+    *******************************************************************************/
+   public QueryStat getQueryStat()
+   {
+      return (this.queryStat);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryStat
+    *******************************************************************************/
+   public void setQueryStat(QueryStat queryStat)
+   {
+      this.queryStat = queryStat;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void setQueryInQueryStat(Bson query)
+   {
+      if(queryStat != null && query != null)
+      {
+         queryStat.setQueryText(query.toString());
+
+         ////////////////////////////////////////////////////////////////
+         // todo - if we support joins in the future, do them here too //
+         ////////////////////////////////////////////////////////////////
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void logQuery(String tableName, String actionName, List<Bson> query, Long queryStartTime)
+   {
+
+      if(System.getProperty("qqq.mongodb.logQueries", "false").equals("true"))
+      {
+         try
+         {
+            if(System.getProperty("qqq.mongodb.logQueries.output", "logger").equalsIgnoreCase("system.out"))
+            {
+               System.out.println("Table: " + tableName + ", Action: " + actionName + ", Query: " + query);
+
+               if(queryStartTime != null)
+               {
+                  System.out.println("Query Took [" + QValueFormatter.formatValue(DisplayFormat.COMMAS, (System.currentTimeMillis() - queryStartTime)) + "] ms");
+               }
+            }
+            else
+            {
+               LOG.debug("Running Query", logPair("table", tableName), logPair("action", actionName), logPair("query", query), logPair("millis", queryStartTime == null ? null : (System.currentTimeMillis() - queryStartTime)));
+            }
+         }
+         catch(Exception e)
+         {
+            LOG.debug("Error logging query...", e);
+         }
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoClientContainer.java

@@ -0,0 +1,158 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import com.mongodb.client.ClientSession;
+import com.mongodb.client.MongoClient;
+
+
+/*******************************************************************************
+ ** Wrapper around a MongoClient, ClientSession, and a boolean to help signal
+ ** where it was opened (e.g., so you know if you need to close it yourself, or
+ ** if it came from someone else (e.g., via an input transaction)).
+ *******************************************************************************/
+public class MongoClientContainer
+{
+   private MongoClient   mongoClient;
+   private ClientSession mongoSession;
+   private boolean       needToClose;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public MongoClientContainer(MongoClient mongoClient, ClientSession mongoSession, boolean needToClose)
+   {
+      this.mongoClient = mongoClient;
+      this.mongoSession = mongoSession;
+      this.needToClose = needToClose;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for mongoClient
+    *******************************************************************************/
+   public MongoClient getMongoClient()
+   {
+      return (this.mongoClient);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for mongoClient
+    *******************************************************************************/
+   public void setMongoClient(MongoClient mongoClient)
+   {
+      this.mongoClient = mongoClient;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for mongoClient
+    *******************************************************************************/
+   public MongoClientContainer withMongoClient(MongoClient mongoClient)
+   {
+      this.mongoClient = mongoClient;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for mongoSession
+    *******************************************************************************/
+   public ClientSession getMongoSession()
+   {
+      return (this.mongoSession);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for mongoSession
+    *******************************************************************************/
+   public void setMongoSession(ClientSession mongoSession)
+   {
+      this.mongoSession = mongoSession;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for mongoSession
+    *******************************************************************************/
+   public MongoClientContainer withMongoSession(ClientSession mongoSession)
+   {
+      this.mongoSession = mongoSession;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for needToClose
+    *******************************************************************************/
+   public boolean getNeedToClose()
+   {
+      return (this.needToClose);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for needToClose
+    *******************************************************************************/
+   public void setNeedToClose(boolean needToClose)
+   {
+      this.needToClose = needToClose;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for needToClose
+    *******************************************************************************/
+   public MongoClientContainer withNeedToClose(boolean needToClose)
+   {
+      this.needToClose = needToClose;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void closeIfNeeded()
+   {
+      if(needToClose)
+      {
+         mongoSession.close();
+         mongoClient.close();
+      }
+   }
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateAction.java

@@ -0,0 +1,275 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.AggregateIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Accumulators;
+import com.mongodb.client.model.Aggregates;
+import com.mongodb.client.model.BsonField;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MongoDBAggregateAction extends AbstractMongoDBAction implements AggregateInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
+
+   private ActionTimeoutHelper actionTimeoutHelper;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("checkstyle:indentation")
+   public AggregateOutput execute(AggregateInput aggregateInput) throws QException
+   {
+      MongoClientContainer mongoClientContainer = null;
+
+      Long       queryStartTime = System.currentTimeMillis();
+      List<Bson> queryToLog     = new ArrayList<>();
+
+      try
+      {
+         AggregateOutput        aggregateOutput  = new AggregateOutput();
+         QTableMetaData         table            = aggregateInput.getTable();
+         String                 backendTableName = getBackendTableName(table);
+         MongoDBBackendMetaData backend          = (MongoDBBackendMetaData) aggregateInput.getBackend();
+
+         mongoClientContainer = openClient(backend, null); // todo - aggregate input has no transaction!?
+         MongoDatabase             database   = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
+         MongoCollection<Document> collection = database.getCollection(backendTableName);
+
+         QQueryFilter filter      = aggregateInput.getFilter();
+         Bson         searchQuery = makeSearchQueryDocument(table, filter);
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         actionTimeoutHelper = new ActionTimeoutHelper(aggregateInput.getTimeoutSeconds(), TimeUnit.SECONDS, new TimeoutCanceller(mongoClientContainer));
+         actionTimeoutHelper.start();
+
+         /////////////////////////////////////////////////////////////////////////
+         // we have to submit a list of BSON objects to the aggregate function. //
+         // the first one is the search query                                   //
+         // second is the group-by stuff, which we'll explain as we build it    //
+         /////////////////////////////////////////////////////////////////////////
+         List<Bson> bsonList = new ArrayList<>();
+         bsonList.add(Aggregates.match(searchQuery));
+         setQueryInQueryStat(searchQuery);
+         queryToLog = bsonList;
+
+         //////////////////////////////////////////////////////////////////////////////////////
+         // if there are group-by fields, then we need to build a document with those fields //
+         // not sure what the whole name, $name is, but, go go mongo                         //
+         //////////////////////////////////////////////////////////////////////////////////////
+         Document groupValueDocument = new Document();
+         if(CollectionUtils.nullSafeHasContents(aggregateInput.getGroupBys()))
+         {
+            for(GroupBy groupBy : aggregateInput.getGroupBys())
+            {
+               String name = getFieldBackendName(table.getField(groupBy.getFieldName()));
+               groupValueDocument.append(name, "$" + name);
+            }
+         }
+
+         ////////////////////////////////////////////////////////////////////
+         // next build a list of accumulator fields - for aggregate values //
+         ////////////////////////////////////////////////////////////////////
+         List<BsonField> bsonFields = new ArrayList<>();
+         for(Aggregate aggregate : aggregateInput.getAggregates())
+         {
+            String fieldName  = aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase();
+            String expression = "$" + getFieldBackendName(table.getField(aggregate.getFieldName()));
+
+            bsonFields.add(switch(aggregate.getOperator())
+            {
+               case COUNT -> Accumulators.sum(fieldName, 1); // count... do a sum of 1's
+               case COUNT_DISTINCT -> throw new QException("Count Distinct is not supported for MongoDB tables at this time.");
+               case SUM -> Accumulators.sum(fieldName, expression);
+               case MIN -> Accumulators.min(fieldName, expression);
+               case MAX -> Accumulators.max(fieldName, expression);
+               case AVG -> Accumulators.avg(fieldName, expression);
+            });
+         }
+
+         ///////////////////////////////////////////////////////////////////////////////////
+         // add the group-by fields and the aggregates in the group stage of the pipeline //
+         ///////////////////////////////////////////////////////////////////////////////////
+         bsonList.add(Aggregates.group(groupValueDocument, bsonFields));
+
+         //////////////////////////////////////////////
+         // if there are any order-bys, add them too //
+         //////////////////////////////////////////////
+         if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys()))
+         {
+            Document sortValue = new Document();
+            for(QFilterOrderBy orderBy : filter.getOrderBys())
+            {
+               String fieldName;
+               if(orderBy instanceof QFilterOrderByAggregate orderByAggregate)
+               {
+                  Aggregate aggregate = orderByAggregate.getAggregate();
+                  fieldName = aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase();
+               }
+               else if(orderBy instanceof QFilterOrderByGroupBy orderByGroupBy)
+               {
+                  fieldName = "_id." + getFieldBackendName(table.getField(orderByGroupBy.getGroupBy().getFieldName()));
+               }
+               else
+               {
+                  ///////////////////////////////////////////////////
+                  // does this happen?  should it be "_id." if so? //
+                  ///////////////////////////////////////////////////
+                  fieldName = getFieldBackendName(table.getField(orderBy.getFieldName()));
+               }
+
+               sortValue.append(fieldName, orderBy.getIsAscending() ? 1 : -1);
+            }
+
+            bsonList.add(new Document("$sort", sortValue));
+         }
+
+         ////////////////////////////////////////////////////////
+         // todo - system property to control (like print-sql) //
+         ////////////////////////////////////////////////////////
+         // LOG.debug(bsonList.toString());
+
+         ///////////////////////////
+         // execute the aggregate //
+         ///////////////////////////
+         AggregateIterable<Document> aggregates = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList);
+
+         List<AggregateResult> results = new ArrayList<>();
+         aggregateOutput.setResults(results);
+
+         /////////////////////
+         // process results //
+         /////////////////////
+         for(Document document : aggregates)
+         {
+            /////////////////////////////////////////////////////////////////////////
+            // once we've started getting results, go ahead and cancel the timeout //
+            /////////////////////////////////////////////////////////////////////////
+            actionTimeoutHelper.cancel();
+            setQueryStatFirstResultTime();
+
+            AggregateResult result = new AggregateResult();
+            results.add(result);
+
+            ////////////////////////////////////////////////////////////////
+            // get group by values (if there are any) out of the document //
+            ////////////////////////////////////////////////////////////////
+            for(GroupBy groupBy : CollectionUtils.nonNullList(aggregateInput.getGroupBys()))
+            {
+               Document idDocument = (Document) document.get("_id");
+               Object   value      = idDocument.get(groupBy.getFieldName());
+               result.withGroupByValue(groupBy, ValueUtils.getValueAsFieldType(groupBy.getType(), value));
+            }
+
+            //////////////////////////////////////////
+            // get aggregate values out of document //
+            //////////////////////////////////////////
+            for(Aggregate aggregate : aggregateInput.getAggregates())
+            {
+               QFieldMetaData field     = table.getField(aggregate.getFieldName());
+               QFieldType     fieldType = aggregate.getFieldType();
+               if(fieldType == null)
+               {
+                  fieldType = field.getType();
+               }
+               if(fieldType.equals(QFieldType.INTEGER) && (aggregate.getOperator().equals(AggregateOperator.AVG)))
+               {
+                  fieldType = QFieldType.DECIMAL;
+               }
+
+               Object value = document.get(aggregate.getFieldName() + "_" + aggregate.getOperator().toString().toLowerCase());
+               result.withAggregateValue(aggregate, ValueUtils.getValueAsFieldType(fieldType, value));
+            }
+         }
+
+         return (aggregateOutput);
+      }
+      catch(Exception e)
+      {
+         if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
+         {
+            setQueryStatFirstResultTime();
+            throw (new QUserFacingException("Aggregate timed out."));
+         }
+
+         /*
+         /////////////////////////////////////////////////////////////////////////////////////
+         // this was copied from RDBMS - not sure where/how/if it's being used there though //
+         /////////////////////////////////////////////////////////////////////////////////////
+         if(isCancelled)
+         {
+            throw (new QUserFacingException("Aggregate was cancelled."));
+         }
+         */
+
+         LOG.warn("Error executing aggregate", e);
+         throw new QException("Error executing aggregate", e);
+      }
+      finally
+      {
+         logQuery(getBackendTableName(aggregateInput.getTable()), "aggregate", queryToLog, queryStartTime);
+
+         if(mongoClientContainer != null)
+         {
+            mongoClientContainer.closeIfNeeded();
+         }
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountAction.java

@@ -0,0 +1,137 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.AggregateIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Accumulators;
+import com.mongodb.client.model.Aggregates;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MongoDBCountAction extends AbstractMongoDBAction implements CountInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
+
+   private ActionTimeoutHelper actionTimeoutHelper;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public CountOutput execute(CountInput countInput) throws QException
+   {
+      MongoClientContainer mongoClientContainer = null;
+
+      Long       queryStartTime = System.currentTimeMillis();
+      List<Bson> queryToLog     = new ArrayList<>();
+
+      try
+      {
+         CountOutput            countOutput      = new CountOutput();
+         QTableMetaData         table            = countInput.getTable();
+         String                 backendTableName = getBackendTableName(table);
+         MongoDBBackendMetaData backend          = (MongoDBBackendMetaData) countInput.getBackend();
+
+         mongoClientContainer = openClient(backend, null); // todo - count input has no transaction!?
+         MongoDatabase             database   = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
+         MongoCollection<Document> collection = database.getCollection(backendTableName);
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         actionTimeoutHelper = new ActionTimeoutHelper(countInput.getTimeoutSeconds(), TimeUnit.SECONDS, new TimeoutCanceller(mongoClientContainer));
+         actionTimeoutHelper.start();
+
+         QQueryFilter filter      = countInput.getFilter();
+         Bson         searchQuery = makeSearchQueryDocument(table, filter);
+         queryToLog.add(searchQuery);
+         setQueryInQueryStat(searchQuery);
+
+         List<Bson> bsonList = List.of(
+            Aggregates.match(searchQuery),
+            Aggregates.group("_id", Accumulators.sum("count", 1)));
+
+         AggregateIterable<Document> aggregate = collection.aggregate(mongoClientContainer.getMongoSession(), bsonList);
+
+         Document document = aggregate.first();
+         countOutput.setCount(document == null ? 0 : document.get("count", Integer.class));
+
+         actionTimeoutHelper.cancel();
+         setQueryStatFirstResultTime();
+
+         return (countOutput);
+      }
+      catch(Exception e)
+      {
+         if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
+         {
+            setQueryStatFirstResultTime();
+            throw (new QUserFacingException("Count timed out."));
+         }
+
+         /*
+         /////////////////////////////////////////////////////////////////////////////////////
+         // this was copied from RDBMS - not sure where/how/if it's being used there though //
+         /////////////////////////////////////////////////////////////////////////////////////
+         if(isCancelled)
+         {
+            throw (new QUserFacingException("Count was cancelled."));
+         }
+         */
+
+         LOG.warn("Error executing count", e);
+         throw new QException("Error executing count", e);
+      }
+      finally
+      {
+         logQuery(getBackendTableName(countInput.getTable()), "count", queryToLog, queryStartTime);
+
+         if(mongoClientContainer != null)
+         {
+            mongoClientContainer.closeIfNeeded();
+         }
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteAction.java

@@ -0,0 +1,138 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.result.DeleteResult;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.bson.types.ObjectId;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MongoDBDeleteAction extends AbstractMongoDBAction implements DeleteInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public boolean supportsQueryFilterInput()
+   {
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public DeleteOutput execute(DeleteInput deleteInput) throws QException
+   {
+      MongoClientContainer mongoClientContainer = null;
+
+      Long       queryStartTime = System.currentTimeMillis();
+      List<Bson> queryToLog     = new ArrayList<>();
+
+      try
+      {
+         DeleteOutput           deleteOutput     = new DeleteOutput();
+         QTableMetaData         table            = deleteInput.getTable();
+         String                 backendTableName = getBackendTableName(table);
+         MongoDBBackendMetaData backend          = (MongoDBBackendMetaData) deleteInput.getBackend();
+
+         mongoClientContainer = openClient(backend, deleteInput.getTransaction());
+         MongoDatabase             database   = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
+         MongoCollection<Document> collection = database.getCollection(backendTableName);
+
+         QQueryFilter queryFilter = deleteInput.getQueryFilter();
+         Bson         searchQuery;
+         if(CollectionUtils.nullSafeHasContents(deleteInput.getPrimaryKeys()))
+         {
+            searchQuery = Filters.in("_id", deleteInput.getPrimaryKeys().stream().map(id -> new ObjectId(ValueUtils.getValueAsString(id))).toList());
+         }
+         else if(queryFilter != null && queryFilter.hasAnyCriteria())
+         {
+            QQueryFilter filter = queryFilter;
+            searchQuery = makeSearchQueryDocument(table, filter);
+         }
+         else
+         {
+            LOG.info("Missing both primary keys and a search filter in delete request - exiting with noop", logPair("tableName", table.getName()));
+            return (deleteOutput);
+         }
+
+         queryToLog.add(searchQuery);
+
+         ////////////////////////////////////////////////////////
+         // todo - system property to control (like print-sql) //
+         ////////////////////////////////////////////////////////
+         // LOG.debug(searchQuery);
+
+         DeleteResult deleteResult = collection.deleteMany(mongoClientContainer.getMongoSession(), searchQuery);
+         deleteOutput.setDeletedRecordCount((int) deleteResult.getDeletedCount());
+
+         //////////////////////////////////////////////////////////////////////////
+         // todo any way to get records with errors or warnings for deleteOutput //
+         //////////////////////////////////////////////////////////////////////////
+
+         return (deleteOutput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error executing delete", e);
+         throw new QException("Error executing delete", e);
+      }
+      finally
+      {
+         logQuery(getBackendTableName(deleteInput.getTable()), "delete", queryToLog, queryStartTime);
+
+         if(mongoClientContainer != null)
+         {
+            mongoClientContainer.closeIfNeeded();
+         }
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertAction.java

@@ -0,0 +1,146 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.result.InsertManyResult;
+import org.bson.BsonValue;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MongoDBInsertAction extends AbstractMongoDBAction implements InsertInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBInsertAction.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public InsertOutput execute(InsertInput insertInput) throws QException
+   {
+      MongoClientContainer mongoClientContainer = null;
+      InsertOutput         rs                   = new InsertOutput();
+      List<QRecord>        outputRecords        = new ArrayList<>();
+      rs.setRecords(outputRecords);
+
+      Long       queryStartTime = System.currentTimeMillis();
+      List<Bson> queryToLog     = new ArrayList<>();
+
+      try
+      {
+         QTableMetaData         table            = insertInput.getTable();
+         String                 backendTableName = getBackendTableName(table);
+         MongoDBBackendMetaData backend          = (MongoDBBackendMetaData) insertInput.getBackend();
+
+         mongoClientContainer = openClient(backend, insertInput.getTransaction());
+         MongoDatabase             database   = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
+         MongoCollection<Document> collection = database.getCollection(backendTableName);
+
+         ///////////////////////////////////////////////////////////////////////////
+         // page over input record list (assuming some size of batch is too big?) //
+         ///////////////////////////////////////////////////////////////////////////
+         for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), getPageSize()))
+         {
+            //////////////////////////////////////////////////////////////////
+            // build list of documents from records w/o errors in this page //
+            //////////////////////////////////////////////////////////////////
+            List<Document> documentList = new ArrayList<>();
+            for(QRecord record : page)
+            {
+               if(CollectionUtils.nullSafeHasContents(record.getErrors()))
+               {
+                  continue;
+               }
+
+               Document document = recordToDocument(table, record);
+               documentList.add(document);
+               queryToLog.add(document);
+            }
+
+            /////////////////////////////////////
+            // skip pages that were all errors //
+            /////////////////////////////////////
+            if(documentList.isEmpty())
+            {
+               continue;
+            }
+
+            ///////////////////////////////////////////////
+            // actually do the insert                    //
+            // todo - how are errors returned by mongo?? //
+            ///////////////////////////////////////////////
+            InsertManyResult insertManyResult = collection.insertMany(mongoClientContainer.getMongoSession(), documentList);
+
+            /////////////////////////////////
+            // put ids on inserted records //
+            /////////////////////////////////
+            int index = 0;
+            for(QRecord record : page)
+            {
+               QRecord outputRecord = new QRecord(record);
+               rs.addRecord(outputRecord);
+
+               if(CollectionUtils.nullSafeIsEmpty(record.getErrors()))
+               {
+                  BsonValue insertedId = insertManyResult.getInsertedIds().get(index++);
+                  String    idString   = insertedId.asObjectId().getValue().toString();
+                  outputRecord.setValue(table.getPrimaryKeyField(), idString);
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing insert: " + e.getMessage(), e);
+      }
+      finally
+      {
+         logQuery(getBackendTableName(insertInput.getTable()), "insert", queryToLog, queryStartTime);
+
+         if(mongoClientContainer != null)
+         {
+            mongoClientContainer.closeIfNeeded();
+         }
+      }
+
+      return (rs);
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryAction.java

@@ -0,0 +1,186 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MongoDBQueryAction extends AbstractMongoDBAction implements QueryInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBBackendModule.class);
+
+   private ActionTimeoutHelper actionTimeoutHelper;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public QueryOutput execute(QueryInput queryInput) throws QException
+   {
+      MongoClientContainer mongoClientContainer = null;
+
+      Long       queryStartTime = System.currentTimeMillis();
+      List<Bson> queryToLog     = new ArrayList<>();
+
+      try
+      {
+         QueryOutput            queryOutput      = new QueryOutput(queryInput);
+         QTableMetaData         table            = queryInput.getTable();
+         String                 backendTableName = getBackendTableName(table);
+         MongoDBBackendMetaData backend          = (MongoDBBackendMetaData) queryInput.getBackend();
+
+         mongoClientContainer = openClient(backend, queryInput.getTransaction());
+         MongoDatabase             database   = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
+         MongoCollection<Document> collection = database.getCollection(backendTableName);
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         actionTimeoutHelper = new ActionTimeoutHelper(queryInput.getTimeoutSeconds(), TimeUnit.SECONDS, new TimeoutCanceller(mongoClientContainer));
+         actionTimeoutHelper.start();
+
+         /////////////////////////
+         // set up filter/query //
+         /////////////////////////
+         QQueryFilter filter      = queryInput.getFilter();
+         Bson         searchQuery = makeSearchQueryDocument(table, filter);
+         queryToLog.add(searchQuery);
+         setQueryInQueryStat(searchQuery);
+
+         ////////////////////////////////////////////////////////////
+         // create cursor - further adjustments to it still follow //
+         ////////////////////////////////////////////////////////////
+         FindIterable<Document> cursor = collection.find(mongoClientContainer.getMongoSession(), searchQuery);
+
+         ///////////////////////////////////
+         // add a sort operator if needed //
+         ///////////////////////////////////
+         if(filter != null && CollectionUtils.nullSafeHasContents(filter.getOrderBys()))
+         {
+            Document sortDocument = new Document();
+            queryToLog.add(sortDocument);
+            for(QFilterOrderBy orderBy : filter.getOrderBys())
+            {
+               String fieldBackendName = getFieldBackendName(table.getField(orderBy.getFieldName()));
+               sortDocument.put(fieldBackendName, orderBy.getIsAscending() ? 1 : -1);
+            }
+            cursor.sort(sortDocument);
+         }
+
+         ////////////////////////
+         // apply skip & limit //
+         ////////////////////////
+         if(filter != null)
+         {
+            if(filter.getSkip() != null)
+            {
+               cursor.skip(filter.getSkip());
+            }
+
+            if(filter.getLimit() != null)
+            {
+               cursor.limit(filter.getLimit());
+            }
+         }
+
+         ////////////////////////////////////////////
+         // iterate over results, building records //
+         ////////////////////////////////////////////
+         for(Document document : cursor)
+         {
+            /////////////////////////////////////////////////////////////////////////
+            // once we've started getting results, go ahead and cancel the timeout //
+            /////////////////////////////////////////////////////////////////////////
+            actionTimeoutHelper.cancel();
+            setQueryStatFirstResultTime();
+
+            QRecord record = documentToRecord(table, document);
+            queryOutput.addRecord(record);
+
+            if(queryInput.getAsyncJobCallback().wasCancelRequested())
+            {
+               LOG.info("Breaking query job, as requested.");
+               break;
+            }
+         }
+
+         return (queryOutput);
+      }
+      catch(Exception e)
+      {
+         if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
+         {
+            setQueryStatFirstResultTime();
+            throw (new QUserFacingException("Query timed out."));
+         }
+
+         /*
+         /////////////////////////////////////////////////////////////////////////////////////
+         // this was copied from RDBMS - not sure where/how/if it's being used there though //
+         /////////////////////////////////////////////////////////////////////////////////////
+         if(isCancelled)
+         {
+            throw (new QUserFacingException("Query was cancelled."));
+         }
+         */
+
+         LOG.warn("Error executing query", e);
+         throw new QException("Error executing query", e);
+      }
+      finally
+      {
+         logQuery(getBackendTableName(queryInput.getTable()), "query", queryToLog, queryStartTime);
+
+         if(mongoClientContainer != null)
+         {
+            mongoClientContainer.closeIfNeeded();
+         }
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransaction.java

@@ -0,0 +1,215 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.ClientSession;
+import com.mongodb.client.MongoClient;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** MongoDB implementation of backend transaction.
+ **
+ ** Stores a mongoClient and clientSession.
+ **
+ ** Also keeps track of if the specific mongo backend being used supports transactions,
+ ** as, it appears that single-node instances do not, and they throw errors if
+ ** you try to do transaction operations in them...  This is configured by the
+ ** corresponding field in the backend metaData.
+ *******************************************************************************/
+public class MongoDBTransaction extends QBackendTransaction
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBTransaction.class);
+
+   private boolean       transactionsSupported;
+   private MongoClient   mongoClient;
+   private ClientSession clientSession;
+
+   private Instant openedAt                  = Instant.now();
+   private Integer logSlowTransactionSeconds = null;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public MongoDBTransaction(MongoDBBackendMetaData backend, MongoClient mongoClient)
+   {
+      this.transactionsSupported = backend.getTransactionsSupported();
+      ClientSession clientSession = mongoClient.startSession();
+
+      if(transactionsSupported)
+      {
+         clientSession.startTransaction();
+      }
+
+      String propertyName = "qqq.mongodb.logSlowTransactionSeconds";
+      try
+      {
+         logSlowTransactionSeconds = Integer.parseInt(System.getProperty(propertyName, "10"));
+      }
+      catch(Exception e)
+      {
+         LOG.debug("Error reading property [" + propertyName + "] value as integer", e);
+      }
+
+      this.mongoClient = mongoClient;
+      this.clientSession = clientSession;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void commit() throws QException
+   {
+      try
+      {
+         Instant commitAt = Instant.now();
+
+         Duration duration = Duration.between(openedAt, commitAt);
+         if(logSlowTransactionSeconds != null && duration.compareTo(Duration.ofSeconds(logSlowTransactionSeconds)) > 0)
+         {
+            LOG.info("Committing long-running transaction", logPair("durationSeconds", duration.getSeconds()));
+         }
+         else
+         {
+            LOG.debug("Committing transaction");
+         }
+
+         if(transactionsSupported)
+         {
+            this.clientSession.commitTransaction();
+            LOG.debug("Commit complete");
+         }
+         else
+         {
+            LOG.debug("Request to commit, but transactions not supported in this mongodb backend");
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error committing transaction", e);
+         throw new QException("Error committing transaction: " + e.getMessage(), e);
+      }
+      finally
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // reset this - as after one commit, the transaction is essentially re-opened for any future statements that run on it //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         openedAt = Instant.now();
+         if(transactionsSupported)
+         {
+            this.clientSession.startTransaction();
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void rollback() throws QException
+   {
+      try
+      {
+         if(transactionsSupported)
+         {
+            LOG.info("Rolling back transaction");
+            this.clientSession.abortTransaction();
+            LOG.info("Rollback complete");
+         }
+         else
+         {
+            LOG.debug("Request to rollback, but transactions not supported in this mongodb backend");
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error rolling back transaction", e);
+         throw new QException("Error rolling back transaction: " + e.getMessage(), e);
+      }
+      finally
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // reset this - as after one commit, the transaction is essentially re-opened for any future statements that run on it //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         openedAt = Instant.now();
+         if(transactionsSupported)
+         {
+            this.clientSession.startTransaction();
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void close()
+   {
+      try
+      {
+         this.clientSession.close();
+         this.mongoClient.close();
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error closing connection - possible mongo connection leak", e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for mongoClient
+    **
+    *******************************************************************************/
+   public MongoClient getMongoClient()
+   {
+      return mongoClient;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for clientSession
+    **
+    *******************************************************************************/
+   public ClientSession getClientSession()
+   {
+      return clientSession;
+   }
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateAction.java

@@ -0,0 +1,169 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.UpdateActionRecordSplitHelper;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.ListingHash;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Updates;
+import com.mongodb.client.result.UpdateResult;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+import org.bson.types.ObjectId;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class MongoDBUpdateAction extends AbstractMongoDBAction implements UpdateInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(MongoDBUpdateAction.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public UpdateOutput execute(UpdateInput updateInput) throws QException
+   {
+      MongoClientContainer   mongoClientContainer = null;
+      QTableMetaData         table                = updateInput.getTable();
+      String                 backendTableName     = getBackendTableName(table);
+      MongoDBBackendMetaData backend              = (MongoDBBackendMetaData) updateInput.getBackend();
+
+      UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper();
+      updateActionRecordSplitHelper.init(updateInput);
+
+      UpdateOutput rs = new UpdateOutput();
+      rs.setRecords(updateActionRecordSplitHelper.getOutputRecords());
+
+      if(!updateActionRecordSplitHelper.getHaveAnyWithoutErrors())
+      {
+         LOG.info("Exiting early - all records have some error.");
+         return (rs);
+      }
+
+      try
+      {
+         mongoClientContainer = openClient(backend, updateInput.getTransaction());
+         MongoDatabase             database   = mongoClientContainer.getMongoClient().getDatabase(backend.getDatabaseName());
+         MongoCollection<Document> collection = database.getCollection(backendTableName);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         // process each distinct list of fields being updated (e.g., each different SQL statement) //
+         /////////////////////////////////////////////////////////////////////////////////////////////
+         ListingHash<List<String>, QRecord> recordsByFieldBeingUpdated = updateActionRecordSplitHelper.getRecordsByFieldBeingUpdated();
+         for(Map.Entry<List<String>, List<QRecord>> entry : recordsByFieldBeingUpdated.entrySet())
+         {
+            updateRecordsWithMatchingListOfFields(updateInput, mongoClientContainer, collection, table, entry.getValue(), entry.getKey());
+         }
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing update: " + e.getMessage(), e);
+      }
+      finally
+      {
+         if(mongoClientContainer != null)
+         {
+            mongoClientContainer.closeIfNeeded();
+         }
+      }
+
+      return (rs);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void updateRecordsWithMatchingListOfFields(UpdateInput updateInput, MongoClientContainer mongoClientContainer, MongoCollection<Document> collection, QTableMetaData table, List<QRecord> recordList, List<String> fieldsBeingUpdated)
+   {
+      boolean allAreTheSame = UpdateActionRecordSplitHelper.areAllValuesBeingUpdatedTheSame(updateInput, recordList, fieldsBeingUpdated);
+      if(allAreTheSame)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if all records w/ this set of fields have the same values, we can do 1 big updateMany on the whole list //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         updateRecordsWithMatchingValuesAndFields(mongoClientContainer, collection, table, recordList, fieldsBeingUpdated);
+      }
+      else
+      {
+         /////////////////////////////////////////////////////////////////////////
+         // else, if not all are being updated the same, then update one-by-one //
+         /////////////////////////////////////////////////////////////////////////
+         for(QRecord record : recordList)
+         {
+            updateRecordsWithMatchingValuesAndFields(mongoClientContainer, collection, table, List.of(record), fieldsBeingUpdated);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void updateRecordsWithMatchingValuesAndFields(MongoClientContainer mongoClientContainer, MongoCollection<Document> collection, QTableMetaData table, List<QRecord> recordList, List<String> fieldsBeingUpdated)
+   {
+      Long       queryStartTime = System.currentTimeMillis();
+      List<Bson> queryToLog     = new ArrayList<>();
+
+      QRecord        firstRecord = recordList.get(0);
+      List<ObjectId> ids         = recordList.stream().map(r -> new ObjectId(r.getValueString("id"))).toList();
+      Bson           filter      = Filters.in("_id", ids);
+      queryToLog.add(filter);
+
+      List<Bson> updates = new ArrayList<>();
+      for(String fieldName : fieldsBeingUpdated)
+      {
+         QFieldMetaData field            = table.getField(fieldName);
+         String         fieldBackendName = getFieldBackendName(field);
+         Bson set = Updates.set(fieldBackendName, firstRecord.getValue(fieldName));
+         updates.add(set);
+         queryToLog.add(set);
+      }
+      Bson changes = Updates.combine(updates);
+
+      UpdateResult updateResult = collection.updateMany(mongoClientContainer.getMongoSession(), filter, changes);
+      // todo - anything with the output??
+
+      logQuery(getBackendTableName(table), "update", queryToLog, queryStartTime);
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/actions/TimeoutCanceller.java

@@ -1,6 +1,6 @@
 /*
  * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
  * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
  * contact@kingsrook.com
  * https://github.com/Kingsrook/
@@ -19,26 +19,50 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-package com.kingsrook.qqq.backend.core.actions.interfaces;
+package com.kingsrook.qqq.backend.module.mongodb.actions;
 
 
-import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 
 
 /*******************************************************************************
- **
+ ** Helper to cancel statements that timeout.
  *******************************************************************************/
-public interface QActionInterface
+public class TimeoutCanceller implements Runnable
 {
+   private static final QLogger              LOG = QLogger.getLogger(TimeoutCanceller.class);
+   private final        MongoClientContainer mongoClientContainer;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public TimeoutCanceller(MongoClientContainer mongoClientContainer)
+   {
+      this.mongoClientContainer = mongoClientContainer;
+   }
+
+
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   default QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
+   @Override
+   public void run()
    {
-      return (new QBackendTransaction());
-   }
+      try
+      {
+         mongoClientContainer.closeIfNeeded();
+         LOG.info("Cancelled timed out query");
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error trying to cancel statement after timeout", e);
+      }
 
+      throw (new QRuntimeException("Statement timed out and was cancelled."));
+   }
 }

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBBackendMetaData.java

@@ -0,0 +1,343 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
+
+
+/*******************************************************************************
+ ** Meta-data to provide details of a MongoDB backend (e.g., connection params)
+ *******************************************************************************/
+public class MongoDBBackendMetaData extends QBackendMetaData
+{
+   private String  host;
+   private Integer port;
+   private String  databaseName;
+   private String  username;
+   private String  password;
+   private String  authSourceDatabase;
+   private String  urlSuffix;
+
+   private boolean transactionsSupported = true;
+
+
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public MongoDBBackendMetaData()
+   {
+      super();
+      setBackendType(MongoDBBackendModule.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter, override to help fluent flows
+    *******************************************************************************/
+   @Override
+   public MongoDBBackendMetaData withName(String name)
+   {
+      setName(name);
+      return this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for host
+    **
+    *******************************************************************************/
+   public String getHost()
+   {
+      return host;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for host
+    **
+    *******************************************************************************/
+   public void setHost(String host)
+   {
+      this.host = host;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for host
+    **
+    *******************************************************************************/
+   public MongoDBBackendMetaData withHost(String host)
+   {
+      this.host = host;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for port
+    **
+    *******************************************************************************/
+   public Integer getPort()
+   {
+      return port;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for port
+    **
+    *******************************************************************************/
+   public void setPort(Integer port)
+   {
+      this.port = port;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for port
+    **
+    *******************************************************************************/
+   public MongoDBBackendMetaData withPort(Integer port)
+   {
+      this.port = port;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for username
+    **
+    *******************************************************************************/
+   public String getUsername()
+   {
+      return username;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for username
+    **
+    *******************************************************************************/
+   public void setUsername(String username)
+   {
+      this.username = username;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for username
+    **
+    *******************************************************************************/
+   public MongoDBBackendMetaData withUsername(String username)
+   {
+      this.username = username;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for password
+    **
+    *******************************************************************************/
+   public String getPassword()
+   {
+      return password;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for password
+    **
+    *******************************************************************************/
+   public void setPassword(String password)
+   {
+      this.password = password;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for password
+    **
+    *******************************************************************************/
+   public MongoDBBackendMetaData withPassword(String password)
+   {
+      this.password = password;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Called by the QInstanceEnricher - to do backend-type-specific enrichments.
+    ** Original use case is:  reading secrets into fields (e.g., passwords).
+    *******************************************************************************/
+   @Override
+   public void enrich()
+   {
+      super.enrich();
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+      username = interpreter.interpret(username);
+      password = interpreter.interpret(password);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for urlSuffix
+    *******************************************************************************/
+   public String getUrlSuffix()
+   {
+      return (this.urlSuffix);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for urlSuffix
+    *******************************************************************************/
+   public void setUrlSuffix(String urlSuffix)
+   {
+      this.urlSuffix = urlSuffix;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for urlSuffix
+    *******************************************************************************/
+   public MongoDBBackendMetaData withUrlSuffix(String urlSuffix)
+   {
+      this.urlSuffix = urlSuffix;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for databaseName
+    *******************************************************************************/
+   public String getDatabaseName()
+   {
+      return (this.databaseName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for databaseName
+    *******************************************************************************/
+   public void setDatabaseName(String databaseName)
+   {
+      this.databaseName = databaseName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for databaseName
+    *******************************************************************************/
+   public MongoDBBackendMetaData withDatabaseName(String databaseName)
+   {
+      this.databaseName = databaseName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for transactionsSupported
+    *******************************************************************************/
+   public boolean getTransactionsSupported()
+   {
+      return (this.transactionsSupported);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for transactionsSupported
+    *******************************************************************************/
+   public void setTransactionsSupported(boolean transactionsSupported)
+   {
+      this.transactionsSupported = transactionsSupported;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for transactionsSupported
+    *******************************************************************************/
+   public MongoDBBackendMetaData withTransactionsSupported(boolean transactionsSupported)
+   {
+      this.transactionsSupported = transactionsSupported;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for authSourceDatabase
+    *******************************************************************************/
+   public String getAuthSourceDatabase()
+   {
+      return (this.authSourceDatabase);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for authSourceDatabase
+    *******************************************************************************/
+   public void setAuthSourceDatabase(String authSourceDatabase)
+   {
+      this.authSourceDatabase = authSourceDatabase;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for authSourceDatabase
+    *******************************************************************************/
+   public MongoDBBackendMetaData withAuthSourceDatabase(String authSourceDatabase)
+   {
+      this.authSourceDatabase = authSourceDatabase;
+      return (this);
+   }
+
+}

qqq-backend-module-mongodb/src/main/java/com/kingsrook/qqq/backend/module/mongodb/model/metadata/MongoDBTableBackendDetails.java

@@ -0,0 +1,81 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
+import com.kingsrook.qqq.backend.module.mongodb.MongoDBBackendModule;
+
+
+/*******************************************************************************
+ ** Extension of QTableBackendDetails, with details specific to a MongoDB table.
+ *******************************************************************************/
+public class MongoDBTableBackendDetails extends QTableBackendDetails
+{
+   private String tableName;
+
+
+
+   /*******************************************************************************
+    ** Default Constructor.
+    *******************************************************************************/
+   public MongoDBTableBackendDetails()
+   {
+      super();
+      setBackendType(MongoDBBackendModule.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    **
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    **
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent Setter for tableName
+    **
+    *******************************************************************************/
+   public MongoDBTableBackendDetails withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/BaseTest.java

@@ -0,0 +1,157 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb;
+
+
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.module.mongodb.actions.AbstractMongoDBAction;
+import com.kingsrook.qqq.backend.module.mongodb.actions.MongoClientContainer;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoDatabase;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+
+/*******************************************************************************
+ ** Base for all tests in this module
+ *******************************************************************************/
+public class BaseTest
+{
+   private static final QLogger LOG = QLogger.getLogger(BaseTest.class);
+
+   private static GenericContainer<?> mongoDBContainer;
+
+   private static final String MONGO_IMAGE = "mongo:4.2.0-bionic";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeAll
+   static void beforeAll()
+   {
+      System.setProperty("qqq.mongodb.logQueries", "true");
+
+      mongoDBContainer = new GenericContainer<>(DockerImageName.parse(MONGO_IMAGE))
+         .withEnv("MONGO_INITDB_ROOT_USERNAME", TestUtils.MONGO_USERNAME)
+         .withEnv("MONGO_INITDB_ROOT_PASSWORD", TestUtils.MONGO_PASSWORD)
+         .withEnv("MONGO_INITDB_DATABASE", TestUtils.MONGO_DATABASE)
+         .withExposedPorts(TestUtils.MONGO_PORT);
+
+      mongoDBContainer.start();
+   }
+
+
+
+   /*******************************************************************************
+    ** init the QContext with the instance from TestUtils and a new session
+    *******************************************************************************/
+   @BeforeEach
+   void baseBeforeEach()
+   {
+      QContext.init(TestUtils.defineInstance(), new QSession());
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // host could(?) be different, and mapped port will be, so set them in backend meta-data based on our running container //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      MongoDBBackendMetaData backend = (MongoDBBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME);
+      backend.setHost(mongoDBContainer.getHost());
+      backend.setPort(mongoDBContainer.getMappedPort(TestUtils.MONGO_PORT));
+   }
+
+
+
+   /*******************************************************************************
+    ** clear the QContext
+    *******************************************************************************/
+   @AfterEach
+   void baseAfterEach()
+   {
+      clearDatabase();
+
+      QContext.clear();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected static void clearDatabase()
+   {
+      ///////////////////////////////////////
+      // clear test database between tests //
+      ///////////////////////////////////////
+      MongoClient   mongoClient = getMongoClient();
+      MongoDatabase database    = mongoClient.getDatabase(TestUtils.MONGO_DATABASE);
+      database.drop();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected static MongoClient getMongoClient()
+   {
+      MongoDBBackendMetaData backend              = (MongoDBBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME);
+      MongoClientContainer   mongoClientContainer = new AbstractMongoDBAction().openClient(backend, null);
+      MongoClient            mongoClient          = mongoClientContainer.getMongoClient();
+      return mongoClient;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @AfterAll
+   static void afterAll()
+   {
+      // this.mongoDbReplicaSet.close();
+      mongoDBContainer.close();
+   }
+
+
+
+   /*******************************************************************************
+    ** if needed, re-initialize the QInstance in context.
+    *******************************************************************************/
+   protected static void reInitInstanceInContext(QInstance qInstance)
+   {
+      if(qInstance.equals(QContext.getQInstance()))
+      {
+         LOG.warn("Unexpected condition - the same qInstance that is already in the QContext was passed into reInit.  You probably want a new QInstance object instance.");
+      }
+      QContext.init(qInstance, new QSession());
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/TestUtils.java

@@ -0,0 +1,353 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
+
+
+/*******************************************************************************
+ ** Test Utils class for this module
+ **
+ ** Note - tons of copying from RDMBS... wouldn't it be nice to share??
+ *******************************************************************************/
+public class TestUtils
+{
+   public static final String DEFAULT_BACKEND_NAME = "default";
+
+   public static final String TABLE_NAME_PERSON = "personTable";
+
+   public static final String TABLE_NAME_STORE               = "store";
+   public static final String TABLE_NAME_ORDER               = "order";
+   public static final String TABLE_NAME_ORDER_INSTRUCTIONS  = "orderInstructions";
+   public static final String TABLE_NAME_ITEM                = "item";
+   public static final String TABLE_NAME_ORDER_LINE          = "orderLine";
+   public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic";
+   public static final String TABLE_NAME_WAREHOUSE           = "warehouse";
+   public static final String TABLE_NAME_WAREHOUSE_STORE_INT = "warehouseStoreInt";
+
+   public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess";
+
+   public static final String  MONGO_USERNAME = "mongoUser";
+   public static final String  MONGO_PASSWORD = "password";
+   public static final Integer MONGO_PORT     = 27017;
+   public static final String  MONGO_DATABASE = "testDatabase";
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QInstance defineInstance()
+   {
+      QInstance qInstance = new QInstance();
+      qInstance.addBackend(defineBackend());
+      qInstance.addTable(defineTablePerson());
+      qInstance.addPossibleValueSource(definePvsPerson());
+      addOmsTablesAndJoins(qInstance);
+      qInstance.setAuthentication(defineAuthentication());
+      return (qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    ** Define the authentication used in standard tests - using 'mock' type.
+    **
+    *******************************************************************************/
+   public static QAuthenticationMetaData defineAuthentication()
+   {
+      return new QAuthenticationMetaData()
+         .withName("mock")
+         .withType(QAuthenticationType.MOCK);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static MongoDBBackendMetaData defineBackend()
+   {
+      return (new MongoDBBackendMetaData()
+         .withName(DEFAULT_BACKEND_NAME)
+         .withHost("localhost")
+         .withPort(TestUtils.MONGO_PORT)
+         .withUsername(TestUtils.MONGO_USERNAME)
+         .withPassword(TestUtils.MONGO_PASSWORD)
+         .withAuthSourceDatabase("admin")
+         .withDatabaseName(TestUtils.MONGO_DATABASE)
+         .withTransactionsSupported(false));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineTablePerson()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_PERSON)
+         .withLabel("Person")
+         .withRecordLabelFormat("%s %s")
+         .withRecordLabelFields("firstName", "lastName")
+         .withBackendName(DEFAULT_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.STRING).withBackendName("_id"))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("metaData.createDate"))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("metaData.modifyDate"))
+         .withField(new QFieldMetaData("seqNo", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("firstName", QFieldType.STRING))
+         .withField(new QFieldMetaData("lastName", QFieldType.STRING))
+         .withField(new QFieldMetaData("birthDate", QFieldType.DATE))
+         .withField(new QFieldMetaData("email", QFieldType.STRING))
+         .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN))
+         .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL))
+         .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("homeTown", QFieldType.STRING))
+         .withBackendDetails(new MongoDBTableBackendDetails()
+            .withTableName(TABLE_NAME_PERSON));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QPossibleValueSource definePvsPerson()
+   {
+      return (new QPossibleValueSource()
+         .withName(TABLE_NAME_PERSON)
+         .withType(QPossibleValueSourceType.TABLE)
+         .withTableName(TABLE_NAME_PERSON)
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
+      );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void addOmsTablesAndJoins(QInstance qInstance)
+   {
+      qInstance.addTable(defineBaseTable(TABLE_NAME_STORE, "store")
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("name")
+         .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("key"))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order")
+         .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeKey"))
+         .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine"))
+         .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem")))
+         .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_STORE))
+         .withField(new QFieldMetaData("billToPersonId", QFieldType.STRING).withPossibleValueSourceName(TABLE_NAME_PERSON))
+         .withField(new QFieldMetaData("shipToPersonId", QFieldType.STRING).withPossibleValueSourceName(TABLE_NAME_PERSON))
+         .withField(new QFieldMetaData("currentOrderInstructionsId", QFieldType.STRING).withPossibleValueSourceName(TABLE_NAME_PERSON))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_INSTRUCTIONS, "order_instructions")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName("order.storeKey")
+            .withJoinNameChain(List.of("orderInstructionsJoinOrder")))
+         .withField(new QFieldMetaData("orderId", QFieldType.STRING))
+         .withField(new QFieldMetaData("instructions", QFieldType.STRING))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item")
+         .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeKey"))
+         .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine")))
+         .withField(new QFieldMetaData("sku", QFieldType.STRING))
+         .withField(new QFieldMetaData("description", QFieldType.STRING))
+         .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_STORE))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_LINE, "order_line")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName("order.storeKey")
+            .withJoinNameChain(List.of("orderJoinOrderLine")))
+         .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_LINE_ITEM_EXTRINSIC).withJoinName("orderLineJoinLineItemExtrinsic"))
+         .withField(new QFieldMetaData("orderId", QFieldType.STRING))
+         .withField(new QFieldMetaData("sku", QFieldType.STRING))
+         .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_STORE))
+         .withField(new QFieldMetaData("quantity", QFieldType.INTEGER))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_LINE_ITEM_EXTRINSIC, "line_item_extrinsic")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName("order.storeKey")
+            .withJoinNameChain(List.of("orderJoinOrderLine", "orderLineJoinLineItemExtrinsic")))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
+         .withField(new QFieldMetaData("orderLineId", QFieldType.STRING))
+         .withField(new QFieldMetaData("key", QFieldType.STRING))
+         .withField(new QFieldMetaData("value", QFieldType.STRING))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE_STORE_INT, "warehouse_store_int")
+         .withField(new QFieldMetaData("warehouseId", QFieldType.STRING))
+         .withField(new QFieldMetaData("storeKey", QFieldType.INTEGER))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE, "warehouse")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName(TABLE_NAME_WAREHOUSE_STORE_INT + ".storeKey")
+            .withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_WAREHOUSE, TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT)))
+         )
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(TestUtils.TABLE_NAME_WAREHOUSE)
+         .withRightTable(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT)
+         .withInferredName()
+         .withJoinOn(new JoinOn("id", "warehouseId"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinStore")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_STORE)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("storeKey", "key"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinBillToPerson")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_PERSON)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("billToPersonId", "id"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinShipToPerson")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_PERSON)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("shipToPersonId", "id"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("itemJoinStore")
+         .withLeftTable(TABLE_NAME_ITEM)
+         .withRightTable(TABLE_NAME_STORE)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("storeKey", "key"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinOrderLine")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_ORDER_LINE)
+         .withType(JoinType.ONE_TO_MANY)
+         .withJoinOn(new JoinOn("id", "orderId"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderLineJoinItem")
+         .withLeftTable(TABLE_NAME_ORDER_LINE)
+         .withRightTable(TABLE_NAME_ITEM)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("sku", "sku"))
+         .withJoinOn(new JoinOn("storeKey", "storeKey"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderLineJoinLineItemExtrinsic")
+         .withLeftTable(TABLE_NAME_ORDER_LINE)
+         .withRightTable(TABLE_NAME_LINE_ITEM_EXTRINSIC)
+         .withType(JoinType.ONE_TO_MANY)
+         .withJoinOn(new JoinOn("id", "orderLineId"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinCurrentOrderInstructions")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS)
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("currentOrderInstructionsId", "id"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderInstructionsJoinOrder")
+         .withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS)
+         .withRightTable(TABLE_NAME_ORDER)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("orderId", "id"))
+      );
+
+      qInstance.addPossibleValueSource(new QPossibleValueSource()
+         .withName("store")
+         .withType(QPossibleValueSourceType.TABLE)
+         .withTableName(TABLE_NAME_STORE)
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
+      );
+
+      qInstance.addSecurityKeyType(new QSecurityKeyType()
+         .withName(TABLE_NAME_STORE)
+         .withAllAccessKeyName(SECURITY_KEY_STORE_ALL_ACCESS)
+         .withPossibleValueSourceName(TABLE_NAME_STORE));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QTableMetaData defineBaseTable(String tableName, String backendTableName)
+   {
+      return new QTableMetaData()
+         .withName(tableName)
+         .withBackendName(DEFAULT_BACKEND_NAME)
+         .withBackendDetails(new MongoDBTableBackendDetails().withTableName(backendTableName))
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.STRING))
+         .withField(new QFieldMetaData("key", QFieldType.INTEGER));
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBAggregateActionTest.java

@@ -0,0 +1,93 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.GroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByAggregate;
+import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.QFilterOrderByGroupBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBQueryAction 
+ *******************************************************************************/
+class MongoDBAggregateActionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("isEmployed", true).withValue("annualSalary", 1),
+         new QRecord().withValue("firstName", "Linda").withValue("lastName", "Kelkhoff").withValue("isEmployed", true).withValue("annualSalary", 5),
+         new QRecord().withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("isEmployed", true).withValue("annualSalary", 3),
+         new QRecord().withValue("firstName", "James").withValue("lastName", "Maes").withValue("isEmployed", true).withValue("annualSalary", 5),
+         new QRecord().withValue("firstName", "J.D.").withValue("lastName", "Maes").withValue("isEmployed", false).withValue("annualSalary", 0)
+      ));
+      new InsertAction().execute(insertInput);
+
+      {
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+         aggregateInput.setFilter(new QQueryFilter()
+            .withOrderBy(new QFilterOrderByAggregate(new Aggregate("annualSalary", AggregateOperator.MAX)).withIsAscending(false))
+            .withOrderBy(new QFilterOrderByGroupBy(new GroupBy(QFieldType.STRING, "lastName")))
+         );
+         aggregateInput.withAggregate(new Aggregate("id", AggregateOperator.COUNT));
+         aggregateInput.withAggregate(new Aggregate("annualSalary", AggregateOperator.SUM));
+         aggregateInput.withAggregate(new Aggregate("annualSalary", AggregateOperator.MAX));
+         aggregateInput.withGroupBy(new GroupBy(QFieldType.STRING, "lastName"));
+         aggregateInput.withGroupBy(new GroupBy(QFieldType.BOOLEAN, "isEmployed"));
+         AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+         // todo - actual assertions
+      }
+      {
+         AggregateInput aggregateInput = new AggregateInput();
+         aggregateInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+         aggregateInput.withAggregate(new Aggregate("id", AggregateOperator.COUNT));
+         aggregateInput.withAggregate(new Aggregate("annualSalary", AggregateOperator.AVG));
+         AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
+         // todo - actual assertions
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBCountActionTest.java

@@ -0,0 +1,80 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBQueryAction 
+ *******************************************************************************/
+class MongoDBCountActionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////////
+      // directly insert some mongo records //
+      ////////////////////////////////////////
+      MongoDatabase             database   = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE);
+      MongoCollection<Document> collection = database.getCollection(TestUtils.TABLE_NAME_PERSON);
+      collection.insertMany(List.of(
+         Document.parse("""
+            {"firstName": "Darin", "lastName": "Kelkhoff"}"""),
+         Document.parse("""
+            {"firstName": "Tylers", "lastName": "Sample"}"""),
+         Document.parse("""
+            {"firstName": "Tylers", "lastName": "Simple"}"""),
+         Document.parse("""
+            {"firstName": "Thom", "lastName": "Chutterloin"}""")
+      ));
+
+      CountInput countInput = new CountInput();
+      countInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      assertEquals(4, new CountAction().execute(countInput).getCount());
+
+      countInput.setFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "Tylers")));
+      assertEquals(2, new CountAction().execute(countInput).getCount());
+
+      countInput.setFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "assdf")));
+      assertEquals(0, new CountAction().execute(countInput).getCount());
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBDeleteActionTest.java

@@ -0,0 +1,102 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBQueryAction 
+ *******************************************************************************/
+class MongoDBDeleteActionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////////
+      // directly insert some mongo records //
+      ////////////////////////////////////////
+      MongoDatabase             database   = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE);
+      MongoCollection<Document> collection = database.getCollection(TestUtils.TABLE_NAME_PERSON);
+      collection.insertMany(List.of(
+         Document.parse("""
+            {"firstName": "Darin", "lastName": "Kelkhoff"}"""),
+         Document.parse("""
+            {"firstName": "Tylers", "lastName": "Sample"}"""),
+         Document.parse("""
+            {"firstName": "Tylers", "lastName": "Simple"}"""),
+         Document.parse("""
+            {"firstName": "Thom", "lastName": "Chutterloin"}""")
+      ));
+      assertEquals(4, collection.countDocuments());
+
+      //////////////////////////////////////////
+      // do a delete by id (look it up first) //
+      //////////////////////////////////////////
+      {
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+         queryInput.setFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "Darin")));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         String      id0         = queryOutput.getRecords().get(0).getValueString("id");
+
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+         deleteInput.setPrimaryKeys(List.of(id0));
+         assertEquals(1, new DeleteAction().execute(deleteInput).getDeletedRecordCount());
+      }
+      assertEquals(3, collection.countDocuments());
+
+      ///////////////////////////
+      // do a delete by filter //
+      ///////////////////////////
+      {
+         DeleteInput deleteInput = new DeleteInput();
+         deleteInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+         deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, "Tylers")));
+         assertEquals(2, new DeleteAction().execute(deleteInput).getDeletedRecordCount());
+      }
+      assertEquals(1, collection.countDocuments());
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBInsertActionTest.java

@@ -0,0 +1,101 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBQueryAction 
+ *******************************************************************************/
+class MongoDBInsertActionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("firstName", "Darin")
+            .withValue("unmappedField", 1701)
+            .withValue("unmappedList", new ArrayList<>(List.of("A", "B", "C")))
+            .withValue("unmappedObject", new HashMap<>(Map.of("A", 1, "C", true))),
+         new QRecord().withValue("firstName", "Tim"),
+         new QRecord().withValue("firstName", "Tyler")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      /////////////////////////////////////////
+      // make sure id got put on all records //
+      /////////////////////////////////////////
+      for(QRecord record : insertOutput.getRecords())
+      {
+         assertNotNull(record.getValueString("id"));
+      }
+
+      ///////////////////////////////////////////////////
+      // directly query mongo for the inserted records //
+      ///////////////////////////////////////////////////
+      MongoDatabase             database   = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE);
+      MongoCollection<Document> collection = database.getCollection(TestUtils.TABLE_NAME_PERSON);
+      assertEquals(3, collection.countDocuments());
+      for(Document document : collection.find())
+      {
+         /////////////////////////////////////////////////////////////
+         // make sure values got set - including some nested values //
+         /////////////////////////////////////////////////////////////
+         assertNotNull(document.get("firstName"));
+         assertNotNull(document.get("metaData"));
+         assertThat(document.get("metaData")).isInstanceOf(Document.class);
+         assertNotNull(((Document) document.get("metaData")).get("createDate"));
+      }
+
+      Document document = collection.find(new Document("firstName", "Darin")).first();
+      assertNotNull(document);
+      assertEquals(1701, document.get("unmappedField"));
+      assertEquals(List.of("A", "B", "C"), document.get("unmappedList"));
+      assertEquals(Map.of("A", 1, "C", true), document.get("unmappedObject"));
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBQueryActionTest.java

@@ -0,0 +1,970 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.temporal.ChronoUnit;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBQueryAction 
+ *******************************************************************************/
+class MongoDBQueryActionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach() throws QException
+   {
+      primeTestDatabase();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void primeTestDatabase() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("seqNo", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("birthDate", LocalDate.parse("1980-05-31")).withValue("email", "darin.kelkhoff@gmail.com").withValue("isEmployed", true).withValue("annualSalary", 25000).withValue("daysWorked", 27).withValue("homeTown", "Chester"),
+         new QRecord().withValue("seqNo", 2).withValue("firstName", "James").withValue("lastName", "Maes").withValue("birthDate", LocalDate.parse("1980-05-15")).withValue("email", "jmaes@mmltholdings.com").withValue("isEmployed", true).withValue("annualSalary", 26000).withValue("daysWorked", 124).withValue("homeTown", "Chester"),
+         new QRecord().withValue("seqNo", 3).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("birthDate", LocalDate.parse("1976-05-28")).withValue("email", "tchamberlain@mmltholdings.com").withValue("isEmployed", false).withValue("annualSalary", null).withValue("daysWorked", 0).withValue("homeTown", "Decatur"),
+         new QRecord().withValue("seqNo", 4).withValue("firstName", "Tyler").withValue("lastName", "Samples").withValue("birthDate", null).withValue("email", "tsamples@mmltholdings.com").withValue("isEmployed", true).withValue("annualSalary", 30000).withValue("daysWorked", 99).withValue("homeTown", "Texas"),
+         new QRecord().withValue("seqNo", 5).withValue("firstName", "Garret").withValue("lastName", "Richardson").withValue("birthDate", LocalDate.parse("1981-01-01")).withValue("email", "grichardson@mmltholdings.com").withValue("isEmployed", true).withValue("annualSalary", 1000000).withValue("daysWorked", 232).withValue("homeTown", null)
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+      MongoDatabase database = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE);
+
+      MongoCollection<Document> storeCollection = database.getCollection(TestUtils.TABLE_NAME_STORE);
+      storeCollection.insertMany(List.of(
+         Document.parse("""
+            {"key":1, "name": "Q-Mart"}"""),
+         Document.parse("""
+            {"key":2, "name": "QQQ 'R' Us"}"""),
+         Document.parse("""
+            {"key":3, "name": "QDepot"}""")
+      ));
+
+      MongoCollection<Document> orderCollection = database.getCollection(TestUtils.TABLE_NAME_ORDER);
+      orderCollection.insertMany(List.of(
+         Document.parse("""
+            {"key": 1, "storeKey":1, "billToPersonId": 1, "shipToPersonId": 1}}"""),
+         Document.parse("""
+            {"key": 2, "storeKey":1, "billToPersonId": 1, "shipToPersonId": 2}}"""),
+         Document.parse("""
+            {"key": 3, "storeKey":1, "billToPersonId": 2, "shipToPersonId": 3}}"""),
+         Document.parse("""
+            {"key": 4, "storeKey":2, "billToPersonId": 4, "shipToPersonId": 5}}"""),
+         Document.parse("""
+            {"key": 5, "storeKey":2, "billToPersonId": 5, "shipToPersonId": 4}}"""),
+         Document.parse("""
+            {"key": 6, "storeKey":3, "billToPersonId": 5, "shipToPersonId": null}}"""),
+         Document.parse("""
+            {"key": 7, "storeKey":3, "billToPersonId": null, "shipToPersonId": 5}"""),
+         Document.parse("""
+            {"key": 8, "storeKey":3, "billToPersonId": null, "shipToPersonId": 5}""")
+      ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      //////////////////////////////////////////////////////////
+      // let's not use the primed-database rows for this test //
+      //////////////////////////////////////////////////////////
+      clearDatabase();
+
+      ////////////////////////////////////////
+      // directly insert some mongo records //
+      ////////////////////////////////////////
+      MongoDatabase             database   = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE);
+      MongoCollection<Document> collection = database.getCollection(TestUtils.TABLE_NAME_PERSON);
+      collection.insertMany(List.of(
+         Document.parse("""
+            {  "metaData": {"createDate": "2023-01-09T01:01:01.123Z", "modifyDate": "2023-01-09T02:02:02.123Z", "oops": "All Crunchberries"},
+               "firstName": "Darin",
+               "lastName": "Kelkhoff",
+               "unmappedField": 1701,
+               "unmappedList": [1,2,3],
+               "unmappedObject": {
+                  "A": "B",
+                  "One": 2,
+                  "subSub": {
+                     "so": true
+                  }
+               }
+            }"""),
+         Document.parse("""
+            {"metaData": {"createDate": "2023-01-09T03:03:03.123Z", "modifyDate": "2023-01-09T04:04:04.123Z"}, "firstName": "Tylers", "lastName": "Sample"}""")
+      ));
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+      assertEquals(2, queryOutput.getRecords().size());
+
+      QRecord record = queryOutput.getRecords().get(0);
+      assertEquals(Instant.parse("2023-01-09T01:01:01.123Z"), record.getValueInstant("createDate"));
+      assertEquals(Instant.parse("2023-01-09T02:02:02.123Z"), record.getValueInstant("modifyDate"));
+      assertThat(record.getValue("id")).isInstanceOf(String.class);
+      assertEquals("Darin", record.getValueString("firstName"));
+      assertEquals("Kelkhoff", record.getValueString("lastName"));
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // test that un-mapped (or un-structured) fields come through, with their shape as they exist in the mongo record //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      assertEquals(1701, record.getValueInteger("unmappedField"));
+      assertEquals(List.of(1, 2, 3), record.getValue("unmappedList"));
+      assertEquals(Map.of("A", "B", "One", 2, "subSub", Map.of("so", true)), record.getValue("unmappedObject"));
+      assertEquals(Map.of("oops", "All Crunchberries"), record.getValue("metaData"));
+
+      record = queryOutput.getRecords().get(1);
+      assertEquals(Instant.parse("2023-01-09T03:03:03.123Z"), record.getValueInstant("createDate"));
+      assertEquals(Instant.parse("2023-01-09T04:04:04.123Z"), record.getValueInstant("modifyDate"));
+      assertEquals("Tylers", record.getValueString("firstName"));
+      assertEquals("Sample", record.getValueString("lastName"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QueryInput initQueryRequest()
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      return queryInput;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testUnfilteredQuery() throws QException
+   {
+      QueryInput  queryInput  = initQueryRequest();
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testEqualsQuery() throws QException
+   {
+      String email = "darin.kelkhoff@gmail.com";
+
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.EQUALS)
+            .withValues(List.of(email)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+      assertEquals(email, queryOutput.getRecords().get(0).getValueString("email"), "Should find expected email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotEqualsQuery() throws QException
+   {
+      String email = "darin.kelkhoff@gmail.com";
+
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.NOT_EQUALS)
+            .withValues(List.of(email)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotEqualsOrIsNullQuery() throws QException
+   {
+      /////////////////////////////////////////////////////////////////////////////
+      // 5 rows, 1 has a null salary, 1 has 1,000,000.                           //
+      // first confirm that query for != returns 3 (the null does NOT come back) //
+      // then, confirm that != or is null gives the (more humanly expected) 4.   //
+      /////////////////////////////////////////////////////////////////////////////
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("annualSalary")
+            .withOperator(QCriteriaOperator.NOT_EQUALS)
+            .withValues(List.of(1_000_000))));
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows");
+
+      queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("annualSalary")
+            .withOperator(QCriteriaOperator.NOT_EQUALS_OR_IS_NULL)
+            .withValues(List.of(1_000_000))));
+      queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testInQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.IN)
+            .withValues(List.of(2, 4)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(2) || r.getValueInteger("seqNo").equals(4)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotInQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.NOT_IN)
+            .withValues(List.of(2, 3, 4)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testStartsWith() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.STARTS_WITH)
+            .withValues(List.of("darin")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testContains() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.CONTAINS)
+            .withValues(List.of("kelkhoff")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testLike() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.LIKE)
+            .withValues(List.of("%kelk%")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotLike() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.NOT_LIKE)
+            .withValues(List.of("%kelk%")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testEndsWith() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.ENDS_WITH)
+            .withValues(List.of("gmail.com")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotStartsWith() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.NOT_STARTS_WITH)
+            .withValues(List.of("darin")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotContains() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.NOT_CONTAINS)
+            .withValues(List.of("kelkhoff")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testNotEndsWith() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("email")
+            .withOperator(QCriteriaOperator.NOT_ENDS_WITH)
+            .withValues(List.of("gmail.com")))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testLessThanQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.LESS_THAN)
+            .withValues(List.of(3)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(2)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testLessThanOrEqualsQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS)
+            .withValues(List.of(2)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(2)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testGreaterThanQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.GREATER_THAN)
+            .withValues(List.of(3)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(4) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testGreaterThanOrEqualsQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.GREATER_THAN_OR_EQUALS)
+            .withValues(List.of(4)))
+      );
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(4) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testIsBlankQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("birthDate")
+            .withOperator(QCriteriaOperator.IS_BLANK)
+         ));
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testIsNotBlankQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("firstName")
+            .withOperator(QCriteriaOperator.IS_NOT_BLANK)
+         ));
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testBetweenQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.BETWEEN)
+            .withValues(List.of(2, 4))
+         ));
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(2) || r.getValueInteger("seqNo").equals(3) || r.getValueInteger("seqNo").equals(4)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    * [
+    *    And Filter
+    *    {
+    *       filters=
+    *       [
+    *          Not Filter
+    *          {
+    *             filter=And Filter
+    *             {
+    *                filters=
+    *                [
+    *                   Operator Filter
+    *                   {
+    *                      fieldName='seqNo', operator='$gte', value=2
+    *                   },
+    *                   Operator Filter
+    *                   {
+    *                      fieldName='seqNo', operator='$lte', value=4
+    *                   }
+    *                ]
+    *             }
+    *          }
+    *       ]
+    *    }
+    * ]
+    *******************************************************************************/
+   @Test
+   public void testNotBetweenQuery() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria()
+            .withFieldName("seqNo")
+            .withOperator(QCriteriaOperator.NOT_BETWEEN)
+            .withValues(List.of(2, 4))
+         ));
+      QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+      Assertions.assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("seqNo").equals(1) || r.getValueInteger("seqNo").equals(5)), "Should find expected ids");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testFilterExpressions() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("email", "-").withValue("firstName", "past").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().minus(3, ChronoUnit.DAYS)),
+         new QRecord().withValue("email", "-").withValue("firstName", "future").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().plus(3, ChronoUnit.DAYS))
+      ));
+      new InsertAction().execute(insertInput);
+
+      {
+         QueryInput queryInput = initQueryRequest();
+         queryInput.setFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest")))
+            .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now()))));
+         QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+         assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+      }
+
+      {
+         QueryInput queryInput = initQueryRequest();
+         queryInput.setFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest")))
+            .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS)))));
+         QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+         assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+      }
+
+      {
+         QueryInput queryInput = initQueryRequest();
+         queryInput.setFilter(new QQueryFilter()
+            .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest")))
+            .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS)))));
+         QueryOutput queryOutput = new MongoDBQueryAction().execute(queryInput);
+         assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row");
+         Assertions.assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testEmptyInList() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IN, List.of())));
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(0, queryOutput.getRecords().size(), "IN empty list should find nothing.");
+
+      queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.NOT_IN, List.of())));
+      queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(5, queryOutput.getRecords().size(), "NOT_IN empty list should find everything.");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOr() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+         .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin")))
+         .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim")))
+      );
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "OR should find 2 rows");
+      assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin"));
+      assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNestedFilterAndOrOr() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+         .withSubFilters(List.of(
+            new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.AND)
+               .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James")))
+               .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Maes"))),
+            new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.AND)
+               .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin")))
+               .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff")))
+         ))
+      );
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(2, queryOutput.getRecords().size(), "Complex query should find 2 rows");
+      assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("lastName").equals("Maes"));
+      assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("lastName").equals("Kelkhoff"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNestedFilterOrAndAnd() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withBooleanOperator(QQueryFilter.BooleanOperator.AND)
+         .withSubFilters(List.of(
+            new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James")))
+               .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))),
+            new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff")))
+               .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain")))
+         ))
+      );
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row");
+      assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testNestedFilterAndTopLevelFilter() throws QException
+   {
+      QueryInput queryInput = initQueryRequest();
+      queryInput.setFilter(new QQueryFilter()
+         .withCriteria(new QFilterCriteria("seqNo", QCriteriaOperator.EQUALS, 3))
+         .withBooleanOperator(QQueryFilter.BooleanOperator.AND)
+         .withSubFilters(List.of(
+            new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James")))
+               .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))),
+            new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff")))
+               .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain")))
+         ))
+      );
+      QueryOutput queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row");
+      assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueInteger("seqNo").equals(3) && r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain"));
+
+      queryInput.getFilter().setCriteria(List.of(new QFilterCriteria("seqNo", QCriteriaOperator.NOT_EQUALS, 3)));
+      queryOutput = new QueryAction().execute(queryInput);
+      assertEquals(0, queryOutput.getRecords().size(), "Next complex query should find 0 rows");
+   }
+
+
+
+   /*******************************************************************************
+    ** queries on the store table, where the primary key (id) is the security field
+    *******************************************************************************/
+   @Test
+   void testRecordSecurityPrimaryKeyFieldNoFilters() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_STORE);
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(3);
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(1)
+         .anyMatch(r -> r.getValueInteger("key").equals(1));
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(1)
+         .anyMatch(r -> r.getValueInteger("key").equals(2));
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession());
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, null));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, Collections.emptyList()));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, List.of(1, 3)));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(2)
+         .anyMatch(r -> r.getValueInteger("key").equals(1))
+         .anyMatch(r -> r.getValueInteger("key").equals(3));
+   }
+
+
+
+   /*******************************************************************************
+    ** not really expected to be any different from where we filter on the primary key,
+    ** but just good to make sure
+    *******************************************************************************/
+   @Test
+   void testRecordSecurityForeignKeyFieldNoFilters() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(8);
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(3)
+         .allMatch(r -> r.getValueInteger("storeKey").equals(1));
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(2)
+         .allMatch(r -> r.getValueInteger("storeKey").equals(2));
+
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession());
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, null));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, Collections.emptyList()));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, List.of(1, 3)));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(6)
+         .allMatch(r -> r.getValueInteger("storeKey").equals(1) || r.getValueInteger("storeKey").equals(3));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRecordSecurityWithFilters() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7))));
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(6);
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7))));
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(2)
+         .allMatch(r -> r.getValueInteger("storeKey").equals(1));
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7))));
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 5));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("key", QCriteriaOperator.BETWEEN, List.of(2, 7))));
+      QContext.setQSession(new QSession());
+      assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty();
+
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("storeKey", QCriteriaOperator.IN, List.of(1, 2))));
+      QContext.setQSession(new QSession().withSecurityKeyValues(TestUtils.TABLE_NAME_STORE, List.of(1, 3)));
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(3)
+         .allMatch(r -> r.getValueInteger("storeKey").equals(1));
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBTransactionTest.java

@@ -0,0 +1,114 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
+import com.mongodb.MongoCommandException;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBTransaction 
+ *******************************************************************************/
+class MongoDBTransactionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    ** Our testcontainer only runs a single mongo, so it doesn't support transactions.
+    ** The Backend built by TestUtils is configured to with transactionsSupported = false
+    ** make sure things all work like this.
+    *******************************************************************************/
+   @Test
+   void testWithTransactionsDisabled() throws QException
+   {
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      insertInput.setRecords(List.of(new QRecord().withValue("firstName", "Darin")));
+
+      QBackendTransaction transaction = QBackendTransaction.openFor(insertInput);
+      assertNotNull(transaction);
+      assertThat(transaction).isInstanceOf(MongoDBTransaction.class);
+      MongoDBTransaction mongoDBTransaction = (MongoDBTransaction) transaction;
+      assertNotNull(mongoDBTransaction.getMongoClient());
+      assertNotNull(mongoDBTransaction.getClientSession());
+
+      insertInput.setTransaction(transaction);
+      new InsertAction().execute(insertInput);
+      transaction.commit();
+   }
+
+
+
+   /*******************************************************************************
+    ** make sure we throw an error if we do turn on transaction support, but our
+    ** mongo backend can't handle them
+    *******************************************************************************/
+   @Test
+   void testWithTransactionsEnabled() throws QException
+   {
+      MongoDBBackendMetaData backend = (MongoDBBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME);
+
+      try
+      {
+         backend.setTransactionsSupported(true);
+
+         InsertInput insertInput = new InsertInput();
+         insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+         insertInput.setRecords(List.of(new QRecord().withValue("firstName", "Darin")));
+
+         QBackendTransaction transaction = QBackendTransaction.openFor(insertInput);
+         assertNotNull(transaction);
+         assertThat(transaction).isInstanceOf(MongoDBTransaction.class);
+         MongoDBTransaction mongoDBTransaction = (MongoDBTransaction) transaction;
+         assertNotNull(mongoDBTransaction.getMongoClient());
+         assertNotNull(mongoDBTransaction.getClientSession());
+
+         insertInput.setTransaction(transaction);
+
+         assertThatThrownBy(() -> new InsertAction().execute(insertInput))
+            .isInstanceOf(QException.class)
+            .hasRootCauseInstanceOf(MongoCommandException.class);
+
+         assertThatThrownBy(() -> transaction.commit())
+            .isInstanceOf(QException.class)
+            .hasRootCauseInstanceOf(MongoCommandException.class);
+      }
+      finally
+      {
+         backend.setTransactionsSupported(false);
+      }
+   }
+
+}

qqq-backend-module-mongodb/src/test/java/com/kingsrook/qqq/backend/module/mongodb/actions/MongoDBUpdateActionTest.java

@@ -0,0 +1,87 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.mongodb.actions;
+
+
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.module.mongodb.BaseTest;
+import com.kingsrook.qqq.backend.module.mongodb.TestUtils;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.result.InsertManyResult;
+import org.bson.BsonValue;
+import org.bson.Document;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for MongoDBUpdateAction
+ *******************************************************************************/
+class MongoDBUpdateActionTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      ////////////////////////////////////////
+      // directly insert some mongo records //
+      ////////////////////////////////////////
+      MongoDatabase             database   = getMongoClient().getDatabase(TestUtils.MONGO_DATABASE);
+      MongoCollection<Document> collection = database.getCollection(TestUtils.TABLE_NAME_PERSON);
+      InsertManyResult insertManyResult = collection.insertMany(List.of(
+         Document.parse("""
+            {"metaData": {"createDate": "2023-01-09T03:03:03.123Z", "modifyDate": "2023-01-09T04:04:04.123Z"}, "firstName": "Tylers", "lastName": "Sample"}""")
+      ));
+      BsonValue insertedId = insertManyResult.getInsertedIds().values().iterator().next();
+
+      ////////////////////////////////////
+      // update using qqq update action //
+      ////////////////////////////////////
+      UpdateInput updateInput = new UpdateInput();
+      updateInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      updateInput.setRecords(List.of(
+         new QRecord().withValue("id", insertedId.asObjectId().getValue().toString()).withValue("firstName", "Tyler").withValue("lastName", "Sample")
+      ));
+      UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
+
+      /////////////////////////////////////////////////
+      // directly query mongo for the updated record //
+      /////////////////////////////////////////////////
+      Document document = collection.find(new Document("firstName", "Tyler")).first();
+      assertNotNull(document);
+      assertEquals("Tyler", document.get("firstName"));
+      assertNotEquals(Instant.parse("2023-01-09T04:04:04.123Z"), ((Document) document.get("metaData")).get("modifyDate"));
+   }
+
+}

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java

@@ -22,20 +22,27 @@
 package com.kingsrook.qqq.backend.module.rdbms;
 
 
+import java.sql.Connection;
+import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSAggregateAction;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSCountAction;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSDeleteAction;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSInsertAction;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSQueryAction;
+import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction;
 import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSUpdateAction;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
@@ -46,6 +53,10 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDe
  *******************************************************************************/
 public class RDBMSBackendModule implements QBackendModuleInterface
 {
+   private static final QLogger LOG = QLogger.getLogger(RDBMSBackendModule.class);
+
+
+
    /*******************************************************************************
     ** Method where a backend module must be able to provide its type (name).
     *******************************************************************************/
@@ -142,4 +153,24 @@ public class RDBMSBackendModule implements QBackendModuleInterface
       return (new RDBMSAggregateAction());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QBackendTransaction openTransaction(AbstractTableActionInput input) throws QException
+   {
+      try
+      {
+         LOG.debug("Opening transaction");
+         Connection connection = AbstractRDBMSAction.getConnection(input);
+         return (new RDBMSTransaction(connection));
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error opening transaction: " + e.getMessage(), e);
+      }
+   }
+
 }