Merge branch 'release/0.18.0'

CE-567 Add concept of security lock Scope - e.g., READ-WRITE (blockin…

.circleci/adjust-pom-version.sh

@@ -5,8 +5,8 @@ if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then
    exit 1;
 fi
 
-if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ]; then
-   echo "On a primary branch [$CIRCLE_BRANCH] - will not edit the pom version.";
+if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
+   echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
    exit 0;
 fi
 

pom.xml

@@ -44,7 +44,7 @@
    </modules>
 
    <properties>
-      <revision>0.16.0</revision>
+      <revision>0.18.0</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -166,7 +167,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                ///////////////////////////////////////////////////
                // validate security keys on the table are given //
                ///////////////////////////////////////////////////
-               for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+               for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
                {
                   if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
                   {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -378,7 +379,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
    {
       Map<String, Serializable> securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
          securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizer.java

@@ -29,6 +29,7 @@ import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -42,18 +43,16 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 /*******************************************************************************
  ** Standard/re-usable post-insert customizer, for the use case where, when we
  ** do an insert into table "parent", we want a record automatically inserted into
- ** table "child", and there's a foreign key in "parent", pointed at "child"
- ** e.g., named: "parent.childId".
+ ** table "child".  Optionally (based on RelationshipType), there can be a foreign
+ ** key in "parent", pointed at "child".  e.g., named: "parent.childId".
  **
- ** A similar use-case would have the foreign key in the child table - in which case,
- ** we could add a "Type" enum, plus abstract method to get our "Type", then logic
- ** to switch behavior based on type.  See existing type enum, but w/ only 1 case :)
  *******************************************************************************/
 public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInsertCustomizer
 {
    public enum RelationshipType
    {
-      PARENT_POINTS_AT_CHILD
+      PARENT_POINTS_AT_CHILD,
+      CHILD_POINTS_AT_PARENT
    }
 
 
@@ -68,10 +67,17 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
     *******************************************************************************/
    public abstract String getChildTableName();
 
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public abstract String getForeignKeyFieldName();
+   public String getForeignKeyFieldName()
+   {
+      return (null);
+   }
+
+
 
    /*******************************************************************************
     **
@@ -88,7 +94,7 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
    {
       try
       {
-         List<QRecord>  rs               = new ArrayList<>();
+         List<QRecord>  rs               = records;
          List<QRecord>  childrenToInsert = new ArrayList<>();
          QTableMetaData table            = getInsertInput().getTable();
          QTableMetaData childTable       = getInsertInput().getInstance().getTable(getChildTableName());
@@ -97,12 +103,37 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          // iterate over the inserted records, building a list child records to insert //
          // for ones missing a value in the foreign key field.                         //
          ////////////////////////////////////////////////////////////////////////////////
-         for(QRecord record : records)
+         switch(getRelationshipType())
          {
-            if(record.getValue(getForeignKeyFieldName()) == null)
+            case PARENT_POINTS_AT_CHILD ->
             {
-               childrenToInsert.add(buildChildForRecord(record));
+               String foreignKeyFieldName = getForeignKeyFieldName();
+               try
+               {
+                  table.getField(foreignKeyFieldName);
+               }
+               catch(Exception e)
+               {
+                  throw new QRuntimeException("For RelationshipType.PARENT_POINTS_AT_CHILD, a valid foreignKeyFieldName in the parent table must be given.  "
+                     + "[" + foreignKeyFieldName + "] is not a valid field name in table [" + table.getName() + "]");
+               }
+
+               for(QRecord record : records)
+               {
+                  if(record.getValue(foreignKeyFieldName) == null)
+                  {
+                     childrenToInsert.add(buildChildForRecord(record));
+                  }
+               }
             }
+            case CHILD_POINTS_AT_PARENT ->
+            {
+               for(QRecord record : records)
+               {
+                  childrenToInsert.add(buildChildForRecord(record));
+               }
+            }
+            default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
          }
 
          ///////////////////////////////////////////////////////////////////////////////////
@@ -129,51 +160,70 @@ public abstract class ChildInserterPostInsertCustomizer extends AbstractPostInse
          /////////////////////////////////////////////////////////////////////////////////
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////
+         // for the PARENT_POINTS_AT_CHILD relationship type:
          // iterate over the original list of records again - for any that need a child (e.g., are missing   //
          // foreign key), set their foreign key to a newly inserted child's key, and add them to be updated. //
          //////////////////////////////////////////////////////////////////////////////////////////////////////
-         List<QRecord> recordsToUpdate = new ArrayList<>();
-         for(QRecord record : records)
+         switch(getRelationshipType())
          {
-            Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
-            if(record.getValue(getForeignKeyFieldName()) == null)
+            case PARENT_POINTS_AT_CHILD ->
             {
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               // get the corresponding child record, if it has any errors, set that as a warning in the parent //
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               QRecord childRecord = insertedRecordIterator.next();
-               if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
+               rs = new ArrayList<>();
+               List<QRecord> recordsToUpdate = new ArrayList<>();
+               for(QRecord record : records)
                {
-                  for(QStatusMessage error : childRecord.getErrors())
+                  Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+                  if(record.getValue(getForeignKeyFieldName()) == null)
                   {
-                     record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////
+                     // get the corresponding child record, if it has any errors, set that as a warning in the parent //
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////
+                     QRecord childRecord = insertedRecordIterator.next();
+                     if(CollectionUtils.nullSafeHasContents(childRecord.getErrors()))
+                     {
+                        for(QStatusMessage error : childRecord.getErrors())
+                        {
+                           record.addWarning(new QWarningMessage("Error creating child " + childTable.getLabel() + " (" + error.toString() + ")"));
+                        }
+                        rs.add(record);
+                        continue;
+                     }
+
+                     Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
+                     recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
+                     record.setValue(getForeignKeyFieldName(), foreignKey);
+                     rs.add(record);
+                  }
+                  else
+                  {
+                     rs.add(record);
                   }
-                  rs.add(record);
-                  continue;
                }
 
-               Serializable foreignKey = childRecord.getValue(childTable.getPrimaryKeyField());
-               recordsToUpdate.add(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withValue(getForeignKeyFieldName(), foreignKey));
-               record.setValue(getForeignKeyFieldName(), foreignKey);
-               rs.add(record);
+               ////////////////////////////////////////////////////////////////////////////
+               // update the originally inserted records to reference their new children //
+               ////////////////////////////////////////////////////////////////////////////
+               UpdateInput updateInput = new UpdateInput();
+               updateInput.setTableName(getInsertInput().getTableName());
+               updateInput.setRecords(recordsToUpdate);
+               updateInput.setTransaction(this.insertInput.getTransaction());
+               new UpdateAction().execute(updateInput);
             }
-            else
+            case CHILD_POINTS_AT_PARENT ->
             {
-               rs.add(record);
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // todo - some version of looking at the inserted children to confirm that they were inserted, and updating the parents with warnings if they weren't //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             }
+            default -> throw new IllegalStateException("Unexpected value: " + getRelationshipType());
          }
 
-         ////////////////////////////////////////////////////////////////////////////
-         // update the originally inserted records to reference their new children //
-         ////////////////////////////////////////////////////////////////////////////
-         UpdateInput updateInput = new UpdateInput();
-         updateInput.setTableName(getInsertInput().getTableName());
-         updateInput.setRecords(recordsToUpdate);
-         updateInput.setTransaction(this.insertInput.getTransaction());
-         new UpdateAction().execute(updateInput);
-
          return (rs);
       }
+      catch(RuntimeException re)
+      {
+         throw (re);
+      }
       catch(Exception e)
       {
          throw new RuntimeException("Error inserting new child records for new parent records", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java

@@ -67,4 +67,15 @@ public interface BaseQueryInterface
       }
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default void cancelAction()
+   {
+      //////////////////////////////////////////////
+      // initially at least, a noop in base class //
+      //////////////////////////////////////////////
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -231,7 +231,19 @@ public class ExecuteCodeAction
     *******************************************************************************/
    public static void addApiUtilityToContext(Map<String, Serializable> context, ScriptRevision scriptRevision)
    {
-      if(!StringUtils.hasContent(scriptRevision.getApiName()) || !StringUtils.hasContent(scriptRevision.getApiVersion()))
+      addApiUtilityToContext(context, scriptRevision.getApiName(), scriptRevision.getApiVersion());
+   }
+
+
+
+   /*******************************************************************************
+    ** Try to (dynamically) load the ApiScriptUtils object from the api middleware
+    ** module -- in case the runtime doesn't have that module deployed (e.g, not in
+    ** the project pom).
+    *******************************************************************************/
+   public static void addApiUtilityToContext(Map<String, Serializable> context, String apiName, String apiVersion)
+   {
+      if(!StringUtils.hasContent(apiName) || !StringUtils.hasContent(apiVersion))
       {
          return;
       }
@@ -239,7 +251,7 @@ public class ExecuteCodeAction
       try
       {
          Class<?> apiScriptUtilsClass  = Class.forName("com.kingsrook.qqq.api.utils.ApiScriptUtils");
-         Object   apiScriptUtilsObject = apiScriptUtilsClass.getConstructor(String.class, String.class).newInstance(scriptRevision.getApiName(), scriptRevision.getApiVersion());
+         Object   apiScriptUtilsObject = apiScriptUtilsClass.getConstructor(String.class, String.class).newInstance(apiName, apiVersion);
          context.put("api", (Serializable) apiScriptUtilsObject);
       }
       catch(ClassNotFoundException e)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/Log4jCodeExecutionLogger.java

@@ -41,6 +41,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    private QCodeReference qCodeReference;
    private String         uuid = UUID.randomUUID().toString();
 
+   private boolean includeUUID = true;
 
 
    /*******************************************************************************
@@ -52,7 +53,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
       this.qCodeReference = executeCodeInput.getCodeReference();
 
       String inputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(executeCodeInput.getInput()), 250, "...");
-      LOG.info("Starting script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with input: " + inputString);
+      LOG.info("Starting script execution: " + qCodeReference.getName() + (includeUUID ? ", uuid: " + uuid : "") + ", with input: " + inputString);
    }
 
 
@@ -63,7 +64,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    @Override
    public void acceptLogLine(String logLine)
    {
-      LOG.info("Script log: " + uuid + ": " + logLine);
+      LOG.info("Script log: " + (includeUUID ? uuid + ": " : "") + logLine);
    }
 
 
@@ -74,7 +75,7 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    @Override
    public void acceptException(Exception exception)
    {
-      LOG.info("Script Exception: " + uuid, exception);
+      LOG.info("Script Exception: " + (includeUUID ? uuid : ""), exception);
    }
 
 
@@ -86,7 +87,38 @@ public class Log4jCodeExecutionLogger implements QCodeExecutionLoggerInterface
    public void acceptExecutionEnd(Serializable output)
    {
       String outputString = StringUtils.safeTruncate(ValueUtils.getValueAsString(output), 250, "...");
-      LOG.info("Finished script execution: " + qCodeReference.getName() + ", uuid: " + uuid + ", with output: " + outputString);
+      LOG.info("Finished script execution: " + qCodeReference.getName() + (includeUUID ? ", uuid: " + uuid : "") + ", with output: " + outputString);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for includeUUID
+    *******************************************************************************/
+   public boolean getIncludeUUID()
+   {
+      return (this.includeUUID);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for includeUUID
+    *******************************************************************************/
+   public void setIncludeUUID(boolean includeUUID)
+   {
+      this.includeUUID = includeUUID;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for includeUUID
+    *******************************************************************************/
+   public Log4jCodeExecutionLogger withIncludeUUID(boolean includeUUID)
+   {
+      this.includeUUID = includeUUID;
+      return (this);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/logging/SystemOutExecutionLogger.java

@@ -0,0 +1,88 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.scripts.logging;
+
+
+import java.io.Serializable;
+import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** Implementation of a code execution logger that logs to System.out and
+ ** System.err (for exceptions)
+ *******************************************************************************/
+public class SystemOutExecutionLogger implements QCodeExecutionLoggerInterface
+{
+   private QCodeReference qCodeReference;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionStart(ExecuteCodeInput executeCodeInput)
+   {
+      this.qCodeReference = executeCodeInput.getCodeReference();
+
+      String inputString = ValueUtils.getValueAsString(executeCodeInput.getInput());
+      System.out.println("Starting script execution: " + qCodeReference.getName() + ", with input: " + inputString);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptLogLine(String logLine)
+   {
+      System.out.println("Script log: " + logLine);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptException(Exception exception)
+   {
+      System.out.println("Script Exception: " + exception.getMessage());
+      exception.printStackTrace();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void acceptExecutionEnd(Serializable output)
+   {
+      String outputString = ValueUtils.getValueAsString(output);
+      System.out.println("Finished script execution: " + qCodeReference.getName() + ", with output: " + outputString);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -41,6 +42,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class AggregateAction
 {
+   private static final QLogger LOG = QLogger.getLogger(AggregateAction.class);
+
+   private AggregateInterface aggregateInterface;
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -56,7 +63,7 @@ public class AggregateAction
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(aggregateInput.getBackend());
 
-      AggregateInterface aggregateInterface = qModule.getAggregateInterface();
+      aggregateInterface = qModule.getAggregateInterface();
       aggregateInterface.setQueryStat(queryStat);
       AggregateOutput aggregateOutput = aggregateInterface.execute(aggregateInput);
 
@@ -64,4 +71,20 @@ public class AggregateAction
 
       return aggregateOutput;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(aggregateInterface == null)
+      {
+         LOG.warn("aggregateInterface object was null when requested to cancel");
+         return;
+      }
+
+      aggregateInterface.cancelAction();
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -41,6 +42,12 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
  *******************************************************************************/
 public class CountAction
 {
+   private static final QLogger LOG = QLogger.getLogger(CountAction.class);
+
+   private CountInterface countInterface;
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -56,7 +63,7 @@ public class CountAction
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend());
 
-      CountInterface countInterface = qModule.getCountInterface();
+      countInterface = qModule.getCountInterface();
       countInterface.setQueryStat(queryStat);
       CountOutput countOutput = countInterface.execute(countInput);
 
@@ -64,4 +71,20 @@ public class CountAction
 
       return countOutput;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(countInterface == null)
+      {
+         LOG.warn("countInterface object was null when requested to cancel");
+         return;
+      }
+
+      countInterface.cancelAction();
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCusto
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.LogPair;
@@ -321,6 +322,8 @@ public class DeleteAction
       QTableMetaData table               = deleteInput.getTable();
       List<QRecord>  primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
 
+      ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE);
+
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-delete customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -76,6 +76,7 @@ public class QueryAction
    private Optional<AbstractPostQueryCustomizer> postQueryRecordCustomizer;
 
    private QueryInput               queryInput;
+   private QueryInterface           queryInterface;
    private QPossibleValueTranslator qPossibleValueTranslator;
 
 
@@ -121,7 +122,7 @@ public class QueryAction
       QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
       QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
 
-      QueryInterface queryInterface = qModule.getQueryInterface();
+      queryInterface = qModule.getQueryInterface();
       queryInterface.setQueryStat(queryStat);
       QueryOutput queryOutput = queryInterface.execute(queryInput);
 
@@ -339,4 +340,20 @@ public class QueryAction
          }
       }
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(queryInterface == null)
+      {
+         LOG.warn("queryInterface object was null when requested to cancel");
+         return;
+      }
+
+      queryInterface.cancelAction();
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -61,6 +61,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
@@ -209,14 +211,16 @@ public class UpdateAction
       {
          validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList.get());
       }
+      else
+      {
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+      }
 
       if(updateInput.getInputSource().shouldValidateRequiredFields())
       {
          validateRequiredFields(updateInput);
       }
 
-      ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
-
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-update customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
@@ -287,6 +291,8 @@ public class UpdateAction
       QTableMetaData table           = updateInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
+      List<RecordSecurityLock> onlyWriteLocks = RecordSecurityLockFilters.filterForOnlyWriteLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()));
+
       for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 1000))
       {
          List<Serializable> primaryKeysToLookup = new ArrayList<>();
@@ -320,6 +326,8 @@ public class UpdateAction
             }
          }
 
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+
          for(QRecord record : page)
          {
             Serializable value = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), record.getValue(table.getPrimaryKeyField()));
@@ -332,6 +340,19 @@ public class UpdateAction
             {
                record.addError(new NotFoundStatusMessage("No record was found to update for " + primaryKeyField.getLabel() + " = " + value));
             }
+            else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////////
+               // if the table has any write-only locks, validate their values here, on the old-records //
+               ///////////////////////////////////////////////////////////////////////////////////////////
+               for(RecordSecurityLock lock : onlyWriteLocks)
+               {
+                  QRecord      oldRecord = lookedUpRecords.get(value);
+                  QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
+                  Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
+                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+               }
+            }
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ActionTimeoutHelper.java

@@ -0,0 +1,109 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+
+/*******************************************************************************
+ ** For actions that may want to set a timeout, and cancel themselves if they run
+ ** too long - this class helps.
+ **
+ ** Construct with the timeout (delay & timeUnit), and a runnable that takes care
+ ** of doing the cancel (e.g., cancelling a JDBC statement).
+ **
+ ** Call start() to make a future get scheduled (note, if delay was null or <= 0,
+ ** then it doesn't get scheduled at all).
+ **
+ ** Call cancel() if the action got far enough/completed, to cancel the future.
+ **
+ ** You can check didTimeout (getDidTimeout()) to know if the timeout did occur.
+ *******************************************************************************/
+public class ActionTimeoutHelper
+{
+   private final Integer            delay;
+   private final TimeUnit           timeUnit;
+   private final Runnable           runnable;
+   private       ScheduledFuture<?> future;
+
+   private boolean didTimeout = false;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public ActionTimeoutHelper(Integer delay, TimeUnit timeUnit, Runnable runnable)
+   {
+      this.delay = delay;
+      this.timeUnit = timeUnit;
+      this.runnable = runnable;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void start()
+   {
+      if(delay == null || delay <= 0)
+      {
+         return;
+      }
+
+      future = Executors.newSingleThreadScheduledExecutor().schedule(() ->
+      {
+         didTimeout = true;
+         runnable.run();
+      }, delay, timeUnit);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void cancel()
+   {
+      if(future != null)
+      {
+         future.cancel(true);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for didTimeout
+    **
+    *******************************************************************************/
+   public boolean getDidTimeout()
+   {
+      return didTimeout;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -68,6 +69,7 @@ public class ValidateRecordSecurityLockHelper
    {
       INSERT,
       UPDATE,
+      DELETE,
       SELECT
    }
 
@@ -78,7 +80,7 @@ public class ValidateRecordSecurityLockHelper
     *******************************************************************************/
    public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
    {
-      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table);
+      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table, action);
       if(CollectionUtils.nullSafeIsEmpty(locksToCheck))
       {
          return;
@@ -98,11 +100,12 @@ public class ValidateRecordSecurityLockHelper
 
             for(QRecord record : records)
             {
-               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()))
+               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
                {
-                  /////////////////////////////////////////////////////////////////////////
-                  // if not updating the security field, then no error can come from it! //
-                  /////////////////////////////////////////////////////////////////////////
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+                  // So if we're not updating the security field, then no error can come from it!                        //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
                   continue;
                }
 
@@ -244,11 +247,18 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table)
+   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table, Action action)
    {
-      List<RecordSecurityLock> recordSecurityLocks = table.getRecordSecurityLocks();
+      List<RecordSecurityLock> recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
       List<RecordSecurityLock> locksToCheck        = new ArrayList<>();
 
+      recordSecurityLocks = switch(action)
+      {
+         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks);
+         case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks);
+         default -> throw (new IllegalArgumentException("Unsupported action: " + action));
+      };
+
       ////////////////////////////////////////
       // if there are no locks, just return //
       ////////////////////////////////////////
@@ -281,7 +291,7 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
+   public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
    {
       if(recordSecurityValue == null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -272,7 +272,7 @@ public class QInstanceEnricher
 
          for(QSupplementalTableMetaData supplementalTableMetaData : CollectionUtils.nonNullMap(table.getSupplementalMetaData()).values())
          {
-            supplementalTableMetaData.enrich(table);
+            supplementalTableMetaData.enrich(qInstance, table);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -438,10 +438,13 @@ public class QInstanceValidator
                for(QFieldSection section : table.getSections())
                {
                   validateTableSection(qInstance, table, section, fieldNamesInSections);
-                  if(section.getTier().equals(Tier.T1))
+                  if(assertCondition(section.getTier() != null, "Table " + tableName + " " + section.getName() + " is missing its tier"))
                   {
-                     assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
-                     tier1Section = section;
+                     if(section.getTier().equals(Tier.T1))
+                     {
+                        assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
+                        tier1Section = section;
+                     }
                   }
 
                   assertCondition(!usedSectionNames.contains(section.getName()), "Table " + tableName + " has more than 1 section named " + section.getName());
@@ -586,6 +589,8 @@ public class QInstanceValidator
 
          prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
 
+         assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
+
          boolean hasAnyBadJoins = false;
          for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
          {
@@ -1099,13 +1104,39 @@ public class QInstanceValidator
       boolean hasFields = CollectionUtils.nullSafeHasContents(section.getFieldNames());
       boolean hasWidget = StringUtils.hasContent(section.getWidgetName());
 
-      if(assertCondition(hasFields || hasWidget, "Table " + table.getName() + " section " + section.getName() + " does not have any fields or a widget."))
+      String sectionPrefix = "Table " + table.getName() + " section " + section.getName() + " ";
+      if(assertCondition(hasFields || hasWidget, sectionPrefix + "does not have any fields or a widget."))
       {
          if(table.getFields() != null && hasFields)
          {
             for(String fieldName : section.getFieldNames())
             {
-               assertCondition(table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table.");
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // note - this was originally written as an assertion:                                                                                                                                       //
+               // if(assertCondition(qInstance.getTable(otherTableName) != null, sectionPrefix + "join-field " + fieldName + ", which is referencing an unrecognized table name [" + otherTableName + "]")) //
+               // but... then a field name with dots gives us a bad time here, so...                                                                                                                        //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(fieldName.contains(".") && qInstance.getTable(fieldName.split("\\.")[0]) != null)
+               {
+                  String[] parts            = fieldName.split("\\.");
+                  String   otherTableName   = parts[0];
+                  String   foreignFieldName = parts[1];
+
+                  if(assertCondition(qInstance.getTable(otherTableName) != null, sectionPrefix + "join-field " + fieldName + ", which is referencing an unrecognized table name [" + otherTableName + "]"))
+                  {
+                     List<ExposedJoin> matchedExposedJoins = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> otherTableName.equals(ej.getJoinTable())).toList();
+                     if(assertCondition(CollectionUtils.nullSafeHasContents(matchedExposedJoins), sectionPrefix + "join-field " + fieldName + ", referencing table [" + otherTableName + "] which is not an exposed join on this table."))
+                     {
+                        assertCondition(!matchedExposedJoins.get(0).getIsMany(qInstance), sectionPrefix + "join-field " + fieldName + " references an is-many join, which is not supported.");
+                     }
+                     assertCondition(qInstance.getTable(otherTableName).getFields().containsKey(foreignFieldName), sectionPrefix + "join-field " + fieldName + " specifies a fieldName [" + foreignFieldName + "] which does not exist in that table [" + otherTableName + "].");
+                  }
+               }
+               else
+               {
+                  assertCondition(table.getFields().containsKey(fieldName), sectionPrefix + "specifies fieldName " + fieldName + ", which is not a field on this table.");
+               }
+
                assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections.");
 
                fieldNamesInSections.add(fieldName);
@@ -1113,7 +1144,7 @@ public class QInstanceValidator
          }
          else if(hasWidget)
          {
-            assertCondition(qInstance.getWidget(section.getWidgetName()) != null, "Table " + table.getName() + " section " + section.getName() + " specifies widget " + section.getWidgetName() + ", which is not a widget in this instance.");
+            assertCondition(qInstance.getWidget(section.getWidgetName()) != null, sectionPrefix + "specifies widget " + section.getWidgetName() + ", which is not a widget in this instance.");
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
@@ -246,7 +247,7 @@ public class AuditSingleInput
       setAuditTableName(table.getName());
 
       this.securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
          this.securityKeyValues.put(recordSecurityLock.getFieldName(), record.getValueInteger(recordSecurityLock.getFieldName()));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java

@@ -23,7 +23,9 @@ package com.kingsrook.qqq.backend.core.model.actions.tables;
 
 
 import java.util.Collection;
+import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 
 
 /*******************************************************************************
@@ -49,6 +51,7 @@ public interface QueryOrGetInputInterface
       this.setShouldMaskPasswords(source.getShouldMaskPasswords());
       this.setIncludeAssociations(source.getIncludeAssociations());
       this.setAssociationNamesToInclude(source.getAssociationNamesToInclude());
+      this.setQueryJoins(source.getQueryJoins());
    }
 
    /*******************************************************************************
@@ -146,4 +149,17 @@ public interface QueryOrGetInputInterface
     *******************************************************************************/
    void setAssociationNamesToInclude(Collection<String> associationNamesToInclude);
 
+
+   /*******************************************************************************
+    ** Getter for queryJoins
+    *******************************************************************************/
+   List<QueryJoin> getQueryJoins();
+
+
+   /*******************************************************************************
+    ** Setter for queryJoins
+    **
+    *******************************************************************************/
+   void setQueryJoins(List<QueryJoin> queryJoins);
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/aggregate/AggregateInput.java

@@ -40,6 +40,8 @@ public class AggregateInput extends AbstractTableActionInput
    private List<GroupBy>   groupBys = new ArrayList<>();
    private Integer         limit;
 
+   private Integer timeoutSeconds;
+
    private List<QueryJoin> queryJoins = null;
 
 
@@ -269,4 +271,35 @@ public class AggregateInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for timeoutSeconds
+    *******************************************************************************/
+   public Integer getTimeoutSeconds()
+   {
+      return (this.timeoutSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timeoutSeconds
+    *******************************************************************************/
+   public void setTimeoutSeconds(Integer timeoutSeconds)
+   {
+      this.timeoutSeconds = timeoutSeconds;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timeoutSeconds
+    *******************************************************************************/
+   public AggregateInput withTimeoutSeconds(Integer timeoutSeconds)
+   {
+      this.timeoutSeconds = timeoutSeconds;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/count/CountInput.java

@@ -37,6 +37,8 @@ public class CountInput extends AbstractTableActionInput
 {
    private QQueryFilter filter;
 
+   private Integer timeoutSeconds;
+
    private List<QueryJoin> queryJoins           = null;
    private Boolean         includeDistinctCount = false;
 
@@ -174,4 +176,35 @@ public class CountInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for timeoutSeconds
+    *******************************************************************************/
+   public Integer getTimeoutSeconds()
+   {
+      return (this.timeoutSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timeoutSeconds
+    *******************************************************************************/
+   public void setTimeoutSeconds(Integer timeoutSeconds)
+   {
+      this.timeoutSeconds = timeoutSeconds;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timeoutSeconds
+    *******************************************************************************/
+   public CountInput withTimeoutSeconds(Integer timeoutSeconds)
+   {
+      this.timeoutSeconds = timeoutSeconds;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/get/GetInput.java

@@ -23,11 +23,14 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.get;
 
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 
 
 /*******************************************************************************
@@ -47,6 +50,7 @@ public class GetInput extends AbstractTableActionInput implements QueryOrGetInpu
    private boolean shouldOmitHiddenFields        = true;
    private boolean shouldMaskPasswords           = true;
 
+   private List<QueryJoin> queryJoins = null;
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
@@ -411,4 +415,51 @@ public class GetInput extends AbstractTableActionInput implements QueryOrGetInpu
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for queryJoins
+    *******************************************************************************/
+   public List<QueryJoin> getQueryJoins()
+   {
+      return (this.queryJoins);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryJoins
+    *******************************************************************************/
+   public void setQueryJoins(List<QueryJoin> queryJoins)
+   {
+      this.queryJoins = queryJoins;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryJoins
+    *******************************************************************************/
+   public GetInput withQueryJoins(List<QueryJoin> queryJoins)
+   {
+      this.queryJoins = queryJoins;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryJoins
+    **
+    *******************************************************************************/
+   public GetInput withQueryJoin(QueryJoin queryJoin)
+   {
+      if(this.queryJoins == null)
+      {
+         this.queryJoins = new ArrayList<>();
+      }
+      this.queryJoins.add(queryJoin);
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -30,17 +30,21 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
+import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -60,6 +64,7 @@ public class JoinsContext
    // note - will have entries for all tables, not just aliases. //
    ////////////////////////////////////////////////////////////////
    private final Map<String, String> aliasToTableNameMap = new HashMap<>();
+   private       Level               logLevel            = Level.OFF;
 
 
 
@@ -69,62 +74,23 @@ public class JoinsContext
     *******************************************************************************/
    public JoinsContext(QInstance instance, String tableName, List<QueryJoin> queryJoins, QQueryFilter filter) throws QException
    {
+      log("--- START ----------------------------------------------------------------------", logPair("mainTable", tableName));
       this.instance = instance;
       this.mainTableName = tableName;
       this.queryJoins = new MutableList<>(queryJoins);
 
       for(QueryJoin queryJoin : this.queryJoins)
       {
+         log("Processing input query join", logPair("joinTable", queryJoin.getJoinTable()), logPair("alias", queryJoin.getAlias()), logPair("baseTableOrAlias", queryJoin.getBaseTableOrAlias()), logPair("joinMetaDataName", () -> queryJoin.getJoinMetaData().getName()));
          processQueryJoin(queryJoin);
       }
 
       ///////////////////////////////////////////////////////////////
       // ensure any joins that contribute a recordLock are present //
       ///////////////////////////////////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks())))
       {
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         // ok - so - the join name chain is going to be like this:                                       //
-         // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
-         // - securityFieldName = order.clientId                                                          //
-         // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
-         // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
-         // and step (via tmpTable variable) back to the securityField                                    //
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
-         Collections.reverse(joinNameChain);
-
-         QTableMetaData tmpTable = instance.getTable(mainTableName);
-
-         for(String joinName : joinNameChain)
-         {
-            if(this.queryJoins.stream().anyMatch(queryJoin ->
-            {
-               QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> findJoinMetaData(instance, tableName, queryJoin.getJoinTable()));
-               return (joinMetaData != null && Objects.equals(joinMetaData.getName(), joinName));
-            }))
-            {
-               continue;
-            }
-
-            QJoinMetaData join = instance.getJoin(joinName);
-            if(join.getLeftTable().equals(tmpTable.getName()))
-            {
-               QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin);
-               tmpTable = instance.getTable(join.getRightTable());
-            }
-            else if(join.getRightTable().equals(tmpTable.getName()))
-            {
-               QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin); //
-               tmpTable = instance.getTable(join.getLeftTable());
-            }
-            else
-            {
-               throw (new QException("Error adding security lock joins to query - table name [" + tmpTable.getName() + "] not found in join [" + joinName + "]"));
-            }
-         }
+         ensureRecordSecurityLockIsRepresented(instance, tableName, recordSecurityLock);
       }
 
       ensureFilterIsRepresented(filter);
@@ -141,6 +107,86 @@ public class JoinsContext
          }
       }
       */
+
+      log("Constructed JoinsContext", logPair("mainTableName", this.mainTableName), logPair("queryJoins", this.queryJoins.stream().map(qj -> qj.getJoinTable()).collect(Collectors.joining(","))));
+      log("--- END ------------------------------------------------------------------------");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void ensureRecordSecurityLockIsRepresented(QInstance instance, String tableName, RecordSecurityLock recordSecurityLock) throws QException
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // ok - so - the join name chain is going to be like this:                                       //
+      // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
+      // - securityFieldName = order.clientId                                                          //
+      // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
+      // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
+      // and step (via tmpTable variable) back to the securityField                                    //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
+      Collections.reverse(joinNameChain);
+      log("Evaluating recordSecurityLock", logPair("recordSecurityLock", recordSecurityLock.getFieldName()), logPair("joinNameChain", joinNameChain));
+
+      QTableMetaData tmpTable = instance.getTable(mainTableName);
+
+      for(String joinName : joinNameChain)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // check the joins currently in the query - if any are for this table, then we don't need to add one //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QueryJoin> matchingJoins = this.queryJoins.stream().filter(queryJoin ->
+         {
+            QJoinMetaData joinMetaData = null;
+            if(queryJoin.getJoinMetaData() != null)
+            {
+               joinMetaData = queryJoin.getJoinMetaData();
+            }
+            else
+            {
+               joinMetaData = findJoinMetaData(instance, tableName, queryJoin.getJoinTable());
+            }
+            return (joinMetaData != null && Objects.equals(joinMetaData.getName(), joinName));
+         }).toList();
+
+         if(CollectionUtils.nullSafeHasContents(matchingJoins))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // note - if a user added a join as an outer type, we need to change it to be inner, for the security purpose. //
+            // todo - is this always right?  what about nulls-allowed?                                                     //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            log("- skipping join already in the query", logPair("joinName", joinName));
+
+            if(matchingJoins.get(0).getType().equals(QueryJoin.Type.LEFT) || matchingJoins.get(0).getType().equals(QueryJoin.Type.RIGHT))
+            {
+               log("- - although... it was here as an outer - so switching it to INNER", logPair("joinName", joinName));
+               matchingJoins.get(0).setType(QueryJoin.Type.INNER);
+            }
+
+            continue;
+         }
+
+         QJoinMetaData join = instance.getJoin(joinName);
+         if(join.getLeftTable().equals(tmpTable.getName()))
+         {
+            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
+            this.addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)");
+            tmpTable = instance.getTable(join.getRightTable());
+         }
+         else if(join.getRightTable().equals(tmpTable.getName()))
+         {
+            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
+            this.addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)");
+            tmpTable = instance.getTable(join.getLeftTable());
+         }
+         else
+         {
+            throw (new QException("Error adding security lock joins to query - table name [" + tmpTable.getName() + "] not found in join [" + joinName + "]"));
+         }
+      }
    }
 
 
@@ -151,8 +197,15 @@ public class JoinsContext
     ** use this method to add to the list, instead of ever adding directly, as it's
     ** important do to that process step (and we've had bugs when it wasn't done).
     *******************************************************************************/
-   private void addQueryJoin(QueryJoin queryJoin) throws QException
+   private void addQueryJoin(QueryJoin queryJoin, String reason) throws QException
    {
+      log("Adding query join to context",
+         logPair("reason", reason),
+         logPair("joinTable", queryJoin.getJoinTable()),
+         logPair("joinMetaData.name", () -> queryJoin.getJoinMetaData().getName()),
+         logPair("joinMetaData.leftTable", () -> queryJoin.getJoinMetaData().getLeftTable()),
+         logPair("joinMetaData.rightTable", () -> queryJoin.getJoinMetaData().getRightTable())
+      );
       this.queryJoins.add(queryJoin);
       processQueryJoin(queryJoin);
    }
@@ -177,10 +230,46 @@ public class JoinsContext
          addedJoin = false;
          for(QueryJoin queryJoin : queryJoins)
          {
-            /////////////////////////////////////////////////////////////////////
-            // if the join has joinMetaData, then we don't need to process it. //
-            /////////////////////////////////////////////////////////////////////
-            if(queryJoin.getJoinMetaData() == null)
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            // if the join has joinMetaData, then we don't need to process it... unless it needs flipped //
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            QJoinMetaData joinMetaData = queryJoin.getJoinMetaData();
+            if(joinMetaData != null)
+            {
+               boolean isJoinLeftTableInQuery = false;
+               String  joinMetaDataLeftTable  = joinMetaData.getLeftTable();
+               if(joinMetaDataLeftTable.equals(mainTableName))
+               {
+                  isJoinLeftTableInQuery = true;
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // check the other joins in this query - if any of them have this join's left-table as their baseTable, then set the flag to true //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               for(QueryJoin otherJoin : queryJoins)
+               {
+                  if(otherJoin == queryJoin)
+                  {
+                     continue;
+                  }
+
+                  if(Objects.equals(otherJoin.getBaseTableOrAlias(), joinMetaDataLeftTable))
+                  {
+                     isJoinLeftTableInQuery = true;
+                     break;
+                  }
+               }
+
+               /////////////////////////////////////////////////////////////////////////////////
+               // if the join's left-table isn't in the query, then we need to flip the join. //
+               /////////////////////////////////////////////////////////////////////////////////
+               if(!isJoinLeftTableInQuery)
+               {
+                  log("Flipping queryJoin because its leftTable wasn't found in the query", logPair("joinMetaDataName", joinMetaData.getName()), logPair("leftTable", joinMetaDataLeftTable));
+                  queryJoin.setJoinMetaData(joinMetaData.flip());
+               }
+            }
+            else
             {
                //////////////////////////////////////////////////////////////////////
                // try to find a direct join between the main table and this table. //
@@ -190,6 +279,7 @@ public class JoinsContext
                QJoinMetaData found         = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable());
                if(found != null)
                {
+                  log("Found joinMetaData - setting it in queryJoin", logPair("joinMetaDataName", found.getName()), logPair("baseTableName", baseTableName), logPair("joinTable", queryJoin.getJoinTable()));
                   queryJoin.setJoinMetaData(found);
                }
                else
@@ -197,15 +287,13 @@ public class JoinsContext
                   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                   // else, the join must be indirect - so look for an exposedJoin that will have a joinPath that will connect us //
                   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  LOG.debug("Looking for an exposed join...", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()));
-
                   QTableMetaData mainTable          = instance.getTable(mainTableName);
                   boolean        addedAnyQueryJoins = false;
                   for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins()))
                   {
                      if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable()))
                      {
-                        LOG.debug("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
+                        log("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
 
                         /////////////////////////////////////////////////////////////////////////////////////
                         // loop backward through the join path (from the joinTable back to the main table) //
@@ -250,7 +338,7 @@ public class JoinsContext
                                  QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd);
                                  queryJoinToAdd.setType(queryJoin.getType());
                                  addedAnyQueryJoins = true;
-                                 this.addQueryJoin(queryJoinToAdd);
+                                 this.addQueryJoin(queryJoinToAdd, "forExposedJoin");
                               }
                            }
 
@@ -377,9 +465,9 @@ public class JoinsContext
     **
     ** e.g., Given:
     **   FROM `order` INNER JOIN line_item li
-    ** hasAliasOrTable("order") => true
-    ** hasAliasOrTable("li") => false
-    ** hasAliasOrTable("line_item") => true
+    ** hasTable("order") => true
+    ** hasTable("li") => false
+    ** hasTable("line_item") => true
     *******************************************************************************/
    public boolean hasTable(String table)
    {
@@ -415,15 +503,17 @@ public class JoinsContext
 
       for(String filterTable : filterTables)
       {
+         log("Evaluating filterTable", logPair("filterTable", filterTable));
          if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable))
          {
+            log("- table not in query - adding it", logPair("filterTable", filterTable));
             boolean found = false;
             for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values())
             {
                QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join);
                if(queryJoin != null)
                {
-                  this.addQueryJoin(queryJoin);
+                  this.addQueryJoin(queryJoin, "forFilter (join found in instance)");
                   found = true;
                   break;
                }
@@ -432,7 +522,7 @@ public class JoinsContext
             if(!found)
             {
                QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin);
+               this.addQueryJoin(queryJoin, "forFilter (join not found in instance)");
             }
          }
       }
@@ -569,4 +659,14 @@ public class JoinsContext
    {
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void log(String message, LogPair... logPairs)
+   {
+      LOG.log(logLevel, message, null, logPairs);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -42,6 +42,7 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
    private QQueryFilter        filter;
 
    private RecordPipe recordPipe;
+   private Integer    timeoutSeconds;
 
    private boolean shouldTranslatePossibleValues = false;
    private boolean shouldGenerateDisplayValues   = false;
@@ -537,4 +538,35 @@ public class QueryInput extends AbstractTableActionInput implements QueryOrGetIn
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for timeoutSeconds
+    *******************************************************************************/
+   public Integer getTimeoutSeconds()
+   {
+      return (this.timeoutSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for timeoutSeconds
+    *******************************************************************************/
+   public void setTimeoutSeconds(Integer timeoutSeconds)
+   {
+      this.timeoutSeconds = timeoutSeconds;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for timeoutSeconds
+    *******************************************************************************/
+   public QueryInput withTimeoutSeconds(Integer timeoutSeconds)
+   {
+      this.timeoutSeconds = timeoutSeconds;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -421,7 +421,7 @@ public abstract class QRecordEntity
          {
             if(!method.getName().equals("getClass"))
             {
-               LOG.info("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported.");
+               LOG.debug("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported.");
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java

@@ -32,6 +32,9 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
  *******************************************************************************/
 public abstract class MetaDataProducer<T extends TopLevelMetaDataInterface>
 {
+   public static final int DEFAULT_SORT_ORDER = 500;
+
+
 
    /*******************************************************************************
     ** Produce the metaData object.  Generally, you don't want to add it to the instance
@@ -43,11 +46,13 @@ public abstract class MetaDataProducer<T extends TopLevelMetaDataInterface>
 
    /*******************************************************************************
     ** In case this producer needs to run before (or after) others, this method
-    ** can help influence that (e.g., if used by MetaDataProducerHelper).
+    ** can control influence that (e.g., if used by MetaDataProducerHelper).
+    **
+    ** Smaller values run first.
     *******************************************************************************/
    public int getSortOrder()
    {
-      return (500);
+      return (DEFAULT_SORT_ORDER);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.model.metadata.frontend;
 
 
+import java.io.Serializable;
 import java.util.List;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
@@ -38,14 +39,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 @JsonInclude(Include.NON_NULL)
 public class QFrontendFieldMetaData
 {
-   private String     name;
-   private String     label;
-   private QFieldType type;
-   private boolean    isRequired;
-   private boolean    isEditable;
-   private boolean    isHeavy;
-   private String     possibleValueSourceName;
-   private String     displayFormat;
+   private String       name;
+   private String       label;
+   private QFieldType   type;
+   private boolean      isRequired;
+   private boolean      isEditable;
+   private boolean      isHeavy;
+   private String       possibleValueSourceName;
+   private String       displayFormat;
+   private Serializable defaultValue;
 
    private List<FieldAdornment> adornments;
 
@@ -69,6 +71,7 @@ public class QFrontendFieldMetaData
       this.possibleValueSourceName = fieldMetaData.getPossibleValueSourceName();
       this.displayFormat = fieldMetaData.getDisplayFormat();
       this.adornments = fieldMetaData.getAdornments();
+      this.defaultValue = fieldMetaData.getDefaultValue();
    }
 
 
@@ -170,4 +173,14 @@ public class QFrontendFieldMetaData
       return possibleValueSourceName;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for defaultValue
+    **
+    *******************************************************************************/
+   public Serializable getDefaultValue()
+   {
+      return defaultValue;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QAppMetaData.java

@@ -24,6 +24,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.layout;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
@@ -36,7 +38,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  ** MetaData definition of an App - an entity that organizes tables & processes
  ** and can be arranged hierarchically (e.g, apps can contain other apps).
  *******************************************************************************/
-public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRules
+public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface
 {
    private String name;
    private String label;
@@ -414,4 +416,14 @@ public class QAppMetaData implements QAppChildMetaData, MetaDataWithPermissionRu
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addApp(this);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java

@@ -34,6 +34,10 @@ import java.util.List;
  ** - recordSecurityLock.fieldName = order.clientId
  ** - recordSecurityLock.joinNameChain = [orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic]
  ** that is - what's the chain that takes us FROM the security fieldName TO the table with the lock.
+ **
+ ** LockScope controls what the lock prevents users from doing without a valid key.
+ ** - READ_AND_WRITE means that users cannot read or write records without a valid key.
+ ** - WRITE means that users cannot write records without a valid key (but they can read them).
  *******************************************************************************/
 public class RecordSecurityLock
 {
@@ -42,6 +46,8 @@ public class RecordSecurityLock
    private List<String>      joinNameChain;
    private NullValueBehavior nullValueBehavior = NullValueBehavior.DENY;
 
+   private LockScope lockScope = LockScope.READ_AND_WRITE;
+
 
 
    /*******************************************************************************
@@ -66,6 +72,17 @@ public class RecordSecurityLock
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public enum LockScope
+   {
+      READ_AND_WRITE,
+      WRITE
+   }
+
+
+
    /*******************************************************************************
     ** Getter for securityKeyType
     *******************************************************************************/
@@ -188,4 +205,35 @@ public class RecordSecurityLock
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for lockScope
+    *******************************************************************************/
+   public LockScope getLockScope()
+   {
+      return (this.lockScope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for lockScope
+    *******************************************************************************/
+   public void setLockScope(LockScope lockScope)
+   {
+      this.lockScope = lockScope;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for lockScope
+    *******************************************************************************/
+   public RecordSecurityLock withLockScope(LockScope lockScope)
+   {
+      this.lockScope = lockScope;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java

@@ -0,0 +1,80 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.security;
+
+
+import java.util.List;
+
+
+/*******************************************************************************
+ ** standard filtering operations for lists of record security locks.
+ *******************************************************************************/
+public class RecordSecurityLockFilters
+{
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to reads.
+    *******************************************************************************/
+   public static List<RecordSecurityLock> filterForReadLocks(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      return (recordSecurityLocks.stream().filter(rsl -> RecordSecurityLock.LockScope.READ_AND_WRITE.equals(rsl.getLockScope())).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to writes.
+    *******************************************************************************/
+   public static List<RecordSecurityLock> filterForWriteLocks(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      return (recordSecurityLocks.stream().filter(rsl ->
+         RecordSecurityLock.LockScope.READ_AND_WRITE.equals(rsl.getLockScope())
+            || RecordSecurityLock.LockScope.WRITE.equals(rsl.getLockScope()
+         )).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that are WRITE type only.
+    *******************************************************************************/
+   public static List<RecordSecurityLock> filterForOnlyWriteLocks(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      return (recordSecurityLocks.stream().filter(rsl -> RecordSecurityLock.LockScope.WRITE.equals(rsl.getLockScope())).toList());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
 import java.util.List;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -62,7 +63,19 @@ public class ExposedJoin
    /*******************************************************************************
     **
     *******************************************************************************/
-   public Boolean getIsMany()
+   @JsonIgnore
+   public boolean getIsMany()
+   {
+      return (getIsMany(QContext.getQInstance()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @JsonIgnore
+   public Boolean getIsMany(QInstance qInstance)
    {
       if(isMany == null)
       {
@@ -70,8 +83,6 @@ public class ExposedJoin
          {
             try
             {
-               QInstance qInstance = QContext.getQInstance();
-
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // loop backward through the joinPath, starting at the join table (since we don't know the table that this exposedJoin is attached to!) //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -204,5 +215,4 @@ public class ExposedJoin
       this.joinPath = joinPath;
       return (this);
    }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QSupplementalTableMetaData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
 /*******************************************************************************
  ** Base-class for table-level meta-data defined by some supplemental module, etc,
  ** outside of qqq core
@@ -60,7 +63,7 @@ public abstract class QSupplementalTableMetaData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void enrich(QTableMetaData table)
+   public void enrich(QInstance qInstance, QTableMetaData table)
    {
       ////////////////////////
       // noop in base class //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -303,14 +305,18 @@ public class ColumnStatsStep implements BackendStep
          /////////////////////////////////////////////////////////////////////////////////
          // just in case any of these don't fit in an integer, use decimal for them all //
          /////////////////////////////////////////////////////////////////////////////////
-         Aggregate      countNonNullAggregate  = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
-         Aggregate      countDistinctAggregate = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL);
-         Aggregate      sumAggregate           = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL);
-         Aggregate      avgAggregate           = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL);
-         Aggregate      minAggregate           = new Aggregate(fieldName, AggregateOperator.MIN);
-         Aggregate      maxAggregate           = new Aggregate(fieldName, AggregateOperator.MAX);
-         AggregateInput statsAggregateInput    = new AggregateInput();
+         Aggregate countTotalRowsAggregate = new Aggregate(table.getPrimaryKeyField(), AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
+         Aggregate countNonNullAggregate   = new Aggregate(fieldName, AggregateOperator.COUNT).withFieldType(QFieldType.DECIMAL);
+         Aggregate countDistinctAggregate  = new Aggregate(fieldName, AggregateOperator.COUNT_DISTINCT).withFieldType(QFieldType.DECIMAL);
+         Aggregate sumAggregate            = new Aggregate(fieldName, AggregateOperator.SUM).withFieldType(QFieldType.DECIMAL);
+         Aggregate avgAggregate            = new Aggregate(fieldName, AggregateOperator.AVG).withFieldType(QFieldType.DECIMAL);
+         Aggregate minAggregate            = new Aggregate(fieldName, AggregateOperator.MIN);
+         Aggregate maxAggregate            = new Aggregate(fieldName, AggregateOperator.MAX);
+
+         AggregateInput statsAggregateInput = new AggregateInput();
+         statsAggregateInput.withAggregate(countTotalRowsAggregate);
          statsAggregateInput.withAggregate(countNonNullAggregate);
+
          if(doCountDistinct)
          {
             statsAggregateInput.withAggregate(countDistinctAggregate);
@@ -332,6 +338,7 @@ public class ColumnStatsStep implements BackendStep
             statsAggregateInput.withAggregate(maxAggregate);
          }
 
+         BigDecimal totalRows = null;
          if(CollectionUtils.nullSafeHasContents(statsAggregateInput.getAggregates()))
          {
             statsAggregateInput.setTableName(tableName);
@@ -346,6 +353,8 @@ public class ColumnStatsStep implements BackendStep
             {
                AggregateResult statsAggregateResult = statsAggregateOutput.getResults().get(0);
 
+               totalRows = ValueUtils.getValueAsBigDecimal(statsAggregateResult.getAggregateValue(countTotalRowsAggregate));
+
                statsRecord.setValue(countNonNullField.getName(), statsAggregateResult.getAggregateValue(countNonNullAggregate));
                if(doCountDistinct)
                {
@@ -388,6 +397,27 @@ public class ColumnStatsStep implements BackendStep
             }
          }
 
+         /////////////////////
+         // figure count%'s //
+         /////////////////////
+         if(totalRows == null)
+         {
+            totalRows = new BigDecimal(valueCounts.stream().mapToInt(r -> r.getValueInteger("count")).sum());
+         }
+
+         if(totalRows != null && totalRows.compareTo(BigDecimal.ZERO) > 0)
+         {
+            BigDecimal oneHundred = new BigDecimal(100);
+            for(QRecord valueCount : valueCounts)
+            {
+               BigDecimal percent = new BigDecimal(Objects.requireNonNullElse(valueCount.getValueInteger("count"), 0)).divide(totalRows, 4, RoundingMode.HALF_UP).multiply(oneHundred).setScale(2, RoundingMode.HALF_UP);
+               valueCount.setValue("percent", percent);
+            }
+
+            QFieldMetaData percentField = new QFieldMetaData("percent", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.PERCENT_POINT2).withLabel("Percent");
+            QValueFormatter.setDisplayValuesInRecords(Map.of(fieldName, field, "percent", percentField), valueCounts);
+         }
+
          QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(null);
          fields.forEach(qInstanceEnricher::enrichField);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java

@@ -31,7 +31,10 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
@@ -46,6 +49,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -69,7 +74,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
    {
       InsertAction insertAction = new InsertAction();
       InsertInput  insertInput  = new InsertInput();
-      insertInput.setTableName("scriptRevision");
+      insertInput.setTableName(ScriptRevision.TABLE_NAME);
       QBackendTransaction transaction = insertAction.openTransaction(insertInput);
       insertInput.setTransaction(transaction);
 
@@ -87,14 +92,23 @@ public class StoreScriptRevisionProcessStep implements BackendStep
          // get the existing script, to update //
          ////////////////////////////////////////
          GetInput getInput = new GetInput();
-         getInput.setTableName("script");
+         getInput.setTableName(Script.TABLE_NAME);
          getInput.setPrimaryKey(scriptId);
          getInput.setTransaction(transaction);
          GetOutput getOutput = new GetAction().execute(getInput);
          QRecord   script    = getOutput.getRecord();
 
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // in case the app added a security field to the scripts table, make sure the user is allowed to edit the script //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE);
+         if(CollectionUtils.nullSafeHasContents(script.getErrors()))
+         {
+            throw (new QPermissionDeniedException(script.getErrors().get(0).getMessage()));
+         }
+
          QueryInput queryInput = new QueryInput();
-         queryInput.setTableName("scriptRevision");
+         queryInput.setTableName(ScriptRevision.TABLE_NAME);
          queryInput.setFilter(new QQueryFilter()
             .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
             .withOrderBy(new QFilterOrderBy("sequenceNo", false))
@@ -159,6 +173,15 @@ public class StoreScriptRevisionProcessStep implements BackendStep
                   .toQRecord());
             }
          }
+         else if(StringUtils.hasContent(input.getValueString("contents")))
+         {
+            scriptRevisionFileRecords = new ArrayList<>();
+            scriptRevisionFileRecords.add(new ScriptRevisionFile()
+               .withScriptRevisionId(scriptRevisionId)
+               .withFileName("Script.js")
+               .withContents(input.getValueString("contents"))
+               .toQRecord());
+         }
 
          if(CollectionUtils.nullSafeHasContents(scriptRevisionFileRecords))
          {
@@ -174,7 +197,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
          ////////////////////////////////////////////////////
          script.setValue("currentScriptRevisionId", scriptRevision.getValue("id"));
          UpdateInput updateInput = new UpdateInput();
-         updateInput.setTableName("script");
+         updateInput.setTableName(Script.TABLE_NAME);
          updateInput.setRecords(List.of(script));
          updateInput.setTransaction(transaction);
          new UpdateAction().execute(updateInput);
@@ -189,6 +212,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
       catch(Exception e)
       {
          transaction.rollback();
+         throw (e);
       }
       finally
       {

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/customizers/ChildInserterPostInsertCustomizerTest.java

@@ -70,7 +70,7 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
    void testEmptyCases() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
-      addPostInsertActionToTable(qInstance);
+      addPostInsertActionToPersonTable(qInstance);
 
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
@@ -95,10 +95,21 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void addPostInsertActionToTable(QInstance qInstance)
+   private static void addPostInsertActionToPersonTable(QInstance qInstance)
    {
       qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
-         .withCustomizer(TableCustomizers.POST_INSERT_RECORD.getRole(), new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class));
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(PersonPostInsertAddFavoriteShapeCustomizer.class));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void addPostInsertActionToShapeTable(QInstance qInstance)
+   {
+      qInstance.getTable(TestUtils.TABLE_NAME_SHAPE)
+         .withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ShapePostInsertAddPersonCustomizer.class));
    }
 
 
@@ -107,10 +118,10 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testSimpleCase() throws QException
+   void testSimpleParentPointsAtChildCase() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
-      addPostInsertActionToTable(qInstance);
+      addPostInsertActionToPersonTable(qInstance);
 
       assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
 
@@ -135,10 +146,10 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
     **
     *******************************************************************************/
    @Test
-   void testComplexCase() throws QException
+   void testComplexParentPointsAtChildCase() throws QException
    {
       QInstance qInstance = QContext.getQInstance();
-      addPostInsertActionToTable(qInstance);
+      addPostInsertActionToPersonTable(qInstance);
 
       assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
 
@@ -169,6 +180,34 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
    /*******************************************************************************
     **
     *******************************************************************************/
+   @Test
+   void testSimpleChildPointsAtParentCase() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      addPostInsertActionToShapeTable(qInstance);
+
+      assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY).size());
+      assertEquals(0, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_SHAPE).size());
+
+      InsertInput insertInput = new InsertInput();
+      insertInput.setTableName(TestUtils.TABLE_NAME_SHAPE);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("name", "Circle")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      Integer      shapeId      = insertOutput.getRecords().get(0).getValueInteger("id");
+
+      List<QRecord> personRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY);
+      assertEquals(1, personRecords.size());
+      assertEquals(shapeId, personRecords.get(0).getValue("favoriteShapeId"));
+      assertEquals("loves Circle", personRecords.get(0).getValue("lastName"));
+   }
+
+
+
+   /*******************************************************************************
+    ** for the person table - where we do PARENT_POINTS_AT_CHILD
+    *******************************************************************************/
    public static class PersonPostInsertAddFavoriteShapeCustomizer extends ChildInserterPostInsertCustomizer
    {
 
@@ -215,4 +254,47 @@ class ChildInserterPostInsertCustomizerTest extends BaseTest
       }
    }
 
+
+
+   /*******************************************************************************
+    ** for the shape table - where we do CHILD_POINTS_AT_PARENT
+    *******************************************************************************/
+   public static class ShapePostInsertAddPersonCustomizer extends ChildInserterPostInsertCustomizer
+   {
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public QRecord buildChildForRecord(QRecord parentRecord) throws QException
+      {
+         return (new QRecord()
+            .withValue("firstName", "Someone who")
+            .withValue("lastName", "loves " + parentRecord.getValue("name"))
+            .withValue("favoriteShapeId", parentRecord.getValue("id")));
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public String getChildTableName()
+      {
+         return (TestUtils.TABLE_NAME_PERSON_MEMORY);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public RelationshipType getRelationshipType()
+      {
+         return (RelationshipType.CHILD_POINTS_AT_PARENT);
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeActionTest.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.scripts.logging.Log4jCodeExecution
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.NoopCodeExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLoggerInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
+import com.kingsrook.qqq.backend.core.actions.scripts.logging.SystemOutExecutionLogger;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -112,6 +113,21 @@ class ExecuteCodeActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSystemOutLogger() throws QException
+   {
+      QInstance         qInstance         = QContext.getQInstance();
+      ExecuteCodeInput  executeCodeInput  = setupInput(qInstance, Map.of("x", 4), new SystemOutExecutionLogger());
+      ExecuteCodeOutput executeCodeOutput = new ExecuteCodeOutput();
+      new ExecuteCodeAction().run(executeCodeInput, executeCodeOutput);
+      assertEquals(16, executeCodeOutput.getOutput());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteActionTest.java

@@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -365,6 +366,36 @@ class DeleteActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityLockWriteScope() throws QException
+   {
+      TestUtils.updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // try to delete 1, 2, and 3.  2 should be blocked, because it has a writable-By that isn't in our session //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe")));
+      DeleteInput deleteInput = new DeleteInput();
+      deleteInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      deleteInput.setPrimaryKeys(List.of(1, 2, 3));
+      DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+
+      assertEquals(1, deleteOutput.getRecordsWithErrors().size());
+      assertThat(deleteOutput.getRecordsWithErrors().get(0).getErrors().get(0).getMessage())
+         .contains("You do not have permission")
+         .contains("kmarsh")
+         .contains("Only Writable By");
+
+      assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)).getCount());
+      assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 2)))).getCount());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java

@@ -35,9 +35,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -692,4 +695,60 @@ class InsertActionTest extends BaseTest
       assertEquals(3, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_ORDER).size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityLockWriteScope() throws QException
+   {
+      TestUtils.updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords();
+
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("hsimpson")));
+
+      /////////////////////////////////////////////////////////////////////////////////////////
+      // with only hsimpson in our key, make sure we can't insert a row w/ a different value //
+      /////////////////////////////////////////////////////////////////////////////////////////
+      {
+         InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 100).withValue("firstName", "Jean-Luc").withValue("onlyWritableBy", "jkirk")
+         )));
+         List<QErrorMessage> errors = insertOutput.getRecords().get(0).getErrors();
+         assertEquals(1, errors.size());
+         assertThat(errors.get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("jkirk")
+            .contains("Only Writable By");
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure we can insert w/ a null in onlyWritableBy (because key (from test utils) was set to allow null) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 101).withValue("firstName", "Benajamin").withValue("onlyWritableBy", null)
+         )));
+         List<QErrorMessage> errors = insertOutput.getRecords().get(0).getErrors();
+         assertEquals(0, errors.size());
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////
+      // change the null behavior to deny, and try above again, expecting an error //
+      ///////////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY);
+      {
+         InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 102).withValue("firstName", "Katherine").withValue("onlyWritableBy", null)
+         )));
+         List<QErrorMessage> errors = insertOutput.getRecords().get(0).getErrors();
+         assertEquals(1, errors.size());
+         assertThat(errors.get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("without a value")
+            .contains("Only Writable By");
+      }
+
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateActionTest.java

@@ -29,12 +29,18 @@ import java.util.Objects;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
@@ -492,7 +498,7 @@ class UpdateActionTest extends BaseTest
          updateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM);
          updateInput.setRecords(List.of(new QRecord().withValue("id", 20).withValue("sku", "BASIC3")));
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-         assertEquals("No record was found to update for Id = 20", updateOutput.getRecords().get(0).getErrors().get(0).getMessage());
+         assertTrue(updateOutput.getRecords().get(0).getErrors().stream().anyMatch(em -> em.getMessage().equals("No record was found to update for Id = 20")));
       }
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -504,7 +510,7 @@ class UpdateActionTest extends BaseTest
          updateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM);
          updateInput.setRecords(List.of(new QRecord().withValue("id", 10).withValue("orderId", 2).withValue("sku", "BASIC3")));
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-         assertEquals("You do not have permission to update this record - the referenced Order was not found.", updateOutput.getRecords().get(0).getErrors().get(0).getMessage());
+         assertTrue(updateOutput.getRecords().get(0).getErrors().stream().anyMatch(em -> em.getMessage().equals("You do not have permission to update this record - the referenced Order was not found.")));
       }
 
       ///////////////////////////////////////////////////////////
@@ -528,7 +534,7 @@ class UpdateActionTest extends BaseTest
          updateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC);
          updateInput.setRecords(List.of(new QRecord().withValue("id", 200).withValue("key", "updatedKey")));
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-         assertEquals("No record was found to update for Id = 200", updateOutput.getRecords().get(0).getErrors().get(0).getMessage());
+         assertTrue(updateOutput.getRecords().get(0).getErrors().stream().anyMatch(em -> em.getMessage().equals("No record was found to update for Id = 200")));
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -706,4 +712,79 @@ class UpdateActionTest extends BaseTest
       assertEquals(1, TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER).stream().filter(r -> r.getValue("storeId") == null).count());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityLockWriteScope() throws QException
+   {
+      TestUtils.updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // try to update them all 1, 2, and 3.  2 should be blocked, because it has a writable-By that isn't in our session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe")));
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("lastName", "Kelkhoff"),
+            new QRecord().withValue("id", 2).withValue("lastName", "Chamberlain"),
+            new QRecord().withValue("id", 3).withValue("lastName", "Maes")
+         )));
+
+         List<QRecord> errorRecords = updateOutput.getRecords().stream().filter(r -> CollectionUtils.nullSafeHasContents(r.getErrors())).toList();
+         assertEquals(1, errorRecords.size());
+         assertEquals(2, errorRecords.get(0).getValueInteger("id"));
+         assertThat(errorRecords.get(0).getErrors().get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("kmarsh")
+            .contains("Only Writable By");
+
+         assertEquals(2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)
+            .withFilter(new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.IS_NOT_BLANK)))).getCount());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now try to change one of the records to have a different value in the lock-field.  Should fail (as it's not in our session) //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("onlyWritableBy", "ecartman"))));
+
+         List<QRecord> errorRecords = updateOutput.getRecords().stream().filter(r -> CollectionUtils.nullSafeHasContents(r.getErrors())).toList();
+         assertEquals(1, errorRecords.size());
+         assertThat(errorRecords.get(0).getErrors().get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("ecartman")
+            .contains("Only Writable By");
+      }
+
+      ///////////////////////////////////////////////////////////////
+      // add that to our session and confirm we can do that update //
+      ///////////////////////////////////////////////////////////////
+      {
+         QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe", "ecartman")));
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("onlyWritableBy", "ecartman"))));
+         List<QRecord> errorRecords = updateOutput.getRecords().stream().filter(r -> CollectionUtils.nullSafeHasContents(r.getErrors())).toList();
+         assertEquals(0, errorRecords.size());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // change the null behavior to deny, then try to udpate a record and remove its onlyWritableBy //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY);
+      {
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("onlyWritableBy", null))));
+         List<QErrorMessage> errors = updateOutput.getRecords().get(0).getErrors();
+         assertEquals(1, errors.size());
+         assertThat(errors.get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("without a value")
+            .contains("Only Writable By");
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ActionTimeoutHelperTest.java

@@ -0,0 +1,85 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables.helpers;
+
+
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for ActionTimeoutHelper 
+ *******************************************************************************/
+class ActionTimeoutHelperTest extends BaseTest
+{
+   boolean didCancel = false;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testTimesOut()
+   {
+      didCancel = false;
+      ActionTimeoutHelper actionTimeoutHelper = new ActionTimeoutHelper(10, TimeUnit.MILLISECONDS, () -> doCancel());
+      actionTimeoutHelper.start();
+      SleepUtils.sleep(50, TimeUnit.MILLISECONDS);
+      assertTrue(didCancel);
+      assertTrue(actionTimeoutHelper.getDidTimeout());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetsCancelled()
+   {
+      didCancel = false;
+      ActionTimeoutHelper actionTimeoutHelper = new ActionTimeoutHelper(100, TimeUnit.MILLISECONDS, () -> doCancel());
+      actionTimeoutHelper.start();
+      SleepUtils.sleep(10, TimeUnit.MILLISECONDS);
+      actionTimeoutHelper.cancel();
+      assertFalse(didCancel);
+      SleepUtils.sleep(200, TimeUnit.MILLISECONDS);
+      assertFalse(didCancel);
+      assertFalse(actionTimeoutHelper.getDidTimeout());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void doCancel()
+   {
+      didCancel = true;
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -822,6 +822,58 @@ class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSectionsWithJoinFields()
+   {
+      Consumer<QTableMetaData> putAllFieldsInASection = table -> table.addSection(new QFieldSection()
+         .withName("section0")
+         .withTier(Tier.T1)
+         .withFieldNames(new ArrayList<>(table.getFields().keySet())));
+
+      assertValidationFailureReasons(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_ORDER);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_LINE_ITEM + ".sku");
+      }, "orderLine.sku references an is-many join, which is not supported");
+
+      assertValidationSuccess(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_ORDER + ".orderNo");
+      });
+
+      assertValidationFailureReasons(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_ORDER + ".asdf");
+      }, "order.asdf specifies a fieldName [asdf] which does not exist in that table [order].");
+
+      /////////////////////////////////////////////////////////////////////////////
+      // this is aactually allowed, well, just not considered as a join-field... //
+      /////////////////////////////////////////////////////////////////////////////
+      // assertValidationFailureReasons(qInstance ->
+      // {
+      //    QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+      //    putAllFieldsInASection.accept(table);
+      //    table.getSections().get(0).getFieldNames().add("foo.bar");
+      // }, "unrecognized table name [foo]");
+
+      assertValidationFailureReasons(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_SHAPE + ".id");
+      }, "[shape] which is not an exposed join on this table");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStepTest.java

@@ -1,7 +1,29 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
 import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -53,9 +75,20 @@ class ColumnStatsStepTest extends BaseTest
       @SuppressWarnings("unchecked")
       List<QRecord> valueCounts = (List<QRecord>) values.get("valueCounts");
 
-      assertThat(valueCounts.get(0).getValues()).hasFieldOrPropertyWithValue("lastName", "Simpson").hasFieldOrPropertyWithValue("count", 3);
-      assertThat(valueCounts.get(1).getValues()).hasFieldOrPropertyWithValue("lastName", null).hasFieldOrPropertyWithValue("count", 2); // here's the assert for the "" and null record above.
-      assertThat(valueCounts.get(2).getValues()).hasFieldOrPropertyWithValue("lastName", "Flanders").hasFieldOrPropertyWithValue("count", 1);
+      assertThat(valueCounts.get(0).getValues())
+         .hasFieldOrPropertyWithValue("lastName", "Simpson")
+         .hasFieldOrPropertyWithValue("count", 3)
+         .hasFieldOrPropertyWithValue("percent", new BigDecimal("50.00"));
+
+      assertThat(valueCounts.get(1).getValues())
+         .hasFieldOrPropertyWithValue("lastName", null)
+         .hasFieldOrPropertyWithValue("count", 2) // here's the assert for the "" and null record above.
+         .hasFieldOrPropertyWithValue("percent", new BigDecimal("33.33"));
+
+      assertThat(valueCounts.get(2).getValues())
+         .hasFieldOrPropertyWithValue("lastName", "Flanders")
+         .hasFieldOrPropertyWithValue("count", 1)
+         .hasFieldOrPropertyWithValue("percent", new BigDecimal("16.67"));
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -33,15 +33,18 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
 import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -110,6 +113,9 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.reports.RunReportForRecordProcess;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -590,6 +596,7 @@ public class TestUtils
             .withFieldName("order.storeId")
             .withJoinNameChain(List.of("orderLineItem")))
          .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_LINE_ITEM_EXTRINSIC).withJoinName("lineItemLineItemExtrinsic"))
+         .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
@@ -1393,4 +1400,39 @@ public class TestUtils
             ))
          );
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      qInstance.addSecurityKeyType(new QSecurityKeyType()
+         .withName("writableBy"));
+
+      QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.withField(new QFieldMetaData("onlyWritableBy", QFieldType.STRING).withLabel("Only Writable By"));
+      table.withRecordSecurityLock(new RecordSecurityLock()
+         .withSecurityKeyType("writableBy")
+         .withFieldName("onlyWritableBy")
+         .withNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW)
+         .withLockScope(RecordSecurityLock.LockScope.WRITE));
+
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe", "kmarsh")));
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "Darin"),
+         new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("onlyWritableBy", "kmarsh"),
+         new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("onlyWritableBy", "jdoe")
+      )));
+
+      //////////////////////////////////////////////
+      // make sure we can query for all 3 records //
+      //////////////////////////////////////////////
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe")));
+      assertEquals(3, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)).getCount());
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/YamlUtilsTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.utils;
 
 import java.util.Map;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import org.junit.jupiter.api.Test;
 
@@ -31,7 +32,7 @@ import org.junit.jupiter.api.Test;
 /*******************************************************************************
  ** Unit test for YamlUtils
  *******************************************************************************/
-class YamlUtilsTest
+class YamlUtilsTest extends BaseTest
 {
 
    /*******************************************************************************

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/APIBackendModule.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.api.actions.APICountAction;
+import com.kingsrook.qqq.backend.module.api.actions.APIDeleteAction;
 import com.kingsrook.qqq.backend.module.api.actions.APIGetAction;
 import com.kingsrook.qqq.backend.module.api.actions.APIInsertAction;
 import com.kingsrook.qqq.backend.module.api.actions.APIQueryAction;
@@ -136,7 +137,7 @@ public class APIBackendModule implements QBackendModuleInterface
    @Override
    public DeleteInterface getDeleteInterface()
    {
-      return (null); //return (new RDBMSDeleteAction());
+      return (new APIDeleteAction());
    }
 
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/APIDeleteAction.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.actions;
+
+
+import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class APIDeleteAction extends AbstractAPIAction implements DeleteInterface
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public DeleteOutput execute(DeleteInput deleteInput) throws QException
+   {
+      QTableMetaData table = deleteInput.getTable();
+      preAction(deleteInput);
+      return (apiActionUtil.doDelete(table, deleteInput));
+   }
+
+
+
+   /*******************************************************************************
+    ** Specify whether this particular module's update action can & should fetch
+    ** records before updating them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   @Override
+   public boolean supportsPreFetchQuery()
+   {
+      return (false);
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -43,6 +43,8 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -79,6 +81,7 @@ import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
@@ -384,6 +387,41 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    **
+    *
+    *******************************************************************************/
+   public DeleteOutput doDelete(QTableMetaData table, DeleteInput deleteInput) throws QException
+   {
+      try
+      {
+         DeleteOutput deleteOutput = new DeleteOutput();
+
+         String     urlSuffix = buildQueryStringForDelete(deleteInput.getQueryFilter(), deleteInput.getPrimaryKeys());
+         String     url       = buildTableUrl(table);
+         HttpDelete request   = new HttpDelete(url + urlSuffix);
+
+         QHttpResponse response = makeRequest(table, request);
+         if(response.getStatusCode() == 204)
+         {
+            deleteOutput.setDeletedRecordCount(1);
+         }
+         else
+         {
+            deleteOutput.setDeletedRecordCount(0);
+         }
+
+         return (deleteOutput);
+      }
+      catch(Exception e)
+      {
+         LOG.error("Error in API Delete", e);
+         throw new QException("Error executing Delete: " + e.getMessage(), e);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -481,7 +519,7 @@ public class BaseAPIActionUtil
          {
             String wrapperObjectName = getBackendDetails(table).getTableWrapperObjectName();
             jsonObject = JsonUtils.toJSONObject(resultString);
-            if(jsonObject.has(wrapperObjectName))
+            if(jsonObject.has(wrapperObjectName) && !jsonObject.isNull(wrapperObjectName))
             {
                Object o = jsonObject.get(wrapperObjectName);
                if(o instanceof JSONArray jsonArray)
@@ -601,6 +639,17 @@ public class BaseAPIActionUtil
 
 
 
+   /*******************************************************************************
+    ** method to build up delete string based on a given QFilter object
+    **
+    *******************************************************************************/
+   protected String buildQueryStringForDelete(QQueryFilter filter, List<Serializable> primaryKeys) throws QException
+   {
+      return ("");
+   }
+
+
+
    /*******************************************************************************
     ** Do a default query string for a single-record GET - e.g., a query for just 1 record.
     *******************************************************************************/
@@ -701,6 +750,10 @@ public class BaseAPIActionUtil
                throw (new QException("Error setting authorization query parameter", e));
             }
          }
+         case CUSTOM ->
+         {
+            handleCustomAuthorization(request);
+         }
          default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType());
       }
    }
@@ -1349,4 +1402,16 @@ public class BaseAPIActionUtil
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       return (7 * 1024);
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void handleCustomAuthorization(HttpRequestBase request) throws QException
+   {
+      ///////////////////////////////////////////////////////////////////////
+      // nothing to do at this layer, meant to be overridden by subclasses //
+      ///////////////////////////////////////////////////////////////////////
+   }
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/QHttpResponse.java

@@ -69,7 +69,11 @@ public class QHttpResponse
             this.statusProtocolVersion = httpResponse.getStatusLine().getProtocolVersion().toString();
          }
       }
-      this.content = EntityUtils.toString(httpResponse.getEntity());
+
+      if(this.statusCode == null || this.statusCode != 204)
+      {
+         this.content = EntityUtils.toString(httpResponse.getEntity());
+      }
    }
 
 

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java

@@ -31,6 +31,7 @@ public enum AuthorizationType
    API_TOKEN,
    BASIC_AUTH_API_KEY,
    BASIC_AUTH_USERNAME_PASSWORD,
+   CUSTOM,
    OAUTH2,
    API_KEY_QUERY_PARAM,
 }

qqq-backend-module-api/src/test/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtilTest.java

@@ -39,6 +39,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -422,6 +423,25 @@ class BaseAPIActionUtilTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDelete() throws QException
+   {
+      mockApiUtilsHelper.enqueueMockResponse("");
+      mockApiUtilsHelper.enqueueMockResponse(new QHttpResponse().withStatusCode(204).withContent(null));
+
+      DeleteInput deleteInput = new DeleteInput();
+      deleteInput.setTableName(TestUtils.MOCK_TABLE_NAME);
+      deleteInput.setPrimaryKeys(List.of(1));
+      DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+
+      // not sure what to assert in here...
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions;
 
 import java.io.Serializable;
 import java.sql.Connection;
+import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.time.Instant;
@@ -69,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -91,6 +93,9 @@ public abstract class AbstractRDBMSAction implements QActionInterface
 
    protected QueryStat queryStat;
 
+   protected PreparedStatement statement;
+   protected boolean           isCancelled = false;
+
 
 
    /*******************************************************************************
@@ -383,7 +388,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
       QQueryFilter securityFilter = new QQueryFilter();
       securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND);
 
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
          // todo - uh, if it's a RIGHT (or FULL) join, then, this should be isOuter = true, right?
          boolean isOuter = false;
@@ -403,7 +408,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
          }
 
          QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
-         for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks()))
+         for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks())))
          {
             boolean isOuter = queryJoin.getType().equals(QueryJoin.Type.LEFT); // todo full?
             addSubFilterForRecordSecurityLock(instance, session, joinTable, securityFilter, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias(), isOuter);
@@ -1031,7 +1036,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
    {
       if(table != null)
       {
-         for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+         for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
          {
             for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
             {
@@ -1094,4 +1099,28 @@ public abstract class AbstractRDBMSAction implements QActionInterface
       this.queryStat = queryStat;
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void doCancelQuery()
+   {
+      isCancelled = true;
+      if(statement == null)
+      {
+         LOG.warn("Statement was null when requested to cancel query");
+         return;
+      }
+
+      try
+      {
+         statement.cancel();
+      }
+      catch(SQLException e)
+      {
+         LOG.warn("Error trying to cancel query (statement)", e);
+      }
+   }
+
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java

@@ -27,8 +27,11 @@ import java.sql.Connection;
 import java.sql.ResultSet;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
@@ -53,6 +56,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
 {
    private static final QLogger LOG = QLogger.getLogger(RDBMSAggregateAction.class);
 
+   private ActionTimeoutHelper actionTimeoutHelper;
 
 
    /*******************************************************************************
@@ -102,8 +106,21 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
 
          try(Connection connection = getConnection(aggregateInput))
          {
-            QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) ->
+            statement = connection.prepareStatement(sql);
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            actionTimeoutHelper = new ActionTimeoutHelper(aggregateInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql));
+            actionTimeoutHelper.start();
+
+            QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
             {
+               /////////////////////////////////////////////////////////////////////////
+               // once we've started getting results, go ahead and cancel the timeout //
+               /////////////////////////////////////////////////////////////////////////
+               actionTimeoutHelper.cancel();
+
                while(resultSet.next())
                {
                   setQueryStatFirstResultTime();
@@ -156,9 +173,30 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
       }
       catch(Exception e)
       {
+         if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
+         {
+            setQueryStatFirstResultTime();
+            throw (new QUserFacingException("Aggregate query timed out."));
+         }
+
+         if(isCancelled)
+         {
+            throw (new QUserFacingException("Aggregate query was cancelled."));
+         }
+
          LOG.warn("Error executing aggregate", e);
          throw new QException("Error executing aggregate", e);
       }
+      finally
+      {
+         if(actionTimeoutHelper != null)
+         {
+            /////////////////////////////////////////
+            // make sure the timeout got cancelled //
+            /////////////////////////////////////////
+            actionTimeoutHelper.cancel();
+         }
+      }
    }
 
 
@@ -199,4 +237,15 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
       return (StringUtils.join(",", columns));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void cancelAction()
+   {
+      doCancelQuery();
+   }
+
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java

@@ -27,8 +27,11 @@ import java.sql.Connection;
 import java.sql.ResultSet;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
@@ -46,6 +49,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
 {
    private static final QLogger LOG = QLogger.getLogger(RDBMSCountAction.class);
 
+   private ActionTimeoutHelper actionTimeoutHelper;
+
 
 
    /*******************************************************************************
@@ -84,8 +89,21 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
          {
             long mark = System.currentTimeMillis();
 
-            QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) ->
+            statement = connection.prepareStatement(sql);
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            actionTimeoutHelper = new ActionTimeoutHelper(countInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql));
+            actionTimeoutHelper.start();
+
+            QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
             {
+               /////////////////////////////////////////////////////////////////////////
+               // once we've started getting results, go ahead and cancel the timeout //
+               /////////////////////////////////////////////////////////////////////////
+               actionTimeoutHelper.cancel();
+
                if(resultSet.next())
                {
                   rs.setCount(resultSet.getInt("record_count"));
@@ -107,9 +125,41 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
       }
       catch(Exception e)
       {
+         if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
+         {
+            setQueryStatFirstResultTime();
+            throw (new QUserFacingException("Count timed out."));
+         }
+
+         if(isCancelled)
+         {
+            throw (new QUserFacingException("Count was cancelled."));
+         }
+
          LOG.warn("Error executing count", e);
          throw new QException("Error executing count", e);
       }
+      finally
+      {
+         if(actionTimeoutHelper != null)
+         {
+            /////////////////////////////////////////
+            // make sure the timeout got cancelled //
+            /////////////////////////////////////////
+            actionTimeoutHelper.cancel();
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void cancelAction()
+   {
+      doCancelQuery();
    }
 
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java

@@ -83,7 +83,6 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
          boolean    needToCloseConnection = false;
          if(deleteInput.getTransaction() != null && deleteInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -96,7 +96,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
          boolean    needToCloseConnection = false;
          if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -33,9 +33,12 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ActionTimeoutHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -60,6 +63,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 {
    private static final QLogger LOG = QLogger.getLogger(RDBMSQueryAction.class);
 
+   private ActionTimeoutHelper actionTimeoutHelper;
+
 
 
    /*******************************************************************************
@@ -106,7 +111,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          boolean    needToCloseConnection = false;
          if(queryInput.getTransaction() != null && queryInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from queryInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else
@@ -136,14 +140,29 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
          try
          {
+            /////////////////////////////////////
+            // create a statement from the SQL //
+            /////////////////////////////////////
+            statement = createStatement(connection, sql.toString(), queryInput);
+
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // set up & start an actionTimeoutHelper (note, internally it'll deal with the time being null or negative as meaning not to timeout) //
+            ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            actionTimeoutHelper = new ActionTimeoutHelper(queryInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql));
+            actionTimeoutHelper.start();
+
             //////////////////////////////////////////////
             // execute the query - iterate over results //
             //////////////////////////////////////////////
             QueryOutput queryOutput = new QueryOutput(queryInput);
 
-            PreparedStatement statement = createStatement(connection, sql.toString(), queryInput);
             QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
             {
+               /////////////////////////////////////////////////////////////////////////
+               // once we've started getting results, go ahead and cancel the timeout //
+               /////////////////////////////////////////////////////////////////////////
+               actionTimeoutHelper.cancel();
+
                ResultSetMetaData metaData = resultSet.getMetaData();
                while(resultSet.next())
                {
@@ -201,6 +220,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          }
          finally
          {
+            if(actionTimeoutHelper != null)
+            {
+               /////////////////////////////////////////
+               // make sure the timeout got cancelled //
+               /////////////////////////////////////////
+               actionTimeoutHelper.cancel();
+            }
+
             if(needToCloseConnection)
             {
                connection.close();
@@ -209,6 +236,17 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
       }
       catch(Exception e)
       {
+         if(actionTimeoutHelper != null && actionTimeoutHelper.getDidTimeout())
+         {
+            setQueryStatFirstResultTime();
+            throw (new QUserFacingException("Query timed out."));
+         }
+
+         if(isCancelled)
+         {
+            throw (new QUserFacingException("Query was cancelled."));
+         }
+
          LOG.warn("Error executing query", e);
          throw new QException("Error executing query", e);
       }
@@ -282,20 +320,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private boolean filterOutHeavyFieldsIfNeeded(QFieldMetaData field, boolean shouldFetchHeavyFields)
-   {
-      if(!shouldFetchHeavyFields && field.getIsHeavy())
-      {
-         return (false);
-      }
-      return (true);
-   }
-
-
-
    /*******************************************************************************
     ** if we're not fetching heavy fields, instead just get their length.  this
     ** method wraps the field 'sql name' (e.g., column_name or table_name.column_name)
@@ -338,4 +362,14 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
       return (statement);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void cancelAction()
+   {
+      doCancelQuery();
+   }
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java

@@ -131,7 +131,6 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
          boolean    needToCloseConnection = false;
          if(updateInput.getTransaction() != null && updateInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/StatementTimeoutCanceller.java

@@ -0,0 +1,73 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.rdbms.actions;
+
+
+import java.sql.Statement;
+import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Helper to cancel statements that timeout.
+ *******************************************************************************/
+public class StatementTimeoutCanceller implements Runnable
+{
+   private static final QLogger LOG = QLogger.getLogger(StatementTimeoutCanceller.class);
+
+   private final Statement statement;
+   private final String    sql;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public StatementTimeoutCanceller(Statement statement, CharSequence sql)
+   {
+      this.statement = statement;
+      this.sql = sql.toString();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public void run()
+   {
+      try
+      {
+         statement.cancel();
+         LOG.info("Cancelled timed out statement", logPair("sql", sql));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error trying to cancel statement after timeout", e, logPair("sql", sql));
+      }
+
+      throw (new QRuntimeException("Statement timed out and was cancelled."));
+   }
+}

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java

@@ -66,6 +66,7 @@ public class TestUtils
    public static final String TABLE_NAME_PERSONAL_ID_CARD    = "personalIdCard";
    public static final String TABLE_NAME_STORE               = "store";
    public static final String TABLE_NAME_ORDER               = "order";
+   public static final String TABLE_NAME_ORDER_INSTRUCTIONS  = "orderInstructions";
    public static final String TABLE_NAME_ITEM                = "item";
    public static final String TABLE_NAME_ORDER_LINE          = "orderLine";
    public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic";
@@ -245,6 +246,16 @@ public class TestUtils
          .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
          .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
          .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
+         .withField(new QFieldMetaData("currentOrderInstructionsId", QFieldType.INTEGER).withBackendName("current_order_instructions_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_INSTRUCTIONS, "order_instructions")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName("order.storeId")
+            .withJoinNameChain(List.of("orderInstructionsJoinOrder")))
+         .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id"))
+         .withField(new QFieldMetaData("instructions", QFieldType.STRING))
       );
 
       qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item")
@@ -357,6 +368,22 @@ public class TestUtils
          .withJoinOn(new JoinOn("id", "orderLineId"))
       );
 
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinCurrentOrderInstructions")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS)
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("currentOrderInstructionsId", "id"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderInstructionsJoinOrder")
+         .withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS)
+         .withRightTable(TABLE_NAME_ORDER)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("orderId", "id"))
+      );
+
       qInstance.addPossibleValueSource(new QPossibleValueSource()
          .withName("store")
          .withType(QPossibleValueSourceType.TABLE)

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -32,10 +32,12 @@ import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -1695,4 +1697,51 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
 
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException
+   {
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+
+      {
+         /////////////////////////////////////////////////////////
+         // assert a failure if the join to use isn't specified //
+         /////////////////////////////////////////////////////////
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS));
+         assertThatThrownBy(() -> new QueryAction().execute(queryInput)).rootCause().hasMessageContaining("More than 1 join was found");
+      }
+
+      Integer noOfOrders            = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount();
+      Integer noOfOrderInstructions = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)).getCount();
+
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions")));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         assertEquals(noOfOrders, queryOutput.getRecords().size());
+      }
+
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure we can join on order.id = order_instruction.order_id -- and that we get back 1 row per order instruction //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderInstructionsJoinOrder")));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         assertEquals(noOfOrderInstructions, queryOutput.getRecords().size());
+      }
+
+   }
+
 }

qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql

@@ -84,6 +84,7 @@ DROP TABLE IF EXISTS line_item_extrinsic;
 DROP TABLE IF EXISTS order_line;
 DROP TABLE IF EXISTS item;
 DROP TABLE IF EXISTS `order`;
+DROP TABLE IF EXISTS order_instructions;
 DROP TABLE IF EXISTS warehouse_store_int;
 DROP TABLE IF EXISTS store;
 DROP TABLE IF EXISTS warehouse;
@@ -123,7 +124,8 @@ CREATE TABLE `order`
    id INT AUTO_INCREMENT PRIMARY KEY,
    store_id INT REFERENCES store,
    bill_to_person_id INT,
-   ship_to_person_id INT
+   ship_to_person_id INT,
+   current_order_instructions_id INT -- f-key to order_instructions, which also has an f-key back here!
 );
 
 -- variable orders
@@ -136,6 +138,27 @@ INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES
 INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (7, 3, null, 5);
 INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (8, 3, null, 5);
 
+CREATE TABLE order_instructions
+(
+   id INT AUTO_INCREMENT PRIMARY KEY,
+   order_id INT,
+   instructions VARCHAR(250)
+);
+
+-- give orders 1 & 2 multiple versions of the instruction record
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (1, 1, 'order 1 v1');
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (2, 1, 'order 1 v2');
+UPDATE `order` SET current_order_instructions_id = 2 WHERE id=1;
+
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (3, 2, 'order 2 v1');
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (4, 2, 'order 2 v2');
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (5, 2, 'order 2 v3');
+UPDATE `order` SET current_order_instructions_id = 5 WHERE id=2;
+
+-- give all other orders just 1 instruction
+INSERT INTO order_instructions (order_id, instructions) SELECT id, concat('order ', id, ' v1') FROM `order` WHERE current_order_instructions_id IS NULL;
+UPDATE `order` SET current_order_instructions_id = (SELECT MIN(id) FROM order_instructions WHERE order_id = `order`.id) WHERE current_order_instructions_id is null;
+
 CREATE TABLE order_line
 (
    id INT AUTO_INCREMENT PRIMARY KEY,

qqq-dev-tools/CURRENT-SNAPSHOT-VERSION

@@ -1 +1 @@
-0.16.0
+0.18.0

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java

@@ -36,9 +36,12 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.APIVersion;
+import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
 import com.kingsrook.qqq.api.model.actions.HttpApiResponse;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiOperation;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessCustomizers;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInput;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInputFieldsContainer;
@@ -104,6 +107,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -150,6 +154,7 @@ public class ApiImplementation
 
       QTableMetaData table     = validateTableAndVersion(apiInstanceMetaData, version, tableApiName, ApiOperation.QUERY_BY_QUERY_STRING);
       String         tableName = table.getName();
+      String         apiName   = apiInstanceMetaData.getName();
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
@@ -231,6 +236,8 @@ public class ApiImplementation
          badRequestMessages.add("includeCount must be either true or false");
       }
 
+      Map<String, QFieldMetaData> tableApiFields = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, version, tableName));
+
       if(StringUtils.hasContent(orderBy))
       {
          for(String orderByPart : orderBy.split(","))
@@ -238,6 +245,7 @@ public class ApiImplementation
             orderByPart = orderByPart.trim();
             String[] orderByNameDirection = orderByPart.split(" +");
             boolean  asc                  = true;
+            String   apiFieldName         = orderByNameDirection[0];
             if(orderByNameDirection.length == 2)
             {
                if("asc".equalsIgnoreCase(orderByNameDirection[1]))
@@ -250,7 +258,7 @@ public class ApiImplementation
                }
                else
                {
-                  badRequestMessages.add("orderBy direction for field " + orderByNameDirection[0] + " must be either ASC or DESC.");
+                  badRequestMessages.add("orderBy direction for field " + apiFieldName + " must be either ASC or DESC.");
                }
             }
             else if(orderByNameDirection.length > 2)
@@ -258,14 +266,27 @@ public class ApiImplementation
                badRequestMessages.add("Unrecognized format for orderBy clause: " + orderByPart + ".  Expected:  fieldName [ASC|DESC].");
             }
 
-            try
+            QFieldMetaData field = tableApiFields.get(apiFieldName);
+            if(field == null)
             {
-               QFieldMetaData field = table.getField(orderByNameDirection[0]);
-               filter.withOrderBy(new QFilterOrderBy(field.getName(), asc));
+               badRequestMessages.add("Unrecognized orderBy field name: " + apiFieldName + ".");
             }
-            catch(Exception e)
+            else
             {
-               badRequestMessages.add("Unrecognized orderBy field name: " + orderByNameDirection[0] + ".");
+               QFilterOrderBy filterOrderBy = new QFilterOrderBy(field.getName(), asc);
+
+               ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
+               if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
+               {
+                  filterOrderBy.setFieldName(apiFieldMetaData.getReplacedByFieldName());
+               }
+               else if(apiFieldMetaData.getCustomValueMapper() != null)
+               {
+                  ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
+                  customValueMapper.customizeFilterOrderBy(queryInput, filterOrderBy, apiFieldName, apiFieldMetaData);
+               }
+
+               filter.withOrderBy(filterOrderBy);
             }
          }
       }
@@ -289,20 +310,36 @@ public class ApiImplementation
             continue;
          }
 
-         try
+         QFieldMetaData field = tableApiFields.get(name);
+         if(field == null)
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-            // todo - deal with removed fields; fields w/ custom value mappers (need new method(s) there) //
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-
-            QFieldMetaData field = table.getField(name);
+            badRequestMessages.add("Unrecognized filter criteria field: " + name);
+         }
+         else
+         {
+            ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
             for(String value : values)
             {
                if(StringUtils.hasContent(value))
                {
                   try
                   {
-                     filter.addCriteria(parseQueryParamToCriteria(field, name, value));
+                     QFilterCriteria criteria = parseQueryParamToCriteria(field, name, value);
+
+                     /////////////////////////////////////////////
+                     // deal with replaced or customized fields //
+                     /////////////////////////////////////////////
+                     if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
+                     {
+                        criteria.setFieldName(apiFieldMetaData.getReplacedByFieldName());
+                     }
+                     else if(apiFieldMetaData.getCustomValueMapper() != null)
+                     {
+                        ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
+                        customValueMapper.customizeFilterCriteria(queryInput, filter, criteria, name, apiFieldMetaData);
+                     }
+
+                     filter.addCriteria(criteria);
                   }
                   catch(Exception e)
                   {
@@ -311,10 +348,6 @@ public class ApiImplementation
                }
             }
          }
-         catch(Exception e)
-         {
-            badRequestMessages.add("Unrecognized filter criteria field: " + name);
-         }
       }
 
       //////////////////////////////////////////
@@ -350,7 +383,7 @@ public class ApiImplementation
       ArrayList<Map<String, Serializable>> records = new ArrayList<>();
       for(QRecord record : queryOutput.getRecords())
       {
-         records.add(QRecordApiAdapter.qRecordToApiMap(record, tableName, apiInstanceMetaData.getName(), version));
+         records.add(QRecordApiAdapter.qRecordToApiMap(record, tableName, apiName, version));
       }
 
       /////////////////////////////
@@ -986,9 +1019,16 @@ public class ApiImplementation
          {
             String[] ids = paramMap.get(idParam).split(",");
 
-            QTableMetaData table  = QContext.getQInstance().getTable(process.getTableName());
-            QQueryFilter   filter = new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), IN, Arrays.asList(ids)));
-            runProcessInput.setCallback(getCallback(filter));
+            if(StringUtils.hasContent(process.getTableName()))
+            {
+               QTableMetaData table  = QContext.getQInstance().getTable(process.getTableName());
+               QQueryFilter   filter = new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), IN, Arrays.asList(ids)));
+               runProcessInput.setCallback(getProcessCallback(filter));
+            }
+            else
+            {
+               runProcessInput.addValue(idParam, paramMap.get(idParam));
+            }
          }
       }
 
@@ -1517,7 +1557,7 @@ public class ApiImplementation
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QProcessCallback getCallback(QQueryFilter filter)
+   public static QProcessCallback getProcessCallback(QQueryFilter filter)
    {
       return new QProcessCallback()
       {

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -485,7 +485,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
                   .withDescription("How the results of the query should be sorted. SQL-style, comma-separated list of field names, each optionally followed by ASC or DESC (defaults to ASC).")
                   .withIn("query")
                   .withSchema(new Schema().withType("string"))
-                  .withExamples(buildOrderByExamples(primaryKeyApiName, tableApiFields)),
+                  .withExamples(buildOrderByExamples(apiName, primaryKeyApiName, tableApiFields)),
                new Parameter()
                   .withName("booleanOperator")
                   .withDescription("Whether to combine query field as an AND or an OR.  Default is AND.")
@@ -500,10 +500,12 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
          for(QFieldMetaData tableApiField : tableApiFields)
          {
-            String label = tableApiField.getLabel();
+            String fieldName = ApiFieldMetaData.getEffectiveApiFieldName(apiName, tableApiField);
+            String label     = tableApiField.getLabel();
+
             if(!StringUtils.hasContent(label))
             {
-               label = QInstanceEnricher.nameToLabel(tableApiField.getName());
+               label = QInstanceEnricher.nameToLabel(fieldName);
             }
 
             StringBuilder description = new StringBuilder("Query on the " + label + " field.  ");
@@ -517,7 +519,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
             }
 
             queryGet.getParameters().add(new Parameter()
-               .withName(tableApiField.getName())
+               .withName(fieldName)
                .withDescription(description.toString())
                .withIn("query")
                .withExplode(true)
@@ -892,6 +894,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       ////////////////////////////////
       List<Parameter> parameters      = new ArrayList<>();
       ApiProcessInput apiProcessInput = apiProcessMetaData.getInput();
+      String          apiName         = apiInstanceMetaData.getName();
       if(apiProcessInput != null)
       {
          ApiProcessInputFieldsContainer queryStringParams = apiProcessInput.getQueryStringParams();
@@ -912,12 +915,13 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
          if(bodyField != null)
          {
             ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(bodyField);
-            ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiInstanceMetaData.getName());
+            ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiName);
 
             String fieldLabel = bodyField.getLabel();
+            String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiName, bodyField);
             if(!StringUtils.hasContent(fieldLabel))
             {
-               fieldLabel = QInstanceEnricher.nameToLabel(bodyField.getName());
+               fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
             }
 
             String bodyDescription = "Value for the " + fieldLabel;
@@ -979,7 +983,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       ApiProcessOutputInterface output = apiProcessMetaData.getOutput();
       if(!ApiProcessMetaData.AsyncMode.ALWAYS.equals(apiProcessMetaData.getAsyncMode()))
       {
-         responses.putAll(output.getSpecResponses(apiInstanceMetaData.getName()));
+         responses.putAll(output.getSpecResponses(apiName));
       }
       if(!ApiProcessMetaData.AsyncMode.NEVER.equals(apiProcessMetaData.getAsyncMode()))
       {
@@ -1074,13 +1078,16 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
     *******************************************************************************/
    private Parameter processFieldToParameter(ApiInstanceMetaData apiInstanceMetaData, QFieldMetaData field)
    {
-      ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(field);
-      ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiInstanceMetaData.getName());
+      String apiName = apiInstanceMetaData.getName();
 
+      ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(field);
+      ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiName);
+
+      String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiName, field);
       String fieldLabel = field.getLabel();
       if(!StringUtils.hasContent(fieldLabel))
       {
-         fieldLabel = QInstanceEnricher.nameToLabel(field.getName());
+         fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
       }
 
       String description = "Value for the " + fieldLabel + " field.";
@@ -1097,7 +1104,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       Schema fieldSchema = getFieldSchema(field, description, apiInstanceMetaData);
 
       Parameter parameter = new Parameter()
-         .withName(field.getName())
+         .withName(fieldName)
          .withDescription(description)
          .withRequired(field.getIsRequired())
          .withSchema(fieldSchema);
@@ -1213,14 +1220,15 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       for(QFieldMetaData field : tableApiFields)
       {
          String fieldLabel = field.getLabel();
+         String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiInstanceMetaData.getName(), field);
          if(!StringUtils.hasContent(fieldLabel))
          {
-            fieldLabel = QInstanceEnricher.nameToLabel(field.getName());
+            fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
          }
 
          String defaultDescription = fieldLabel + " for the " + table.getLabel() + ".";
          Schema fieldSchema        = getFieldSchema(field, defaultDescription, apiInstanceMetaData);
-         tableFields.put(ApiFieldMetaData.getEffectiveApiFieldName(apiInstanceMetaData.getName(), field), fieldSchema);
+         tableFields.put(fieldName, fieldSchema);
       }
 
       //////////////////////////////////
@@ -1561,7 +1569,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Example> buildOrderByExamples(String primaryKeyApiName, List<? extends QFieldMetaData> tableApiFields)
+   private Map<String, Example> buildOrderByExamples(String apiName, String primaryKeyApiName, List<? extends QFieldMetaData> tableApiFields)
    {
       Map<String, Example> rs = new LinkedHashMap<>();
 
@@ -1569,7 +1577,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       List<String> fieldsForExample5 = new ArrayList<>();
       for(QFieldMetaData tableApiField : tableApiFields)
       {
-         String name = tableApiField.getName();
+         String name = ApiFieldMetaData.getEffectiveApiFieldName(apiName, tableApiField);
          if(primaryKeyApiName.equals(name) || fieldsForExample4.contains(name) || fieldsForExample5.contains(name))
          {
             continue;

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GetTableApiFieldsAction.java

@@ -24,7 +24,10 @@ package com.kingsrook.qqq.api.actions;
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
@@ -50,6 +53,65 @@ import org.apache.commons.lang.BooleanUtils;
  *******************************************************************************/
 public class GetTableApiFieldsAction extends AbstractQActionFunction<GetTableApiFieldsInput, GetTableApiFieldsOutput>
 {
+   private static Map<ApiNameVersionAndTableName, List<QFieldMetaData>>        fieldListCache = new HashMap<>();
+   private static Map<ApiNameVersionAndTableName, Map<String, QFieldMetaData>> fieldMapCache  = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Allow tests (that manipulate meta-data) to clear field caches.
+    *******************************************************************************/
+   public static void clearCaches()
+   {
+      fieldListCache.clear();
+      fieldMapCache.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** convenience (and caching) wrapper
+    *******************************************************************************/
+   public static Map<String, QFieldMetaData> getTableApiFieldMap(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
+   {
+      if(!fieldMapCache.containsKey(apiNameVersionAndTableName))
+      {
+         Map<String, QFieldMetaData> map = getTableApiFieldList(apiNameVersionAndTableName).stream().collect(Collectors.toMap(f -> (ApiFieldMetaData.getEffectiveApiFieldName(apiNameVersionAndTableName.apiName(), f)), f -> f));
+         fieldMapCache.put(apiNameVersionAndTableName, map);
+      }
+
+      return (fieldMapCache.get(apiNameVersionAndTableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** convenience (and caching) wrapper
+    *******************************************************************************/
+   public static List<QFieldMetaData> getTableApiFieldList(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
+   {
+      if(!fieldListCache.containsKey(apiNameVersionAndTableName))
+      {
+         List<QFieldMetaData> value = new GetTableApiFieldsAction().execute(new GetTableApiFieldsInput()
+            .withTableName(apiNameVersionAndTableName.tableName())
+            .withVersion(apiNameVersionAndTableName.apiVersion())
+            .withApiName(apiNameVersionAndTableName.apiName())).getFields();
+         fieldListCache.put(apiNameVersionAndTableName, value);
+      }
+      return (fieldListCache.get(apiNameVersionAndTableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Input-record for convenience methods
+    *******************************************************************************/
+   public record ApiNameVersionAndTableName(String apiName, String apiVersion, String tableName)
+   {
+
+   }
+
+
 
    /*******************************************************************************
     **

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java

@@ -29,12 +29,10 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
-import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
@@ -66,20 +64,6 @@ public class QRecordApiAdapter
 {
    private static final QLogger LOG = QLogger.getLogger(QRecordApiAdapter.class);
 
-   private static Map<ApiNameVersionAndTableName, List<QFieldMetaData>>        fieldListCache = new HashMap<>();
-   private static Map<ApiNameVersionAndTableName, Map<String, QFieldMetaData>> fieldMapCache  = new HashMap<>();
-
-
-
-   /*******************************************************************************
-    ** Allow tests (that manipulate meta-data) to clear field caches.
-    *******************************************************************************/
-   public static void clearCaches()
-   {
-      fieldListCache.clear();
-      fieldMapCache.clear();
-   }
-
 
 
    /*******************************************************************************
@@ -92,7 +76,7 @@ public class QRecordApiAdapter
          return (null);
       }
 
-      List<QFieldMetaData>                tableApiFields = getTableApiFieldList(new ApiNameVersionAndTableName(apiName, apiVersion, tableName));
+      List<QFieldMetaData>                tableApiFields = GetTableApiFieldsAction.getTableApiFieldList(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, apiVersion, tableName));
       LinkedHashMap<String, Serializable> outputRecord   = new LinkedHashMap<>();
 
       /////////////////////////////////////////
@@ -111,7 +95,7 @@ public class QRecordApiAdapter
          else if(apiFieldMetaData.getCustomValueMapper() != null)
          {
             ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
-            value = customValueMapper.produceApiValue(record);
+            value = customValueMapper.produceApiValue(record, apiFieldName);
          }
          else
          {
@@ -157,7 +141,7 @@ public class QRecordApiAdapter
     *******************************************************************************/
    private static boolean isAssociationOmitted(String apiName, String apiVersion, QTableMetaData table, Association association)
    {
-      ApiTableMetaData       thisApiTableMetaData   = ObjectUtils.tryElse(() -> ApiTableMetaDataContainer.of(table).getApiTableMetaData(apiName), new ApiTableMetaData());
+      ApiTableMetaData       thisApiTableMetaData   = ObjectUtils.tryAndRequireNonNullElse(() -> ApiTableMetaDataContainer.of(table).getApiTableMetaData(apiName), new ApiTableMetaData());
       ApiAssociationMetaData apiAssociationMetaData = thisApiTableMetaData.getApiAssociationMetaData().get(association.getName());
       if(apiAssociationMetaData != null)
       {
@@ -185,7 +169,7 @@ public class QRecordApiAdapter
       ////////////////////////////////////////////////////////////////////////////////
       // make map of apiFieldNames (e.g., names as api uses them) to QFieldMetaData //
       ////////////////////////////////////////////////////////////////////////////////
-      Map<String, QFieldMetaData> apiFieldsMap           = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, apiVersion, tableName));
+      Map<String, QFieldMetaData> apiFieldsMap           = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, apiVersion, tableName));
       List<String>                unrecognizedFieldNames = new ArrayList<>();
       QRecord                     qRecord                = new QRecord();
 
@@ -241,7 +225,7 @@ public class QRecordApiAdapter
             else if(apiFieldMetaData.getCustomValueMapper() != null)
             {
                ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
-               customValueMapper.consumeApiValue(qRecord, value, jsonObject);
+               customValueMapper.consumeApiValue(qRecord, value, jsonObject, jsonKey);
             }
             else
             {
@@ -332,7 +316,7 @@ public class QRecordApiAdapter
       {
          if(!supportedVersion.toString().equals(apiVersion))
          {
-            Map<String, QFieldMetaData> versionFields = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName));
+            Map<String, QFieldMetaData> versionFields = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName));
             if(versionFields.containsKey(unrecognizedFieldName))
             {
                versionsWithThisField.add(supportedVersion.toString());
@@ -348,47 +332,4 @@ public class QRecordApiAdapter
       return (null);
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, QFieldMetaData> getTableApiFieldMap(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
-   {
-      if(!fieldMapCache.containsKey(apiNameVersionAndTableName))
-      {
-         Map<String, QFieldMetaData> map = getTableApiFieldList(apiNameVersionAndTableName).stream().collect(Collectors.toMap(f -> (ApiFieldMetaData.getEffectiveApiFieldName(apiNameVersionAndTableName.apiName(), f)), f -> f));
-         fieldMapCache.put(apiNameVersionAndTableName, map);
-      }
-
-      return (fieldMapCache.get(apiNameVersionAndTableName));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QFieldMetaData> getTableApiFieldList(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
-   {
-      if(!fieldListCache.containsKey(apiNameVersionAndTableName))
-      {
-         List<QFieldMetaData> value = new GetTableApiFieldsAction().execute(new GetTableApiFieldsInput()
-            .withTableName(apiNameVersionAndTableName.tableName())
-            .withVersion(apiNameVersionAndTableName.apiVersion())
-            .withApiName(apiNameVersionAndTableName.apiName())).getFields();
-         fieldListCache.put(apiNameVersionAndTableName, value);
-      }
-      return (fieldListCache.get(apiNameVersionAndTableName));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private record ApiNameVersionAndTableName(String apiName, String apiVersion, String tableName)
-   {
-
-   }
 }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/ApiFieldCustomValueMapper.java

@@ -23,6 +23,11 @@ package com.kingsrook.qqq.api.model.actions;
 
 
 import java.io.Serializable;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import org.json.JSONObject;
 
@@ -34,9 +39,11 @@ public abstract class ApiFieldCustomValueMapper
 {
 
    /*******************************************************************************
-    **
+    ** When producing a JSON Object to send over the API (e.g., for a GET), this method
+    ** can run to customize the value that is produced, for the input QRecord's specified
+    ** fieldName
     *******************************************************************************/
-   public Serializable produceApiValue(QRecord record)
+   public Serializable produceApiValue(QRecord record, String apiFieldName)
    {
       /////////////////////
       // null by default //
@@ -46,10 +53,36 @@ public abstract class ApiFieldCustomValueMapper
 
 
 
+   /*******************************************************************************
+    ** When producing a QRecord (the first parameter) from a JSON Object that was
+    ** received from the API (e.g., a POST or PATCH) - this method can run to
+    ** allow customization of the incoming value.
+    *******************************************************************************/
+   public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject, String apiFieldName)
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject)
+   public void customizeFilterCriteria(QueryInput queryInput, QQueryFilter filter, QFilterCriteria criteria, String apiFieldName, ApiFieldMetaData apiFieldMetaData)
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void customizeFilterOrderBy(QueryInput queryInput, QFilterOrderBy orderBy, String apiFieldName, ApiFieldMetaData apiFieldMetaData)
    {
       /////////////////////
       // noop by default //

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaData.java

@@ -125,7 +125,10 @@ public class ApiInstanceMetaData implements ApiOperation.EnabledOperationsProvid
             {
                if(BooleanUtils.isNotTrue(apiTableMetaData.getIsExcluded()))
                {
-                  validator.assertCondition(allVersions.contains(new APIVersion(apiTableMetaData.getInitialVersion())), "Table " + table.getName() + "'s initial API version is not a recognized version for api " + apiName);
+                  if(StringUtils.hasContent(apiTableMetaData.getInitialVersion()))
+                  {
+                     validator.assertCondition(allVersions.contains(new APIVersion(apiTableMetaData.getInitialVersion())), "Table " + table.getName() + "'s initial API version is not a recognized version for api " + apiName);
+                  }
                }
             }
          }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaDataProvider.java

@@ -45,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
@@ -62,7 +63,8 @@ public class ApiInstanceMetaDataProvider
     *******************************************************************************/
    public static void defineAll(QInstance qInstance, String backendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      definePossibleValueSources(qInstance);
+      definePossibleValueSourcesUsedByApiLogTable(qInstance);
+      definePossibleValueSourcesForApiNameAndVersion(qInstance);
       defineAPILogTable(qInstance, backendName, backendDetailEnricher);
       defineAPILogUserTable(qInstance, backendName, backendDetailEnricher);
    }
@@ -72,7 +74,7 @@ public class ApiInstanceMetaDataProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static void definePossibleValueSources(QInstance instance)
+   public static void definePossibleValueSourcesUsedByApiLogTable(QInstance instance)
    {
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_API_LOG_USER)
@@ -104,7 +106,15 @@ public class ApiInstanceMetaDataProvider
             new QPossibleValue<>(429, "429 (Too Many Requests)"),
             new QPossibleValue<>(500, "500 (Internal Server Error)")
          )));
+   }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void definePossibleValueSourcesForApiNameAndVersion(QInstance instance)
+   {
       ////////////////////////////////////////////////////////////////////////////
       // loop over api names and versions, building out possible values sources //
       ////////////////////////////////////////////////////////////////////////////
@@ -121,14 +131,14 @@ public class ApiInstanceMetaDataProvider
          apiNamePossibleValues.add(new QPossibleValue<>(entry.getKey(), entry.getValue().getLabel()));
 
          ApiInstanceMetaData apiInstanceMetaData = entry.getValue();
-         allVersions.addAll(apiInstanceMetaData.getPastVersions());
-         allVersions.addAll(apiInstanceMetaData.getSupportedVersions());
+         allVersions.addAll(CollectionUtils.nonNullCollection(apiInstanceMetaData.getPastVersions()));
+         allVersions.addAll(CollectionUtils.nonNullCollection(apiInstanceMetaData.getSupportedVersions()));
 
          ///////////////////////////////////////////////////////////////////////////////////////////////////////
          // I think we don't want future-versions in this dropdown, I think...                                //
          // grr, actually todo maybe we want this to be a table-backed enum, with past/present/future columns //
          ///////////////////////////////////////////////////////////////////////////////////////////////////////
-         allVersions.addAll(apiInstanceMetaData.getFutureVersions());
+         allVersions.addAll(CollectionUtils.nonNullCollection(apiInstanceMetaData.getFutureVersions()));
       }
 
       instance.addPossibleValueSource(new QPossibleValueSource()

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/tables/ApiTableMetaData.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.api.model.metadata.ApiOperation;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -80,7 +81,7 @@ public class ApiTableMetaData implements ApiOperation.EnabledOperationsProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void enrich(String apiName, QTableMetaData table)
+   public void enrich(QInstance qInstance, String apiName, QTableMetaData table)
    {
       if(initialVersion != null)
       {
@@ -95,7 +96,7 @@ public class ApiTableMetaData implements ApiOperation.EnabledOperationsProvider
 
          for(QFieldMetaData field : CollectionUtils.nonNullList(removedApiFields))
          {
-            new QInstanceEnricher(null).enrichField(field);
+            new QInstanceEnricher(qInstance).enrichField(field);
             ApiFieldMetaData apiFieldMetaData = ensureFieldHasApiSupplementalMetaData(apiName, field);
             if(apiFieldMetaData.getInitialVersion() == null)
             {

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/tables/ApiTableMetaDataContainer.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.api.model.metadata.tables;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import com.kingsrook.qqq.api.ApiSupplementType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -80,13 +81,13 @@ public class ApiTableMetaDataContainer extends QSupplementalTableMetaData
     **
     *******************************************************************************/
    @Override
-   public void enrich(QTableMetaData table)
+   public void enrich(QInstance qInstance, QTableMetaData table)
    {
-      super.enrich(table);
+      super.enrich(qInstance, table);
 
       for(Map.Entry<String, ApiTableMetaData> entry : CollectionUtils.nonNullMap(apis).entrySet())
       {
-         entry.getValue().enrich(entry.getKey(), table);
+         entry.getValue().enrich(qInstance, entry.getKey(), table);
       }
    }
 

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/ApiImplementationTest.java

@@ -23,9 +23,14 @@ package com.kingsrook.qqq.api.actions;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.Month;
+import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.api.BaseTest;
 import com.kingsrook.qqq.api.TestUtils;
+import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
@@ -35,19 +40,23 @@ import com.kingsrook.qqq.api.model.metadata.tables.ApiAssociationMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaDataContainer;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.json.JSONObject;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -66,7 +75,7 @@ class ApiImplementationTest extends BaseTest
    @AfterEach
    void beforeAndAfterEach()
    {
-      QRecordApiAdapter.clearCaches();
+      GetTableApiFieldsAction.clearCaches();
    }
 
 
@@ -188,6 +197,43 @@ class ApiImplementationTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testQueryWithRemovedFields() throws QException
+   {
+      QInstance           qInstance           = QContext.getQInstance();
+      ApiInstanceMetaData apiInstanceMetaData = ApiInstanceMetaDataContainer.of(qInstance).getApiInstanceMetaData(TestUtils.API_NAME);
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecord(new QRecord()
+         .withValue("firstName", "Tim")
+         .withValue("noOfShoes", 2)
+         .withValue("birthDate", LocalDate.of(1980, Month.MAY, 31))
+         .withValue("cost", new BigDecimal("3.50"))
+         .withValue("price", new BigDecimal("9.99"))
+         .withValue("photo", "ABCD".getBytes())));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // query by a field that wasn't in an old api version, but is in the table now - should fail //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+
+      assertThatThrownBy(() ->
+         ApiImplementation.query(apiInstanceMetaData, TestUtils.V2022_Q4, TestUtils.TABLE_NAME_PERSON, MapBuilder.of("noOfShoes", List.of("2"))))
+         .isInstanceOf(QBadRequestException.class)
+         .hasMessageContaining("Unrecognized filter criteria field");
+
+      {
+         /////////////////////////////////////////////
+         // query by a removed field (was replaced) //
+         /////////////////////////////////////////////
+         Map<String, Serializable> queryResult = ApiImplementation.query(apiInstanceMetaData, TestUtils.V2022_Q4, TestUtils.TABLE_NAME_PERSON, MapBuilder.of("shoeCount", List.of("2")));
+         assertEquals(1, queryResult.get("count"));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -198,7 +244,7 @@ class ApiImplementationTest extends BaseTest
        **
        *******************************************************************************/
       @Override
-      public Serializable produceApiValue(QRecord record)
+      public Serializable produceApiValue(QRecord record, String apiFieldName)
       {
          return ("customValue-" + record.getValueString("lastName"));
       }
@@ -209,7 +255,7 @@ class ApiImplementationTest extends BaseTest
        **
        *******************************************************************************/
       @Override
-      public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject)
+      public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject, String apiFieldName)
       {
          String valueString = ValueUtils.getValueAsString(value);
          valueString = valueString.replaceFirst("^stripThisAway-", "");

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -162,6 +162,9 @@ public class QJavalinImplementation
    private static final long MILLIS_BETWEEN_HOT_SWAPS = 2500;
    public static final  long SLOW_LOG_THRESHOLD_MS    = 1000;
 
+   private static final Integer DEFAULT_COUNT_TIMEOUT_SECONDS = 60;
+   private static final Integer DEFAULT_QUERY_TIMEOUT_SECONDS = 60;
+
    private static int DEFAULT_PORT = 8001;
 
    private static Javalin service;
@@ -868,6 +871,8 @@ public class QJavalinImplementation
          getInput.setShouldTranslatePossibleValues(true);
          getInput.setShouldFetchHeavyFields(true);
 
+         getInput.setQueryJoins(processQueryJoinsParam(context));
+
          if("true".equals(context.queryParam("includeAssociations")))
          {
             getInput.setIncludeAssociations(true);
@@ -1075,6 +1080,7 @@ public class QJavalinImplementation
             countInput.setFilter(JsonUtils.toObject(filter, QQueryFilter.class));
          }
 
+         countInput.setTimeoutSeconds(DEFAULT_COUNT_TIMEOUT_SECONDS);
          countInput.setQueryJoins(processQueryJoinsParam(context));
          countInput.setIncludeDistinctCount(QJavalinUtils.queryParamIsTrue(context, "includeDistinct"));
 
@@ -1131,6 +1137,7 @@ public class QJavalinImplementation
          queryInput.setTableName(table);
          queryInput.setShouldGenerateDisplayValues(true);
          queryInput.setShouldTranslatePossibleValues(true);
+         queryInput.setTimeoutSeconds(DEFAULT_QUERY_TIMEOUT_SECONDS);
 
          PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ);