Merge pull request #40 from Kingsrook/dev

refreshign CE-609 with dev

.circleci/adjust-pom-version.sh

@@ -5,8 +5,8 @@ if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then
    exit 1;
 fi
 
-if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ]; then
-   echo "On a primary branch [$CIRCLE_BRANCH] - will not edit the pom version.";
+if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ] || [ \! -z $(echo "$CIRCLE_TAG" | grep "^version-") ]; then
+   echo "On a primary branch or tag [${CIRCLE_BRANCH}${CIRCLE_TAG}] - will not edit the pom version.";
    exit 0;
 fi
 

pom.xml

@@ -44,7 +44,7 @@
    </modules>
 
    <properties>
-      <revision>0.18.0-SNAPSHOT</revision>
+      <revision>0.19.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/AuditAction.java

@@ -44,9 +44,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 
@@ -167,7 +167,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                ///////////////////////////////////////////////////
                // validate security keys on the table are given //
                ///////////////////////////////////////////////////
-               for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+               for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
                {
                   if(auditSingleInput.getSecurityKeyValues() == null || !auditSingleInput.getSecurityKeyValues().containsKey(recordSecurityLock.getSecurityKeyType()))
                   {
@@ -178,6 +178,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                ////////////////////////////////////////////////
                // map names to ids and handle default values //
                ////////////////////////////////////////////////
+               Integer auditTableId = getIdForName("auditTable", auditSingleInput.getAuditTableName());
                Integer auditUserId  = getIdForName("auditUser", Objects.requireNonNullElse(auditSingleInput.getAuditUserName(), getSessionUserName()));
                Instant timestamp    = Objects.requireNonNullElse(auditSingleInput.getTimestamp(), Instant.now());
 
@@ -185,7 +186,7 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
                // build record //
                //////////////////
                QRecord record = new QRecord()
-                  .withValue("tableId", QQQTableAccessor.getTableId(auditSingleInput.getAuditTableName()))
+                  .withValue("auditTableId", auditTableId)
                   .withValue("auditUserId", auditUserId)
                   .withValue("timestamp", timestamp)
                   .withValue("message", auditSingleInput.getMessage())
@@ -287,6 +288,15 @@ public class AuditAction extends AbstractQActionFunction<AuditInput, AuditOutput
             insertInput.setTableName(tableName);
             QRecord record = new QRecord().withValue("name", nameValue);
 
+            if(tableName.equals("auditTable"))
+            {
+               QTableMetaData table = QContext.getQInstance().getTable(nameValue);
+               if(table != null)
+               {
+                  record.setValue("label", table.getLabel());
+               }
+            }
+
             insertInput.setRecords(List.of(record));
             InsertOutput insertOutput = new InsertAction().execute(insertInput);
             id = insertOutput.getRecords().get(0).getValueInteger("id");

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java

@@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -378,7 +379,7 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
    private static Map<String, Serializable> getRecordSecurityKeyValues(QTableMetaData table, QRecord record)
    {
       Map<String, Serializable> securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
          securityKeyValues.put(recordSecurityLock.getSecurityKeyType(), record == null ? null : record.getValue(recordSecurityLock.getFieldName()));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/RecordAutomationStatusUpdater.java

@@ -45,7 +45,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAut
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import org.apache.commons.lang.NotImplementedException;
 
@@ -172,7 +171,7 @@ public class RecordAutomationStatusUpdater
          CountInput countInput = new CountInput();
          countInput.setTableName(TableTrigger.TABLE_NAME);
          countInput.setFilter(new QQueryFilter(
-            new QFilterCriteria("tableId", QCriteriaOperator.EQUALS, QQQTableAccessor.getTableId(table.getName())),
+            new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, table.getName()),
             new QFilterCriteria(triggerEvent.equals(TriggerEvent.POST_INSERT) ? "postInsert" : "postUpdate", QCriteriaOperator.EQUALS, true)
          ));
          CountOutput countOutput = new CountAction().execute(countInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/automation/polling/PollingAutomationPerTableRunner.java

@@ -66,7 +66,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAuto
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TriggerEvent;
 import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -271,7 +270,7 @@ public class PollingAutomationPerTableRunner implements Runnable
          QueryInput queryInput = new QueryInput();
          queryInput.setTableName(TableTrigger.TABLE_NAME);
          queryInput.setFilter(new QQueryFilter(
-            new QFilterCriteria("tableId", QCriteriaOperator.EQUALS, QQQTableAccessor.getTableId(table.getName())),
+            new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, table.getName()),
             new QFilterCriteria(triggerEvent.equals(TriggerEvent.POST_INSERT) ? "postInsert" : "postUpdate", QCriteriaOperator.EQUALS, true)
          ));
          QueryOutput queryOutput = new QueryAction().execute(queryInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -197,7 +197,27 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
+                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
+                  queryJoins.add(queryJoin);
+
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
+                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
+                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  QTableMetaData        table               = exportInput.getTable();
+                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
+                  if(exposedJoinOptional.isEmpty())
+                  {
+                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
+                  }
+                  ExposedJoin exposedJoin = exposedJoinOptional.get();
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
+                  }
+
                   addedJoinNames.add(joinTableName);
                }
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLogg
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -112,6 +114,11 @@ public class ExecuteCodeAction
             context.putAll(input.getInput());
          }
 
+         //////////////////////////////////////////
+         // safely always set the deploymentMode //
+         //////////////////////////////////////////
+         context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
+
          /////////////////////////////////////////////////////////////////////////////////
          // set the qCodeExecutor into any context objects which are QCodeExecutorAware //
          /////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/RecordScriptTestInterface.java

@@ -47,7 +47,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -84,7 +83,7 @@ public class RecordScriptTestInterface implements TestScriptActionInterface
          //////////////////////////////////////////////
          // look up the records being tested against //
          //////////////////////////////////////////////
-         String         tableName = QQQTableAccessor.getTableName(script.getValueInteger("tableId"));
+         String         tableName = script.getValueString("tableName");
          QTableMetaData table     = QContext.getQInstance().getTable(tableName);
          if(table == null)
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreDeleteCusto
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.LogPair;
@@ -321,6 +322,8 @@ public class DeleteAction
       QTableMetaData table               = deleteInput.getTable();
       List<QRecord>  primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get());
 
+      ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE);
+
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-delete customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -61,6 +62,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, ReplaceOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(ReplaceAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -159,6 +163,7 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
       {
          if(weOwnTheTransaction)
          {
+            LOG.warn("Caught top-level ReplaceAction exception - rolling back exception", e);
             transaction.rollback();
          }
          throw (new QException("Error executing replace action", e));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -61,6 +61,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
@@ -209,14 +211,16 @@ public class UpdateAction
       {
          validateRecordsExistAndCanBeAccessed(updateInput, oldRecordList.get());
       }
+      else
+      {
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+      }
 
       if(updateInput.getInputSource().shouldValidateRequiredFields())
       {
          validateRequiredFields(updateInput);
       }
 
-      ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
-
       ///////////////////////////////////////////////////////////////////////////
       // after all validations, run the pre-update customizer, if there is one //
       ///////////////////////////////////////////////////////////////////////////
@@ -287,6 +291,8 @@ public class UpdateAction
       QTableMetaData table           = updateInput.getTable();
       QFieldMetaData primaryKeyField = table.getField(table.getPrimaryKeyField());
 
+      List<RecordSecurityLock> onlyWriteLocks = RecordSecurityLockFilters.filterForOnlyWriteLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks()));
+
       for(List<QRecord> page : CollectionUtils.getPages(updateInput.getRecords(), 1000))
       {
          List<Serializable> primaryKeysToLookup = new ArrayList<>();
@@ -320,6 +326,8 @@ public class UpdateAction
             }
          }
 
+         ValidateRecordSecurityLockHelper.validateSecurityFields(table, updateInput.getRecords(), ValidateRecordSecurityLockHelper.Action.UPDATE);
+
          for(QRecord record : page)
          {
             Serializable value = ValueUtils.getValueAsFieldType(primaryKeyField.getType(), record.getValue(table.getPrimaryKeyField()));
@@ -332,6 +340,19 @@ public class UpdateAction
             {
                record.addError(new NotFoundStatusMessage("No record was found to update for " + primaryKeyField.getLabel() + " = " + value));
             }
+            else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////////
+               // if the table has any write-only locks, validate their values here, on the old-records //
+               ///////////////////////////////////////////////////////////////////////////////////////////
+               for(RecordSecurityLock lock : onlyWriteLocks)
+               {
+                  QRecord      oldRecord = lookedUpRecords.get(value);
+                  QFieldType   fieldType = table.getField(lock.getFieldName()).getType();
+                  Serializable lockValue = ValueUtils.getValueAsFieldType(fieldType, oldRecord.getValue(lock.getFieldName()));
+                  ValidateRecordSecurityLockHelper.validateRecordSecurityValue(table, record, lock, lockValue, fieldType, ValidateRecordSecurityLockHelper.Action.UPDATE);
+               }
+            }
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java

@@ -31,12 +31,16 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -50,9 +54,11 @@ import com.kingsrook.qqq.backend.core.model.querystats.QueryStatCriteriaField;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStatJoinTable;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStatOrderByField;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
+import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
+import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -357,8 +363,8 @@ public class QueryStatManager
                   //////////////////////
                   // set the table id //
                   //////////////////////
-                  Integer tableId = QQQTableAccessor.getTableId(queryStat.getTableName());
-                  queryStat.setTableId(tableId);
+                  Integer qqqTableId = getQQQTableId(queryStat.getTableName());
+                  queryStat.setQqqTableId(qqqTableId);
 
                   //////////////////////////////
                   // build join-table records //
@@ -368,7 +374,7 @@ public class QueryStatManager
                      List<QueryStatJoinTable> queryStatJoinTableList = new ArrayList<>();
                      for(String joinTableName : queryStat.getJoinTableNames())
                      {
-                        queryStatJoinTableList.add(new QueryStatJoinTable().withTableId(QQQTableAccessor.getTableId(joinTableName)));
+                        queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(getQQQTableId(joinTableName)));
                      }
                      queryStat.setQueryStatJoinTableList(queryStatJoinTableList);
                   }
@@ -379,14 +385,14 @@ public class QueryStatManager
                   if(queryStat.getQueryFilter() != null && queryStat.getQueryFilter().hasAnyCriteria())
                   {
                      List<QueryStatCriteriaField> queryStatCriteriaFieldList = new ArrayList<>();
-                     processCriteriaFromFilter(tableId, queryStatCriteriaFieldList, queryStat.getQueryFilter());
+                     processCriteriaFromFilter(qqqTableId, queryStatCriteriaFieldList, queryStat.getQueryFilter());
                      queryStat.setQueryStatCriteriaFieldList(queryStatCriteriaFieldList);
                   }
 
                   if(CollectionUtils.nullSafeHasContents(queryStat.getQueryFilter().getOrderBys()))
                   {
                      List<QueryStatOrderByField> queryStatOrderByFieldList = new ArrayList<>();
-                     processOrderByFromFilter(tableId, queryStatOrderByFieldList, queryStat.getQueryFilter());
+                     processOrderByFromFilter(qqqTableId, queryStatOrderByFieldList, queryStat.getQueryFilter());
                      queryStat.setQueryStatOrderByFieldList(queryStatOrderByFieldList);
                   }
 
@@ -428,7 +434,7 @@ public class QueryStatManager
       /*******************************************************************************
        **
        *******************************************************************************/
-      private static void processCriteriaFromFilter(Integer tableId, List<QueryStatCriteriaField> queryStatCriteriaFieldList, QQueryFilter queryFilter) throws QException
+      private static void processCriteriaFromFilter(Integer qqqTableId, List<QueryStatCriteriaField> queryStatCriteriaFieldList, QQueryFilter queryFilter) throws QException
       {
          for(QFilterCriteria criteria : CollectionUtils.nonNullList(queryFilter.getCriteria()))
          {
@@ -446,13 +452,13 @@ public class QueryStatManager
                String[] parts = fieldName.split("\\.");
                if(parts.length > 1)
                {
-                  queryStatCriteriaField.setTableId(QQQTableAccessor.getTableId(parts[0]));
+                  queryStatCriteriaField.setQqqTableId(getQQQTableId(parts[0]));
                   queryStatCriteriaField.setName(parts[1]);
                }
             }
             else
             {
-               queryStatCriteriaField.setTableId(tableId);
+               queryStatCriteriaField.setQqqTableId(qqqTableId);
                queryStatCriteriaField.setName(fieldName);
             }
 
@@ -461,7 +467,7 @@ public class QueryStatManager
 
          for(QQueryFilter subFilter : CollectionUtils.nonNullList(queryFilter.getSubFilters()))
          {
-            processCriteriaFromFilter(tableId, queryStatCriteriaFieldList, subFilter);
+            processCriteriaFromFilter(qqqTableId, queryStatCriteriaFieldList, subFilter);
          }
       }
 
@@ -470,7 +476,7 @@ public class QueryStatManager
       /*******************************************************************************
        **
        *******************************************************************************/
-      private static void processOrderByFromFilter(Integer tableId, List<QueryStatOrderByField> queryStatOrderByFieldList, QQueryFilter queryFilter) throws QException
+      private static void processOrderByFromFilter(Integer qqqTableId, List<QueryStatOrderByField> queryStatOrderByFieldList, QQueryFilter queryFilter) throws QException
       {
          for(QFilterOrderBy orderBy : CollectionUtils.nonNullList(queryFilter.getOrderBys()))
          {
@@ -484,13 +490,13 @@ public class QueryStatManager
                   String[] parts = fieldName.split("\\.");
                   if(parts.length > 1)
                   {
-                     queryStatOrderByField.setTableId(QQQTableAccessor.getTableId(parts[0]));
+                     queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0]));
                      queryStatOrderByField.setName(parts[1]);
                   }
                }
                else
                {
-                  queryStatOrderByField.setTableId(tableId);
+                  queryStatOrderByField.setQqqTableId(qqqTableId);
                   queryStatOrderByField.setName(fieldName);
                }
 
@@ -499,6 +505,43 @@ public class QueryStatManager
          }
       }
 
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      private static Integer getQQQTableId(String tableName) throws QException
+      {
+         /////////////////////////////
+         // look in the cache table //
+         /////////////////////////////
+         GetInput getInput = new GetInput();
+         getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME);
+         getInput.setUniqueKey(MapBuilder.of("name", tableName));
+         GetOutput getOutput = new GetAction().execute(getInput);
+
+         ////////////////////////
+         // upon cache miss... //
+         ////////////////////////
+         if(getOutput.getRecord() == null)
+         {
+            ///////////////////////////////////////////////////////
+            // insert the record (into the table, not the cache) //
+            ///////////////////////////////////////////////////////
+            QTableMetaData tableMetaData = getInstance().qInstance.getTable(tableName);
+            InsertInput    insertInput   = new InsertInput();
+            insertInput.setTableName(QQQTable.TABLE_NAME);
+            insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel())));
+            InsertOutput insertOutput = new InsertAction().execute(insertInput);
+
+            ///////////////////////////////////
+            // repeat the get from the cache //
+            ///////////////////////////////////
+            getOutput = new GetAction().execute(getInput);
+         }
+
+         return getOutput.getRecord().getValueInteger("id");
+      }
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java

@@ -44,6 +44,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.PermissionDeniedMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -68,6 +69,7 @@ public class ValidateRecordSecurityLockHelper
    {
       INSERT,
       UPDATE,
+      DELETE,
       SELECT
    }
 
@@ -78,7 +80,7 @@ public class ValidateRecordSecurityLockHelper
     *******************************************************************************/
    public static void validateSecurityFields(QTableMetaData table, List<QRecord> records, Action action) throws QException
    {
-      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table);
+      List<RecordSecurityLock> locksToCheck = getRecordSecurityLocks(table, action);
       if(CollectionUtils.nullSafeIsEmpty(locksToCheck))
       {
          return;
@@ -98,11 +100,12 @@ public class ValidateRecordSecurityLockHelper
 
             for(QRecord record : records)
             {
-               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()))
+               if(action.equals(Action.UPDATE) && !record.getValues().containsKey(field.getName()) && RecordSecurityLock.LockScope.READ_AND_WRITE.equals(recordSecurityLock.getLockScope()))
                {
-                  /////////////////////////////////////////////////////////////////////////
-                  // if not updating the security field, then no error can come from it! //
-                  /////////////////////////////////////////////////////////////////////////
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if this is a read-write lock, then if we have the record, it means we were able to read the record. //
+                  // So if we're not updating the security field, then no error can come from it!                        //
+                  /////////////////////////////////////////////////////////////////////////////////////////////////////////
                   continue;
                }
 
@@ -244,11 +247,18 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table)
+   private static List<RecordSecurityLock> getRecordSecurityLocks(QTableMetaData table, Action action)
    {
-      List<RecordSecurityLock> recordSecurityLocks = table.getRecordSecurityLocks();
+      List<RecordSecurityLock> recordSecurityLocks = CollectionUtils.nonNullList(table.getRecordSecurityLocks());
       List<RecordSecurityLock> locksToCheck        = new ArrayList<>();
 
+      recordSecurityLocks = switch(action)
+      {
+         case INSERT, UPDATE, DELETE -> RecordSecurityLockFilters.filterForWriteLocks(recordSecurityLocks);
+         case SELECT -> RecordSecurityLockFilters.filterForReadLocks(recordSecurityLocks);
+         default -> throw (new IllegalArgumentException("Unsupported action: " + action));
+      };
+
       ////////////////////////////////////////
       // if there are no locks, just return //
       ////////////////////////////////////////
@@ -281,7 +291,7 @@ public class ValidateRecordSecurityLockHelper
    /*******************************************************************************
     **
     *******************************************************************************/
-   static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
+   public static void validateRecordSecurityValue(QTableMetaData table, QRecord record, RecordSecurityLock recordSecurityLock, Serializable recordSecurityValue, QFieldType fieldType, Action action)
    {
       if(recordSecurityValue == null)
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -244,8 +244,6 @@ public class SearchPossibleValueSourceAction
          }
       }
 
-      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
-
       // todo - skip & limit as params
       queryFilter.setLimit(250);
 
@@ -257,6 +255,9 @@ public class SearchPossibleValueSourceAction
          input.getDefaultQueryFilter().addSubFilter(queryFilter);
          queryFilter = input.getDefaultQueryFilter();
       }
+
+      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
+
       queryInput.setFilter(queryFilter);
 
       QueryOutput             queryOutput     = new QueryAction().execute(queryInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/context/QContext.java

@@ -84,7 +84,7 @@ public class QContext
          actionStackThreadLocal.get().add(actionInput);
       }
 
-      if(!qInstance.getHasBeenValidated())
+      if(qInstance != null && !qInstance.getHasBeenValidated())
       {
          try
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -272,7 +272,7 @@ public class QInstanceEnricher
 
          for(QSupplementalTableMetaData supplementalTableMetaData : CollectionUtils.nonNullMap(table.getSupplementalMetaData()).values())
          {
-            supplementalTableMetaData.enrich(table);
+            supplementalTableMetaData.enrich(qInstance, table);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -438,10 +438,13 @@ public class QInstanceValidator
                for(QFieldSection section : table.getSections())
                {
                   validateTableSection(qInstance, table, section, fieldNamesInSections);
-                  if(section.getTier().equals(Tier.T1))
+                  if(assertCondition(section.getTier() != null, "Table " + tableName + " " + section.getName() + " is missing its tier"))
                   {
-                     assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
-                     tier1Section = section;
+                     if(section.getTier().equals(Tier.T1))
+                     {
+                        assertCondition(tier1Section == null, "Table " + tableName + " has more than 1 section listed as Tier 1");
+                        tier1Section = section;
+                     }
                   }
 
                   assertCondition(!usedSectionNames.contains(section.getName()), "Table " + tableName + " has more than 1 section named " + section.getName());
@@ -586,6 +589,8 @@ public class QInstanceValidator
 
          prefix = "Table " + table.getName() + " recordSecurityLock (of key type " + securityKeyTypeName + ") ";
 
+         assertCondition(recordSecurityLock.getLockScope() != null, prefix + " is missing its lockScope");
+
          boolean hasAnyBadJoins = false;
          for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
          {
@@ -1099,13 +1104,39 @@ public class QInstanceValidator
       boolean hasFields = CollectionUtils.nullSafeHasContents(section.getFieldNames());
       boolean hasWidget = StringUtils.hasContent(section.getWidgetName());
 
-      if(assertCondition(hasFields || hasWidget, "Table " + table.getName() + " section " + section.getName() + " does not have any fields or a widget."))
+      String sectionPrefix = "Table " + table.getName() + " section " + section.getName() + " ";
+      if(assertCondition(hasFields || hasWidget, sectionPrefix + "does not have any fields or a widget."))
       {
          if(table.getFields() != null && hasFields)
          {
             for(String fieldName : section.getFieldNames())
             {
-               assertCondition(table.getFields().containsKey(fieldName), "Table " + table.getName() + " section " + section.getName() + " specifies fieldName " + fieldName + ", which is not a field on this table.");
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // note - this was originally written as an assertion:                                                                                                                                       //
+               // if(assertCondition(qInstance.getTable(otherTableName) != null, sectionPrefix + "join-field " + fieldName + ", which is referencing an unrecognized table name [" + otherTableName + "]")) //
+               // but... then a field name with dots gives us a bad time here, so...                                                                                                                        //
+               ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               if(fieldName.contains(".") && qInstance.getTable(fieldName.split("\\.")[0]) != null)
+               {
+                  String[] parts            = fieldName.split("\\.");
+                  String   otherTableName   = parts[0];
+                  String   foreignFieldName = parts[1];
+
+                  if(assertCondition(qInstance.getTable(otherTableName) != null, sectionPrefix + "join-field " + fieldName + ", which is referencing an unrecognized table name [" + otherTableName + "]"))
+                  {
+                     List<ExposedJoin> matchedExposedJoins = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> otherTableName.equals(ej.getJoinTable())).toList();
+                     if(assertCondition(CollectionUtils.nullSafeHasContents(matchedExposedJoins), sectionPrefix + "join-field " + fieldName + ", referencing table [" + otherTableName + "] which is not an exposed join on this table."))
+                     {
+                        assertCondition(!matchedExposedJoins.get(0).getIsMany(qInstance), sectionPrefix + "join-field " + fieldName + " references an is-many join, which is not supported.");
+                     }
+                     assertCondition(qInstance.getTable(otherTableName).getFields().containsKey(foreignFieldName), sectionPrefix + "join-field " + fieldName + " specifies a fieldName [" + foreignFieldName + "] which does not exist in that table [" + otherTableName + "].");
+                  }
+               }
+               else
+               {
+                  assertCondition(table.getFields().containsKey(fieldName), sectionPrefix + "specifies fieldName " + fieldName + ", which is not a field on this table.");
+               }
+
                assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + table.getName() + " has field " + fieldName + " listed more than once in its field sections.");
 
                fieldNamesInSections.add(fieldName);
@@ -1113,7 +1144,7 @@ public class QInstanceValidator
          }
          else if(hasWidget)
          {
-            assertCondition(qInstance.getWidget(section.getWidgetName()) != null, "Table " + table.getName() + " section " + section.getName() + " specifies widget " + section.getWidgetName() + ", which is not a widget in this instance.");
+            assertCondition(qInstance.getWidget(section.getWidgetName()) != null, sectionPrefix + "specifies widget " + section.getWidgetName() + ", which is not a widget in this instance.");
          }
       }
    }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java

@@ -347,7 +347,7 @@ public class QMetaDataVariableInterpreter
          if(canParseAsInteger(envValue))
          {
             LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName);
-            return (Integer.parseInt(propertyValue));
+            return (Integer.parseInt(envValue));
          }
          else
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -30,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
@@ -246,7 +247,7 @@ public class AuditSingleInput
       setAuditTableName(table.getName());
 
       this.securityKeyValues = new HashMap<>();
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
          this.securityKeyValues.put(recordSecurityLock.getFieldName(), record.getValueInteger(recordSecurityLock.getFieldName()));
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/QueryOrGetInputInterface.java

@@ -23,7 +23,9 @@ package com.kingsrook.qqq.backend.core.model.actions.tables;
 
 
 import java.util.Collection;
+import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 
 
 /*******************************************************************************
@@ -49,6 +51,7 @@ public interface QueryOrGetInputInterface
       this.setShouldMaskPasswords(source.getShouldMaskPasswords());
       this.setIncludeAssociations(source.getIncludeAssociations());
       this.setAssociationNamesToInclude(source.getAssociationNamesToInclude());
+      this.setQueryJoins(source.getQueryJoins());
    }
 
    /*******************************************************************************
@@ -146,4 +149,17 @@ public interface QueryOrGetInputInterface
     *******************************************************************************/
    void setAssociationNamesToInclude(Collection<String> associationNamesToInclude);
 
+
+   /*******************************************************************************
+    ** Getter for queryJoins
+    *******************************************************************************/
+   List<QueryJoin> getQueryJoins();
+
+
+   /*******************************************************************************
+    ** Setter for queryJoins
+    **
+    *******************************************************************************/
+   void setQueryJoins(List<QueryJoin> queryJoins);
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/get/GetInput.java

@@ -23,11 +23,14 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.get;
 
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 
 
 /*******************************************************************************
@@ -47,6 +50,7 @@ public class GetInput extends AbstractTableActionInput implements QueryOrGetInpu
    private boolean shouldOmitHiddenFields        = true;
    private boolean shouldMaskPasswords           = true;
 
+   private List<QueryJoin> queryJoins = null;
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
    // if you say you want to includeAssociations, you can limit which ones by passing them in associationNamesToInclude. //
@@ -411,4 +415,51 @@ public class GetInput extends AbstractTableActionInput implements QueryOrGetInpu
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for queryJoins
+    *******************************************************************************/
+   public List<QueryJoin> getQueryJoins()
+   {
+      return (this.queryJoins);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for queryJoins
+    *******************************************************************************/
+   public void setQueryJoins(List<QueryJoin> queryJoins)
+   {
+      this.queryJoins = queryJoins;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryJoins
+    *******************************************************************************/
+   public GetInput withQueryJoins(List<QueryJoin> queryJoins)
+   {
+      this.queryJoins = queryJoins;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for queryJoins
+    **
+    *******************************************************************************/
+   public GetInput withQueryJoin(QueryJoin queryJoin)
+   {
+      if(this.queryJoins == null)
+      {
+         this.queryJoins = new ArrayList<>();
+      }
+      this.queryJoins.add(queryJoin);
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -30,17 +30,21 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MutableList;
+import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -60,6 +64,7 @@ public class JoinsContext
    // note - will have entries for all tables, not just aliases. //
    ////////////////////////////////////////////////////////////////
    private final Map<String, String> aliasToTableNameMap = new HashMap<>();
+   private       Level               logLevel            = Level.OFF;
 
 
 
@@ -69,62 +74,23 @@ public class JoinsContext
     *******************************************************************************/
    public JoinsContext(QInstance instance, String tableName, List<QueryJoin> queryJoins, QQueryFilter filter) throws QException
    {
+      log("--- START ----------------------------------------------------------------------", logPair("mainTable", tableName));
       this.instance = instance;
       this.mainTableName = tableName;
       this.queryJoins = new MutableList<>(queryJoins);
 
       for(QueryJoin queryJoin : this.queryJoins)
       {
+         log("Processing input query join", logPair("joinTable", queryJoin.getJoinTable()), logPair("alias", queryJoin.getAlias()), logPair("baseTableOrAlias", queryJoin.getBaseTableOrAlias()), logPair("joinMetaDataName", () -> queryJoin.getJoinMetaData().getName()));
          processQueryJoin(queryJoin);
       }
 
       ///////////////////////////////////////////////////////////////
       // ensure any joins that contribute a recordLock are present //
       ///////////////////////////////////////////////////////////////
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(instance.getTable(tableName).getRecordSecurityLocks())))
       {
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         // ok - so - the join name chain is going to be like this:                                       //
-         // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
-         // - securityFieldName = order.clientId                                                          //
-         // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
-         // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
-         // and step (via tmpTable variable) back to the securityField                                    //
-         ///////////////////////////////////////////////////////////////////////////////////////////////////
-         ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
-         Collections.reverse(joinNameChain);
-
-         QTableMetaData tmpTable = instance.getTable(mainTableName);
-
-         for(String joinName : joinNameChain)
-         {
-            if(this.queryJoins.stream().anyMatch(queryJoin ->
-            {
-               QJoinMetaData joinMetaData = Objects.requireNonNullElseGet(queryJoin.getJoinMetaData(), () -> findJoinMetaData(instance, tableName, queryJoin.getJoinTable()));
-               return (joinMetaData != null && Objects.equals(joinMetaData.getName(), joinName));
-            }))
-            {
-               continue;
-            }
-
-            QJoinMetaData join = instance.getJoin(joinName);
-            if(join.getLeftTable().equals(tmpTable.getName()))
-            {
-               QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin);
-               tmpTable = instance.getTable(join.getRightTable());
-            }
-            else if(join.getRightTable().equals(tmpTable.getName()))
-            {
-               QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin); //
-               tmpTable = instance.getTable(join.getLeftTable());
-            }
-            else
-            {
-               throw (new QException("Error adding security lock joins to query - table name [" + tmpTable.getName() + "] not found in join [" + joinName + "]"));
-            }
-         }
+         ensureRecordSecurityLockIsRepresented(instance, tableName, recordSecurityLock);
       }
 
       ensureFilterIsRepresented(filter);
@@ -141,6 +107,86 @@ public class JoinsContext
          }
       }
       */
+
+      log("Constructed JoinsContext", logPair("mainTableName", this.mainTableName), logPair("queryJoins", this.queryJoins.stream().map(qj -> qj.getJoinTable()).collect(Collectors.joining(","))));
+      log("--- END ------------------------------------------------------------------------");
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void ensureRecordSecurityLockIsRepresented(QInstance instance, String tableName, RecordSecurityLock recordSecurityLock) throws QException
+   {
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      // ok - so - the join name chain is going to be like this:                                       //
+      // for a table:  orderLineItemExtrinsic (that's 2 away from order, where the security field is): //
+      // - securityFieldName = order.clientId                                                          //
+      // - joinNameChain = orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic             //
+      // so - to navigate from the table to the security field, we need to reverse the joinNameChain,  //
+      // and step (via tmpTable variable) back to the securityField                                    //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////
+      ArrayList<String> joinNameChain = new ArrayList<>(CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()));
+      Collections.reverse(joinNameChain);
+      log("Evaluating recordSecurityLock", logPair("recordSecurityLock", recordSecurityLock.getFieldName()), logPair("joinNameChain", joinNameChain));
+
+      QTableMetaData tmpTable = instance.getTable(mainTableName);
+
+      for(String joinName : joinNameChain)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         // check the joins currently in the query - if any are for this table, then we don't need to add one //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////
+         List<QueryJoin> matchingJoins = this.queryJoins.stream().filter(queryJoin ->
+         {
+            QJoinMetaData joinMetaData = null;
+            if(queryJoin.getJoinMetaData() != null)
+            {
+               joinMetaData = queryJoin.getJoinMetaData();
+            }
+            else
+            {
+               joinMetaData = findJoinMetaData(instance, tableName, queryJoin.getJoinTable());
+            }
+            return (joinMetaData != null && Objects.equals(joinMetaData.getName(), joinName));
+         }).toList();
+
+         if(CollectionUtils.nullSafeHasContents(matchingJoins))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // note - if a user added a join as an outer type, we need to change it to be inner, for the security purpose. //
+            // todo - is this always right?  what about nulls-allowed?                                                     //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            log("- skipping join already in the query", logPair("joinName", joinName));
+
+            if(matchingJoins.get(0).getType().equals(QueryJoin.Type.LEFT) || matchingJoins.get(0).getType().equals(QueryJoin.Type.RIGHT))
+            {
+               log("- - although... it was here as an outer - so switching it to INNER", logPair("joinName", joinName));
+               matchingJoins.get(0).setType(QueryJoin.Type.INNER);
+            }
+
+            continue;
+         }
+
+         QJoinMetaData join = instance.getJoin(joinName);
+         if(join.getLeftTable().equals(tmpTable.getName()))
+         {
+            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
+            this.addQueryJoin(queryJoin, "forRecordSecurityLock (non-flipped)");
+            tmpTable = instance.getTable(join.getRightTable());
+         }
+         else if(join.getRightTable().equals(tmpTable.getName()))
+         {
+            QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
+            this.addQueryJoin(queryJoin, "forRecordSecurityLock (flipped)");
+            tmpTable = instance.getTable(join.getLeftTable());
+         }
+         else
+         {
+            throw (new QException("Error adding security lock joins to query - table name [" + tmpTable.getName() + "] not found in join [" + joinName + "]"));
+         }
+      }
    }
 
 
@@ -151,8 +197,15 @@ public class JoinsContext
     ** use this method to add to the list, instead of ever adding directly, as it's
     ** important do to that process step (and we've had bugs when it wasn't done).
     *******************************************************************************/
-   private void addQueryJoin(QueryJoin queryJoin) throws QException
+   private void addQueryJoin(QueryJoin queryJoin, String reason) throws QException
    {
+      log("Adding query join to context",
+         logPair("reason", reason),
+         logPair("joinTable", queryJoin.getJoinTable()),
+         logPair("joinMetaData.name", () -> queryJoin.getJoinMetaData().getName()),
+         logPair("joinMetaData.leftTable", () -> queryJoin.getJoinMetaData().getLeftTable()),
+         logPair("joinMetaData.rightTable", () -> queryJoin.getJoinMetaData().getRightTable())
+      );
       this.queryJoins.add(queryJoin);
       processQueryJoin(queryJoin);
    }
@@ -177,10 +230,46 @@ public class JoinsContext
          addedJoin = false;
          for(QueryJoin queryJoin : queryJoins)
          {
-            /////////////////////////////////////////////////////////////////////
-            // if the join has joinMetaData, then we don't need to process it. //
-            /////////////////////////////////////////////////////////////////////
-            if(queryJoin.getJoinMetaData() == null)
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            // if the join has joinMetaData, then we don't need to process it... unless it needs flipped //
+            ///////////////////////////////////////////////////////////////////////////////////////////////
+            QJoinMetaData joinMetaData = queryJoin.getJoinMetaData();
+            if(joinMetaData != null)
+            {
+               boolean isJoinLeftTableInQuery = false;
+               String  joinMetaDataLeftTable  = joinMetaData.getLeftTable();
+               if(joinMetaDataLeftTable.equals(mainTableName))
+               {
+                  isJoinLeftTableInQuery = true;
+               }
+
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               // check the other joins in this query - if any of them have this join's left-table as their baseTable, then set the flag to true //
+               ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+               for(QueryJoin otherJoin : queryJoins)
+               {
+                  if(otherJoin == queryJoin)
+                  {
+                     continue;
+                  }
+
+                  if(Objects.equals(otherJoin.getBaseTableOrAlias(), joinMetaDataLeftTable))
+                  {
+                     isJoinLeftTableInQuery = true;
+                     break;
+                  }
+               }
+
+               /////////////////////////////////////////////////////////////////////////////////
+               // if the join's left-table isn't in the query, then we need to flip the join. //
+               /////////////////////////////////////////////////////////////////////////////////
+               if(!isJoinLeftTableInQuery)
+               {
+                  log("Flipping queryJoin because its leftTable wasn't found in the query", logPair("joinMetaDataName", joinMetaData.getName()), logPair("leftTable", joinMetaDataLeftTable));
+                  queryJoin.setJoinMetaData(joinMetaData.flip());
+               }
+            }
+            else
             {
                //////////////////////////////////////////////////////////////////////
                // try to find a direct join between the main table and this table. //
@@ -190,6 +279,7 @@ public class JoinsContext
                QJoinMetaData found         = findJoinMetaData(instance, baseTableName, queryJoin.getJoinTable());
                if(found != null)
                {
+                  log("Found joinMetaData - setting it in queryJoin", logPair("joinMetaDataName", found.getName()), logPair("baseTableName", baseTableName), logPair("joinTable", queryJoin.getJoinTable()));
                   queryJoin.setJoinMetaData(found);
                }
                else
@@ -197,15 +287,13 @@ public class JoinsContext
                   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                   // else, the join must be indirect - so look for an exposedJoin that will have a joinPath that will connect us //
                   /////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  LOG.debug("Looking for an exposed join...", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()));
-
                   QTableMetaData mainTable          = instance.getTable(mainTableName);
                   boolean        addedAnyQueryJoins = false;
                   for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins()))
                   {
                      if(queryJoin.getJoinTable().equals(exposedJoin.getJoinTable()))
                      {
-                        LOG.debug("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
+                        log("Found an exposed join", logPair("mainTable", mainTableName), logPair("joinTable", queryJoin.getJoinTable()), logPair("joinPath", exposedJoin.getJoinPath()));
 
                         /////////////////////////////////////////////////////////////////////////////////////
                         // loop backward through the join path (from the joinTable back to the main table) //
@@ -250,7 +338,7 @@ public class JoinsContext
                                  QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd);
                                  queryJoinToAdd.setType(queryJoin.getType());
                                  addedAnyQueryJoins = true;
-                                 this.addQueryJoin(queryJoinToAdd);
+                                 this.addQueryJoin(queryJoinToAdd, "forExposedJoin");
                               }
                            }
 
@@ -377,9 +465,9 @@ public class JoinsContext
     **
     ** e.g., Given:
     **   FROM `order` INNER JOIN line_item li
-    ** hasAliasOrTable("order") => true
-    ** hasAliasOrTable("li") => false
-    ** hasAliasOrTable("line_item") => true
+    ** hasTable("order") => true
+    ** hasTable("li") => false
+    ** hasTable("line_item") => true
     *******************************************************************************/
    public boolean hasTable(String table)
    {
@@ -415,15 +503,17 @@ public class JoinsContext
 
       for(String filterTable : filterTables)
       {
+         log("Evaluating filterTable", logPair("filterTable", filterTable));
          if(!aliasToTableNameMap.containsKey(filterTable) && !Objects.equals(mainTableName, filterTable))
          {
+            log("- table not in query - adding it", logPair("filterTable", filterTable));
             boolean found = false;
             for(QJoinMetaData join : CollectionUtils.nonNullMap(QContext.getQInstance().getJoins()).values())
             {
                QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join);
                if(queryJoin != null)
                {
-                  this.addQueryJoin(queryJoin);
+                  this.addQueryJoin(queryJoin, "forFilter (join found in instance)");
                   found = true;
                   break;
                }
@@ -432,7 +522,7 @@ public class JoinsContext
             if(!found)
             {
                QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER);
-               this.addQueryJoin(queryJoin);
+               this.addQueryJoin(queryJoin, "forFilter (join not found in instance)");
             }
          }
       }
@@ -569,4 +659,14 @@ public class JoinsContext
    {
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private void log(String message, LogPair... logPairs)
+   {
+      LOG.log(logLevel, message, null, logPairs);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java

@@ -39,7 +39,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
 
 
 /*******************************************************************************
@@ -47,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
  *******************************************************************************/
 public class AuditsMetaDataProvider
 {
+   public static final String TABLE_NAME_AUDIT_TABLE  = "auditTable";
    public static final String TABLE_NAME_AUDIT_USER   = "auditUser";
    public static final String TABLE_NAME_AUDIT        = "audit";
    public static final String TABLE_NAME_AUDIT_DETAIL = "auditDetail";
@@ -72,10 +72,10 @@ public class AuditsMetaDataProvider
    {
       instance.addJoin(new QJoinMetaData()
          .withLeftTable(TABLE_NAME_AUDIT)
-         .withRightTable(QQQTable.TABLE_NAME)
+         .withRightTable(TABLE_NAME_AUDIT_TABLE)
          .withInferredName()
          .withType(JoinType.MANY_TO_ONE)
-         .withJoinOn(new JoinOn("tableId", "id")));
+         .withJoinOn(new JoinOn("auditTableId", "id")));
 
       instance.addJoin(new QJoinMetaData()
          .withLeftTable(TABLE_NAME_AUDIT)
@@ -113,14 +113,22 @@ public class AuditsMetaDataProvider
     *******************************************************************************/
    public void defineStandardAuditPossibleValueSources(QInstance instance)
    {
+      instance.addPossibleValueSource(new QPossibleValueSource()
+         .withName(TABLE_NAME_AUDIT_TABLE)
+         .withTableName(TABLE_NAME_AUDIT_TABLE)
+         .withOrderByField("name")
+      );
+
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT_USER)
          .withTableName(TABLE_NAME_AUDIT_USER)
+         .withOrderByField("name")
       );
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT)
          .withTableName(TABLE_NAME_AUDIT)
+         .withOrderByField("id", false)
       );
    }
 
@@ -133,6 +141,7 @@ public class AuditsMetaDataProvider
    {
       List<QTableMetaData> rs = new ArrayList<>();
       rs.add(enrich(backendDetailEnricher, defineAuditUserTable(backendName)));
+      rs.add(enrich(backendDetailEnricher, defineAuditTableTable(backendName)));
       rs.add(enrich(backendDetailEnricher, defineAuditTable(backendName)));
       rs.add(enrich(backendDetailEnricher, defineAuditDetailTable(backendName)));
       return (rs);
@@ -154,6 +163,29 @@ public class AuditsMetaDataProvider
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QTableMetaData defineAuditTableTable(String backendName)
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_AUDIT_TABLE)
+         .withBackendName(backendName)
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("label")
+         .withPrimaryKeyField("id")
+         .withUniqueKey(new UniqueKey("name"))
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+         .withField(new QFieldMetaData("label", QFieldType.STRING))
+         .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))
+         .withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -186,10 +218,10 @@ public class AuditsMetaDataProvider
          .withBackendName(backendName)
          .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))
          .withRecordLabelFormat("%s %s")
-         .withRecordLabelFields("tableId", "recordId")
+         .withRecordLabelFields("auditTableId", "recordId")
          .withPrimaryKeyField("id")
          .withField(new QFieldMetaData("id", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("tableId", QFieldType.INTEGER).withPossibleValueSourceName(QQQTable.TABLE_NAME))
+         .withField(new QFieldMetaData("auditTableId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_TABLE))
          .withField(new QFieldMetaData("auditUserId", QFieldType.INTEGER).withPossibleValueSourceName(TABLE_NAME_AUDIT_USER))
          .withField(new QFieldMetaData("recordId", QFieldType.INTEGER))
          .withField(new QFieldMetaData("message", QFieldType.STRING).withMaxLength(250).withBehavior(ValueTooLongBehavior.TRUNCATE_ELLIPSIS))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/automation/TableTrigger.java

@@ -27,9 +27,9 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
 
 
 /*******************************************************************************
@@ -48,8 +48,8 @@ public class TableTrigger extends QRecordEntity
    @QField(isEditable = false)
    private Instant modifyDate;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME)
+   private String tableName;
 
    @QField(possibleValueSourceName = SavedFilter.TABLE_NAME)
    private Integer filterId;
@@ -191,6 +191,40 @@ public class TableTrigger extends QRecordEntity
 
 
 
+   /*******************************************************************************
+    ** Getter for tableName
+    **
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    **
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    **
+    *******************************************************************************/
+   public TableTrigger withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for filterId
     **
@@ -356,35 +390,4 @@ public class TableTrigger extends QRecordEntity
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for tableId
-    *******************************************************************************/
-   public Integer getTableId()
-   {
-      return (this.tableId);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for tableId
-    *******************************************************************************/
-   public void setTableId(Integer tableId)
-   {
-      this.tableId = tableId;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for tableId
-    *******************************************************************************/
-   public TableTrigger withTableId(Integer tableId)
-   {
-      this.tableId = tableId;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java

@@ -421,7 +421,7 @@ public abstract class QRecordEntity
          {
             if(!method.getName().equals("getClass"))
             {
-               LOG.info("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported.");
+               LOG.debug("Method [" + method.getName() + "] looks like a getter, but its return type, [" + method.getReturnType() + "], isn't supported.");
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -93,6 +93,7 @@ public class QInstance
 
    private Map<String, QSupplementalInstanceMetaData> supplementalMetaData = new LinkedHashMap<>();
 
+   private String              deploymentMode;
    private Map<String, String> environmentValues = new LinkedHashMap<>();
    private String              defaultTimeZoneId = "UTC";
 
@@ -1165,4 +1166,36 @@ public class QInstance
       }
       this.joinGraph = joinGraph;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for deploymentMode
+    *******************************************************************************/
+   public String getDeploymentMode()
+   {
+      return (this.deploymentMode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deploymentMode
+    *******************************************************************************/
+   public void setDeploymentMode(String deploymentMode)
+   {
+      this.deploymentMode = deploymentMode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for deploymentMode
+    *******************************************************************************/
+   public QInstance withDeploymentMode(String deploymentMode)
+   {
+      this.deploymentMode = deploymentMode;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/authentication/Auth0AuthenticationMetaData.java

@@ -60,7 +60,6 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
    private String       auth0ClientSecretField;
    private Serializable qqqRecordIdField;
 
-
    /////////////////////////////////////
    // fields on the accessToken table //
    /////////////////////////////////////
@@ -70,6 +69,14 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
    private String qqqApiKeyField;
    private String expiresInSecondsField;
 
+   /////////////////////////////////////////////////////////////////////////////////
+   // table for storing user sessions, and field names we work with on that table //
+   /////////////////////////////////////////////////////////////////////////////////
+   private String userSessionTableName;
+   private String userSessionUuidField;
+   private String userSessionUserIdField;
+   private String userSessionAccessTokenField;
+
 
 
    /*******************************************************************************

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLock.java

@@ -34,6 +34,10 @@ import java.util.List;
  ** - recordSecurityLock.fieldName = order.clientId
  ** - recordSecurityLock.joinNameChain = [orderJoinOrderLineItem, orderLineItemJoinOrderLineItemExtrinsic]
  ** that is - what's the chain that takes us FROM the security fieldName TO the table with the lock.
+ **
+ ** LockScope controls what the lock prevents users from doing without a valid key.
+ ** - READ_AND_WRITE means that users cannot read or write records without a valid key.
+ ** - WRITE means that users cannot write records without a valid key (but they can read them).
  *******************************************************************************/
 public class RecordSecurityLock
 {
@@ -42,6 +46,8 @@ public class RecordSecurityLock
    private List<String>      joinNameChain;
    private NullValueBehavior nullValueBehavior = NullValueBehavior.DENY;
 
+   private LockScope lockScope = LockScope.READ_AND_WRITE;
+
 
 
    /*******************************************************************************
@@ -66,6 +72,17 @@ public class RecordSecurityLock
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public enum LockScope
+   {
+      READ_AND_WRITE,
+      WRITE
+   }
+
+
+
    /*******************************************************************************
     ** Getter for securityKeyType
     *******************************************************************************/
@@ -188,4 +205,35 @@ public class RecordSecurityLock
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for lockScope
+    *******************************************************************************/
+   public LockScope getLockScope()
+   {
+      return (this.lockScope);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for lockScope
+    *******************************************************************************/
+   public void setLockScope(LockScope lockScope)
+   {
+      this.lockScope = lockScope;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for lockScope
+    *******************************************************************************/
+   public RecordSecurityLock withLockScope(LockScope lockScope)
+   {
+      this.lockScope = lockScope;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/security/RecordSecurityLockFilters.java

@@ -0,0 +1,80 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.security;
+
+
+import java.util.List;
+
+
+/*******************************************************************************
+ ** standard filtering operations for lists of record security locks.
+ *******************************************************************************/
+public class RecordSecurityLockFilters
+{
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to reads.
+    *******************************************************************************/
+   public static List<RecordSecurityLock> filterForReadLocks(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      return (recordSecurityLocks.stream().filter(rsl -> RecordSecurityLock.LockScope.READ_AND_WRITE.equals(rsl.getLockScope())).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that apply to writes.
+    *******************************************************************************/
+   public static List<RecordSecurityLock> filterForWriteLocks(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      return (recordSecurityLocks.stream().filter(rsl ->
+         RecordSecurityLock.LockScope.READ_AND_WRITE.equals(rsl.getLockScope())
+            || RecordSecurityLock.LockScope.WRITE.equals(rsl.getLockScope()
+         )).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** filter a list of locks so that we only see the ones that are WRITE type only.
+    *******************************************************************************/
+   public static List<RecordSecurityLock> filterForOnlyWriteLocks(List<RecordSecurityLock> recordSecurityLocks)
+   {
+      if(recordSecurityLocks == null)
+      {
+         return (null);
+      }
+
+      return (recordSecurityLocks.stream().filter(rsl -> RecordSecurityLock.LockScope.WRITE.equals(rsl.getLockScope())).toList());
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/ExposedJoin.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
 import java.util.List;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -62,7 +63,19 @@ public class ExposedJoin
    /*******************************************************************************
     **
     *******************************************************************************/
-   public Boolean getIsMany()
+   @JsonIgnore
+   public boolean getIsMany()
+   {
+      return (getIsMany(QContext.getQInstance()));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @JsonIgnore
+   public Boolean getIsMany(QInstance qInstance)
    {
       if(isMany == null)
       {
@@ -70,8 +83,6 @@ public class ExposedJoin
          {
             try
             {
-               QInstance qInstance = QContext.getQInstance();
-
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
                // loop backward through the joinPath, starting at the join table (since we don't know the table that this exposedJoin is attached to!) //
                //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -204,5 +215,4 @@ public class ExposedJoin
       this.joinPath = joinPath;
       return (this);
    }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QSupplementalTableMetaData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
 /*******************************************************************************
  ** Base-class for table-level meta-data defined by some supplemental module, etc,
  ** outside of qqq core
@@ -60,7 +63,7 @@ public abstract class QSupplementalTableMetaData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void enrich(QTableMetaData table)
+   public void enrich(QInstance qInstance, QTableMetaData table)
    {
       ////////////////////////
       // noop in base class //

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStat.java

@@ -53,8 +53,8 @@ public class QueryStat extends QRecordEntity
    @QField()
    private Integer firstResultMillis;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(label = "Table", possibleValueSourceName = QQQTable.TABLE_NAME)
+   private Integer qqqTableId;
 
    @QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
    private String action;
@@ -413,31 +413,31 @@ public class QueryStat extends QRecordEntity
 
 
    /*******************************************************************************
-    ** Getter for tableId
+    ** Getter for qqqTableId
     *******************************************************************************/
-   public Integer getTableId()
+   public Integer getQqqTableId()
    {
-      return (this.tableId);
+      return (this.qqqTableId);
    }
 
 
 
    /*******************************************************************************
-    ** Setter for tableId
+    ** Setter for qqqTableId
     *******************************************************************************/
-   public void setTableId(Integer tableId)
+   public void setQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
    }
 
 
 
    /*******************************************************************************
-    ** Fluent setter for tableId
+    ** Fluent setter for qqqTableId
     *******************************************************************************/
-   public QueryStat withTableId(Integer tableId)
+   public QueryStat withQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
       return (this);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatCriteriaField.java

@@ -42,8 +42,8 @@ public class QueryStatCriteriaField extends QRecordEntity
    @QField(possibleValueSourceName = QueryStat.TABLE_NAME)
    private Integer queryStatId;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(label = "Table", possibleValueSourceName = QQQTable.TABLE_NAME)
+   private Integer qqqTableId;
 
    @QField(maxLength = 50, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
    private String name;
@@ -138,31 +138,31 @@ public class QueryStatCriteriaField extends QRecordEntity
 
 
    /*******************************************************************************
-    ** Getter for tableId
+    ** Getter for qqqTableId
     *******************************************************************************/
-   public Integer getTableId()
+   public Integer getQqqTableId()
    {
-      return (this.tableId);
+      return (this.qqqTableId);
    }
 
 
 
    /*******************************************************************************
-    ** Setter for tableId
+    ** Setter for qqqTableId
     *******************************************************************************/
-   public void setTableId(Integer tableId)
+   public void setQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
    }
 
 
 
    /*******************************************************************************
-    ** Fluent setter for tableId
+    ** Fluent setter for qqqTableId
     *******************************************************************************/
-   public QueryStatCriteriaField withTableId(Integer tableId)
+   public QueryStatCriteriaField withQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
       return (this);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatJoinTable.java

@@ -42,8 +42,8 @@ public class QueryStatJoinTable extends QRecordEntity
    @QField(possibleValueSourceName = QueryStat.TABLE_NAME)
    private Integer queryStatId;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(label = "Table", possibleValueSourceName = QQQTable.TABLE_NAME)
+   private Integer qqqTableId;
 
    @QField(maxLength = 10, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
    private String type;
@@ -132,31 +132,31 @@ public class QueryStatJoinTable extends QRecordEntity
 
 
    /*******************************************************************************
-    ** Getter for tableId
+    ** Getter for qqqTableId
     *******************************************************************************/
-   public Integer getTableId()
+   public Integer getQqqTableId()
    {
-      return (this.tableId);
+      return (this.qqqTableId);
    }
 
 
 
    /*******************************************************************************
-    ** Setter for tableId
+    ** Setter for qqqTableId
     *******************************************************************************/
-   public void setTableId(Integer tableId)
+   public void setQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
    }
 
 
 
    /*******************************************************************************
-    ** Fluent setter for tableId
+    ** Fluent setter for qqqTableId
     *******************************************************************************/
-   public QueryStatJoinTable withTableId(Integer tableId)
+   public QueryStatJoinTable withQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
       return (this);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java

@@ -121,7 +121,7 @@ public class QueryStatMetaDataProvider
          .withRecordLabelFields("id")
          .withPrimaryKeyField("id")
          .withFieldsFromEntity(QueryStat.class)
-         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "action", "tableId", "sessionId")))
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "action", "qqqTableId", "sessionId")))
          .withSection(new QFieldSection("data", new QIcon().withName("dataset"), Tier.T2, List.of("queryText", "startTimestamp", "firstResultTimestamp", "firstResultMillis")))
          .withSection(new QFieldSection("joins", new QIcon().withName("merge"), Tier.T2).withWidgetName(joinTablesJoinName + "Widget"))
          .withSection(new QFieldSection("criteria", new QIcon().withName("filter_alt"), Tier.T2).withWidgetName(criteriaFieldsJoinName + "Widget"))
@@ -187,7 +187,8 @@ public class QueryStatMetaDataProvider
       return (new QPossibleValueSource()
          .withType(QPossibleValueSourceType.TABLE)
          .withName(QueryStat.TABLE_NAME)
-         .withTableName(QueryStat.TABLE_NAME));
+         .withTableName(QueryStat.TABLE_NAME))
+         .withOrderByField("id", false);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatOrderByField.java

@@ -42,8 +42,8 @@ public class QueryStatOrderByField extends QRecordEntity
    @QField(possibleValueSourceName = QueryStat.TABLE_NAME)
    private Integer queryStatId;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(label = "Table", possibleValueSourceName = QQQTable.TABLE_NAME)
+   private Integer qqqTableId;
 
    @QField(maxLength = 50, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
    private String name;
@@ -132,31 +132,31 @@ public class QueryStatOrderByField extends QRecordEntity
 
 
    /*******************************************************************************
-    ** Getter for tableId
+    ** Getter for qqqTableId
     *******************************************************************************/
-   public Integer getTableId()
+   public Integer getQqqTableId()
    {
-      return (this.tableId);
+      return (this.qqqTableId);
    }
 
 
 
    /*******************************************************************************
-    ** Setter for tableId
+    ** Setter for qqqTableId
     *******************************************************************************/
-   public void setTableId(Integer tableId)
+   public void setQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
    }
 
 
 
    /*******************************************************************************
-    ** Fluent setter for tableId
+    ** Fluent setter for qqqTableId
     *******************************************************************************/
-   public QueryStatOrderByField withTableId(Integer tableId)
+   public QueryStatOrderByField withQqqTableId(Integer qqqTableId)
    {
-      this.tableId = tableId;
+      this.qqqTableId = qqqTableId;
       return (this);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedfilters/SavedFilter.java

@@ -27,7 +27,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
 
 
 /*******************************************************************************
@@ -49,8 +48,8 @@ public class SavedFilter extends QRecordEntity
    @QField(isRequired = true)
    private String label;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(isEditable = false)
+   private String tableName;
 
    @QField(isEditable = false)
    private String userId;
@@ -181,6 +180,40 @@ public class SavedFilter extends QRecordEntity
 
 
 
+   /*******************************************************************************
+    ** Getter for tableName
+    **
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    **
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    **
+    *******************************************************************************/
+   public SavedFilter withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for userId
     **
@@ -247,35 +280,4 @@ public class SavedFilter extends QRecordEntity
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for tableId
-    *******************************************************************************/
-   public Integer getTableId()
-   {
-      return (this.tableId);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for tableId
-    *******************************************************************************/
-   public void setTableId(Integer tableId)
-   {
-      this.tableId = tableId;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for tableId
-    *******************************************************************************/
-   public SavedFilter withTableId(Integer tableId)
-   {
-      this.tableId = tableId;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedfilters/SavedFiltersMetaDataProvider.java

@@ -88,7 +88,8 @@ public class SavedFiltersMetaDataProvider
          .withName(SavedFilter.TABLE_NAME)
          .withType(QPossibleValueSourceType.TABLE)
          .withTableName(SavedFilter.TABLE_NAME)
-         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
+         .withOrderByField("label");
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/Script.java

@@ -27,7 +27,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.data.QField;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider;
 
 
 /*******************************************************************************
@@ -52,8 +52,8 @@ public class Script extends QRecordEntity
    @QField(possibleValueSourceName = "scriptType")
    private Integer scriptTypeId;
 
-   @QField(possibleValueSourceName = QQQTable.TABLE_NAME, backendName = "qqq_table_id")
-   private Integer tableId;
+   @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME)
+   private String tableName;
 
    @QField()
    private Integer maxBatchSize;
@@ -288,6 +288,37 @@ public class Script extends QRecordEntity
 
 
 
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return (this.tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    *******************************************************************************/
+   public void setTableName(String tableName)
+   {
+      this.tableName = tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableName
+    *******************************************************************************/
+   public Script withTableName(String tableName)
+   {
+      this.tableName = tableName;
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     ** Getter for maxBatchSize
     *******************************************************************************/
@@ -317,35 +348,4 @@ public class Script extends QRecordEntity
       return (this);
    }
 
-
-
-   /*******************************************************************************
-    ** Getter for tableId
-    *******************************************************************************/
-   public Integer getTableId()
-   {
-      return (this.tableId);
-   }
-
-
-
-   /*******************************************************************************
-    ** Setter for tableId
-    *******************************************************************************/
-   public void setTableId(Integer tableId)
-   {
-      this.tableId = tableId;
-   }
-
-
-
-   /*******************************************************************************
-    ** Fluent setter for tableId
-    *******************************************************************************/
-   public Script withTableId(Integer tableId)
-   {
-      this.tableId = tableId;
-      return (this);
-   }
-
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java

@@ -66,7 +66,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.LoadScriptTestDetailsProcessStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptExtractStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptLoadStep;
-import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptPreStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.RunRecordScriptTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.StoreScriptRevisionProcessStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.scripts.TestScriptProcessStep;
@@ -97,11 +96,7 @@ public class ScriptsMetaDataProvider
       defineStandardScriptsPossibleValueSources(instance);
       defineStandardScriptsJoins(instance);
       defineStandardScriptsWidgets(instance);
-
-      // todo - change this from an enum-backed PVS to use qqqTable table, exposed in-app, in API
-      //  so api docs don't always need refreshed
       instance.addPossibleValueSource(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(instance));
-
       instance.addProcess(defineStoreScriptRevisionProcess());
       instance.addProcess(defineTestScriptProcess());
       instance.addProcess(defineLoadScriptTestDetailsProcess());
@@ -179,25 +174,15 @@ public class ScriptsMetaDataProvider
          .withLoadStepClass(RunRecordScriptLoadStep.class)
          .getProcessMetaData();
 
-      ////////////////////////////////////////////////////////////////////////////
-      // add a screen before the extract step - where user selects their script //
-      ////////////////////////////////////////////////////////////////////////////
       processMetaData.addStep(0, new QFrontendStepMetaData()
          .withName("input")
          .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
          .withFormField(new QFieldMetaData("scriptId", QFieldType.INTEGER).withPossibleValueSourceName(Script.TABLE_NAME)
             .withPossibleValueSourceFilter(new QQueryFilter(
                new QFilterCriteria("scriptType.name", QCriteriaOperator.EQUALS, SCRIPT_TYPE_NAME_RECORD),
-               new QFilterCriteria("tableId", QCriteriaOperator.EQUALS, "${input.tableId}")
+               new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, "${input.tableName}")
             ))));
 
-      /////////////////////////////////////////////////////////////////////////////////
-      // now - insert a step before the input screen, where the table name gets read //
-      /////////////////////////////////////////////////////////////////////////////////
-      processMetaData.addStep(0, new QBackendStepMetaData()
-         .withName("preStep")
-         .withCode(new QCodeReference(RunRecordScriptPreStep.class)));
-
       return (processMetaData);
    }
 
@@ -318,19 +303,24 @@ public class ScriptsMetaDataProvider
    {
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(Script.TABLE_NAME)
-         .withTableName(Script.TABLE_NAME));
+         .withTableName(Script.TABLE_NAME)
+         .withOrderByField("name"));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptRevision.TABLE_NAME)
-         .withTableName(ScriptRevision.TABLE_NAME));
+         .withTableName(ScriptRevision.TABLE_NAME)
+         .withOrderByField("scriptId")
+         .withOrderByField("sequenceNo", false));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptType.TABLE_NAME)
-         .withTableName(ScriptType.TABLE_NAME));
+         .withTableName(ScriptType.TABLE_NAME)
+         .withOrderByField("name"));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptLog.TABLE_NAME)
-         .withTableName(ScriptLog.TABLE_NAME));
+         .withTableName(ScriptLog.TABLE_NAME)
+         .withOrderByField("id", false));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptTypeFileMode.NAME)
@@ -398,16 +388,16 @@ public class ScriptsMetaDataProvider
       QTableMetaData tableMetaData = defineStandardTable(backendName, TableTrigger.TABLE_NAME, TableTrigger.class)
          .withRecordLabelFields("id")
          .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
-         .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2, List.of("tableId", "filterId", "scriptId", "priority", "postInsert", "postUpdate")))
+         .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2, List.of("tableName", "filterId", "scriptId", "priority", "postInsert", "postUpdate")))
          .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
 
       tableMetaData.getField("scriptId").withPossibleValueSourceFilter(new QQueryFilter(
          new QFilterCriteria("scriptType.name", QCriteriaOperator.EQUALS, SCRIPT_TYPE_NAME_RECORD),
-         new QFilterCriteria("script.tableId", QCriteriaOperator.EQUALS, "${input.tableId}")
+         new QFilterCriteria("script.tableName", QCriteriaOperator.EQUALS, "${input.tableName}")
       ));
 
       tableMetaData.getField("filterId").withPossibleValueSourceFilter(new QQueryFilter(
-         new QFilterCriteria("tableId", QCriteriaOperator.EQUALS, "${input.tableId}")
+         new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, "${input.tableName}")
       ));
 
       return tableMetaData;
@@ -422,7 +412,7 @@ public class ScriptsMetaDataProvider
    {
       QTableMetaData tableMetaData = defineStandardTable(backendName, Script.TABLE_NAME, Script.class)
          .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "name", "scriptTypeId", "currentScriptRevisionId")))
-         .withSection(new QFieldSection("recordScriptSettings", new QIcon().withName("table_rows"), Tier.T2, List.of("tableId", "maxBatchSize")))
+         .withSection(new QFieldSection("recordScriptSettings", new QIcon().withName("table_rows"), Tier.T2, List.of("tableName", "maxBatchSize")))
          .withSection(new QFieldSection("contents", new QIcon().withName("data_object"), Tier.T2).withWidgetName("scriptViewer"))
          .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
          .withSection(new QFieldSection("lines", new QIcon().withName("horizontal_rule"), Tier.T2).withWidgetName(QJoinMetaData.makeInferredJoinName(Script.TABLE_NAME, ScriptLog.TABLE_NAME)));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTableAccessor.java

@@ -1,137 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.model.tables;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
-
-
-/*******************************************************************************
- ** One-liner we can use to get a QQQTable record, or just its id (which we often want).
- ** Will insert the record if it wasn't already there.
- ** Also uses in-memory cache table, so rather cheap for normal use-case.
- *******************************************************************************/
-public class QQQTableAccessor
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QRecord getQQQTableRecord(String tableName) throws QException
-   {
-      /////////////////////////////
-      // look in the cache table //
-      /////////////////////////////
-      GetInput getInput = new GetInput();
-      getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME);
-      getInput.setUniqueKey(MapBuilder.of("name", tableName));
-      GetOutput getOutput = new GetAction().execute(getInput);
-
-      ////////////////////////
-      // upon cache miss... //
-      ////////////////////////
-      if(getOutput.getRecord() == null)
-      {
-         ///////////////////////////////////////////////////////
-         // insert the record (into the table, not the cache) //
-         ///////////////////////////////////////////////////////
-         QTableMetaData tableMetaData = QContext.getQInstance().getTable(tableName);
-         InsertInput    insertInput   = new InsertInput();
-         insertInput.setTableName(QQQTable.TABLE_NAME);
-         insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel())));
-         InsertOutput insertOutput = new InsertAction().execute(insertInput);
-
-         ///////////////////////////////////
-         // repeat the get from the cache //
-         ///////////////////////////////////
-         getOutput = new GetAction().execute(getInput);
-      }
-
-      return getOutput.getRecord();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static QRecord getQQQTableRecord(Integer id) throws QException
-   {
-      /////////////////////////////
-      // look in the cache table //
-      /////////////////////////////
-      GetInput getInput = new GetInput();
-      getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME);
-      getInput.setPrimaryKey(id);
-      GetOutput getOutput = new GetAction().execute(getInput);
-
-      ////////////////////////
-      // upon cache miss... //
-      ////////////////////////
-      if(getOutput.getRecord() == null)
-      {
-         GetInput sourceGetInput = new GetInput();
-         sourceGetInput.setTableName(QQQTable.TABLE_NAME);
-         sourceGetInput.setPrimaryKey(id);
-         GetOutput sourceGetOutput = new GetAction().execute(sourceGetInput);
-
-         ///////////////////////////////////
-         // repeat the get from the cache //
-         ///////////////////////////////////
-         getOutput = new GetAction().execute(sourceGetInput);
-      }
-
-      return getOutput.getRecord();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static Integer getTableId(String tableName) throws QException
-   {
-      return (getQQQTableRecord(tableName).getValueInteger("id"));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   public static String getTableName(Integer id) throws QException
-   {
-      return (getQQQTableRecord(id).getValueString("name"));
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java

@@ -50,20 +50,9 @@ public class QQQTablesMetaDataProvider
     *******************************************************************************/
    public void defineAll(QInstance instance, String persistentBackendName, String cacheBackendName, Consumer<QTableMetaData> backendDetailEnricher) throws QException
    {
-      if(instance.getTable(QQQTable.TABLE_NAME) == null)
-      {
-         instance.addTable(defineQQQTable(persistentBackendName, backendDetailEnricher));
-      }
-
-      if(instance.getTable(QQQ_TABLE_CACHE_TABLE_NAME) == null)
-      {
-         instance.addTable(defineQQQTableCache(cacheBackendName, backendDetailEnricher));
-      }
-
-      if(instance.getPossibleValueSource(QQQTable.TABLE_NAME) == null)
-      {
-         instance.addPossibleValueSource(defineQQQTablePossibleValueSource());
-      }
+      instance.addTable(defineQQQTable(persistentBackendName, backendDetailEnricher));
+      instance.addTable(defineQQQTableCache(cacheBackendName, backendDetailEnricher));
+      instance.addPossibleValueSource(defineQQQTablePossibleValueSource());
    }
 
 
@@ -138,8 +127,8 @@ public class QQQTablesMetaDataProvider
       return (new QPossibleValueSource()
          .withType(QPossibleValueSourceType.TABLE)
          .withName(QQQTable.TABLE_NAME)
-         .withOrderByField("label")
-         .withTableName(QQQTable.TABLE_NAME));
+         .withTableName(QQQTable.TABLE_NAME))
+         .withOrderByField("label");
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java

@@ -33,7 +33,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 import com.auth0.client.auth.AuthAPI;
 import com.auth0.exception.Auth0Exception;
 import com.auth0.json.auth.TokenHolder;
@@ -52,6 +51,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
@@ -68,11 +68,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.Auth0AuthenticationMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
+import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.SimpleStateKey;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
@@ -80,7 +80,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import org.apache.http.HttpStatus;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
@@ -90,9 +89,23 @@ import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.util.EntityUtils;
 import org.json.JSONArray;
 import org.json.JSONObject;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
+ ** QQQ AuthenticationModule for working with Auth0.
+ **
+ ** createSession can be called with the following fields in its context:
+ **
+ ** System-User session use-case:
+ ** 1: Takes in an "accessToken" (but doesn't store a userSession record).
+ ** 1b: legacy frontend use-case does the same as system-user!
+ **
+ ** Web User session use-cases:
+ ** 2: creates a new session (userSession record) by taking an "accessToken"
+ ** 3: looks up an existing session (userSession record) by taking a "sessionUUID"
+ ** 4: takes an "apiKey" (looked up in metaData.AccessTokenTableName - refreshing accessToken with auth0 if needed).
+ ** 5: takes a "basicAuthString" (encoded username:password), which make a new accessToken in auth0
  **
  *******************************************************************************/
 public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
@@ -104,14 +117,17 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
    /////////////////////////////////////////////////////////////////////////////////////////////////////////////
    public static final int ID_TOKEN_VALIDATION_INTERVAL_SECONDS = 1800;
 
-   public static final String AUTH0_ACCESS_TOKEN_KEY = "sessionId";
-   public static final String API_KEY                = "apiKey";
-   public static final String BASIC_AUTH_KEY         = "basicAuthString";
+   public static final String ACCESS_TOKEN_KEY = "accessToken";
+   public static final String API_KEY          = "apiKey"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
+   public static final String SESSION_UUID_KEY = "sessionUUID";
+   public static final String BASIC_AUTH_KEY   = "basicAuthString"; // todo - look for users of this, see if we can change to use this constant; maybe move constants up?
 
-   public static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
-   public static final String COULD_NOT_DECODE_ERROR   = "Unable to decode access token";
-   public static final String EXPIRED_TOKEN_ERROR      = "Token has expired";
-   public static final String INVALID_TOKEN_ERROR      = "An invalid token was provided";
+   public static final String DO_STORE_USER_SESSION_KEY = "doStoreUserSession";
+
+   static final String TOKEN_NOT_PROVIDED_ERROR = "Access Token was not provided";
+   static final String COULD_NOT_DECODE_ERROR   = "Unable to decode access token";
+   static final String EXPIRED_TOKEN_ERROR      = "Token has expired";
+   static final String INVALID_TOKEN_ERROR      = "An invalid token was provided";
 
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -149,94 +165,121 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
    @Override
    public QSession createSession(QInstance qInstance, Map<String, String> context) throws QAuthenticationException
    {
-      Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
-
-      ///////////////////////////////////////////////////////////
-      // check if we are processing a Basic Auth Session first //
-      ///////////////////////////////////////////////////////////
-      if(context.containsKey(BASIC_AUTH_KEY))
-      {
-         AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
-         try
-         {
-            /////////////////////////////////////////////////
-            // decode the credentials from the header auth //
-            /////////////////////////////////////////////////
-            String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
-            String accessToken       = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
-            context.put(AUTH0_ACCESS_TOKEN_KEY, accessToken);
-         }
-         catch(Auth0Exception e)
-         {
-            ////////////////
-            // ¯\_(ツ)_/¯ //
-            ////////////////
-            String message = "Error handling basic authentication: " + e.getMessage();
-            LOG.error(message, e);
-            throw (new QAuthenticationException(message));
-         }
-      }
-
-      ////////////////////////////////////////////////////////////////////
-      // get the jwt id or qqq translated token from the context object //
-      ////////////////////////////////////////////////////////////////////
-      String accessToken = context.get(AUTH0_ACCESS_TOKEN_KEY);
-
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      if(accessToken != null && StringUtils.isUUID(accessToken))
-      {
-         accessToken = lookupActualAccessToken(metaData, accessToken);
-      }
-
-      ////////////////////////////////////////////////////////
-      // if access token is still null, look for an api key //
-      ////////////////////////////////////////////////////////
-      if(accessToken == null)
-      {
-         String apiKey = context.get(API_KEY);
-         if(apiKey != null)
-         {
-            accessToken = getAccessTokenFromApiKey(metaData, apiKey);
-         }
-      }
-
-      if(accessToken == null)
-      {
-         LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
-         throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
-      }
-
-      //////////////////////////////////////////////////////////////////////////////////////
-      // decode the token locally to make sure it is valid and to look at when it expires //
-      //////////////////////////////////////////////////////////////////////////////////////
       try
       {
+         Auth0AuthenticationMetaData metaData = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
+
+         String accessToken = null;
+         if(CollectionUtils.containsKeyWithNonNullValue(context, SESSION_UUID_KEY))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////
+            // process a sessionUUID - looks up userSession record - cannot create token this way. //
+            /////////////////////////////////////////////////////////////////////////////////////////
+            String sessionUUID = context.get(SESSION_UUID_KEY);
+            LOG.info("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID)));
+            if(sessionUUID != null)
+            {
+               accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID);
+            }
+         }
+         else if(CollectionUtils.containsKeyWithNonNullValue(context, ACCESS_TOKEN_KEY))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if the context contains an access token, then create a new session based on that token.                          //
+            // todo#authHeader - this else/if should maybe be first, but while we have frontend passing both, we want it second //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            accessToken = context.get(ACCESS_TOKEN_KEY);
+            QSession qSession = buildAndValidateSession(qInstance, accessToken);
+
+            ////////////////////////////////////////////////////////////////
+            // build & store userSession db record, if requested to do so //
+            ////////////////////////////////////////////////////////////////
+            if(CollectionUtils.containsKeyWithNonNullValue(context, DO_STORE_USER_SESSION_KEY))
+            {
+               insertUserSession(qInstance, accessToken, qSession);
+               LOG.info("Creating session based on input accessToken and creating a userSession", logPair("userId", qSession.getUser().getIdReference()));
+            }
+            else
+            {
+               ///////////////////////////////////////////////
+               // todo#authHeader - remove all this logging //
+               ///////////////////////////////////////////////
+               String userName = qSession.getUser() != null ? qSession.getUser().getFullName() : null;
+               if(userName != null && !userName.contains("System User"))
+               {
+                  LOG.info("Creating session based on input accessToken but not creating a userSession", logPair("userName", qSession.getUser().getFullName()));
+               }
+            }
+
+            return (qSession);
+         }
+         else if(CollectionUtils.containsKeyWithNonNullValue(context, BASIC_AUTH_KEY))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////
+            // Process a basic auth (username:password)                                         //
+            // by getting an access token from auth0 (re-using from state provider if possible) //
+            //////////////////////////////////////////////////////////////////////////////////////
+            AuthAPI auth = AuthAPI.newBuilder(metaData.getBaseUrl(), metaData.getClientId(), metaData.getClientSecret()).build();
+            try
+            {
+               /////////////////////////////////////////////////
+               // decode the credentials from the header auth //
+               /////////////////////////////////////////////////
+               String base64Credentials = context.get(BASIC_AUTH_KEY).trim();
+               LOG.info("Creating session from basicAuthentication", logPair("base64Credentials", maskForLog(base64Credentials)));
+               accessToken = getAccessTokenFromBase64BasicAuthCredentials(metaData, auth, base64Credentials);
+            }
+            catch(Auth0Exception e)
+            {
+               ////////////////
+               // ¯\_(ツ)_/¯ //
+               ////////////////
+               String message = "Error handling basic authentication: " + e.getMessage();
+               LOG.error(message, e);
+               throw (new QAuthenticationException(message));
+            }
+         }
+         else if(CollectionUtils.containsKeyWithNonNullValue(context, API_KEY))
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////
+            // process an api key - looks up client application token (creating token if needed) //
+            ///////////////////////////////////////////////////////////////////////////////////////
+            String apiKey = context.get(API_KEY);
+            LOG.info("Creating session from apiKey (accessTokenTable)", logPair("apiKey", maskForLog(apiKey)));
+            if(apiKey != null)
+            {
+               accessToken = getAccessTokenFromApiKey(metaData, apiKey);
+            }
+         }
+
+         /* todo confirm this is deprecated
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // check to see if the session id is a UUID, if so, that means we need to look up the 'actual' token in the access_token table //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(accessToken != null && StringUtils.isUUID(accessToken))
+         {
+            accessToken = lookupActualAccessToken(metaData, accessToken);
+         }
+         */
+
+         ///////////////////////////////////////////
+         // if token wasn't found by now, give up //
+         ///////////////////////////////////////////
+         if(accessToken == null)
+         {
+            LOG.warn(TOKEN_NOT_PROVIDED_ERROR);
+            throw (new QAuthenticationException(TOKEN_NOT_PROVIDED_ERROR));
+         }
+
          /////////////////////////////////////////////////////
          // try to build session to see if still valid      //
          // then call method to check more session validity //
          /////////////////////////////////////////////////////
-         QSession qSession = buildQSessionFromToken(accessToken, qInstance);
-         if(isSessionValid(qInstance, qSession))
-         {
-            return (qSession);
-         }
-
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // if we make it here it means we have never validated this token or its been a long //
-         // enough duration so we need to re-verify the token                                 //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
-
-         ////////////////////////////////////////////////////////////////////
-         // put now into state so we dont check until next interval passes //
-         ///////////////////////////////////////////////////////////////////
-         StateProviderInterface spi = getStateProvider();
-         SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
-         spi.put(key, Instant.now());
-
-         return (qSession);
+         return buildAndValidateSession(qInstance, accessToken);
+      }
+      catch(QAuthenticationException qae)
+      {
+         throw (qae);
       }
       catch(JWTDecodeException jde)
       {
@@ -272,6 +315,61 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
 
+   /*******************************************************************************
+    ** Insert a session as a new record into userSession table
+    *******************************************************************************/
+   private void insertUserSession(QInstance qInstance, String accessToken, QSession qSession) throws QException
+   {
+      CapturedContext capturedContext = QContext.capture();
+      try
+      {
+         QContext.init(qInstance, null);
+         QContext.setQSession(getChickenAndEggSession());
+
+         UserSession userSession = new UserSession()
+            .withUuid(qSession.getUuid())
+            .withUserId(qSession.getUser().getIdReference())
+            .withAccessToken(accessToken);
+
+         new InsertAction().execute(new InsertInput(UserSession.TABLE_NAME).withRecordEntity(userSession));
+      }
+      finally
+      {
+         QContext.init(capturedContext);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QSession buildAndValidateSession(QInstance qInstance, String accessToken) throws JwkException
+   {
+      QSession qSession = buildQSessionFromToken(accessToken, qInstance);
+      if(isSessionValid(qInstance, qSession))
+      {
+         return (qSession);
+      }
+
+      //////////////////////////////////////////////////////////////////////////////////////////
+      // if we make it here it means we have never validated this token or it has been a long //
+      // enough duration so we need to re-verify the token                                    //
+      //////////////////////////////////////////////////////////////////////////////////////////
+      qSession = revalidateTokenAndBuildSession(qInstance, accessToken);
+
+      /////////////////////////////////////////////////////////////////////
+      // put now into state so we don't check until next interval passes //
+      /////////////////////////////////////////////////////////////////////
+      StateProviderInterface spi = getStateProvider();
+      SimpleStateKey<String> key = new SimpleStateKey<>(qSession.getIdReference());
+      spi.put(key, Instant.now());
+
+      return (qSession);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -299,7 +397,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
       byte[] credDecoded = Base64.getDecoder().decode(base64Credentials);
       String credentials = new String(credDecoded, StandardCharsets.UTF_8);
 
-      String accessToken = getAccessTokenFromAuth0(metaData, auth, credentials);
+      String accessToken = getAccessTokenForUsernameAndPasswordFromAuth0(metaData, auth, credentials);
       stateProvider.put(accessTokenStateKey, accessToken);
       stateProvider.put(timestampStateKey, Instant.now());
       return (accessToken);
@@ -310,7 +408,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected String getAccessTokenFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
+   protected String getAccessTokenForUsernameAndPasswordFromAuth0(Auth0AuthenticationMetaData metaData, AuthAPI auth, String credentials) throws Auth0Exception
    {
       /////////////////////////////////////
       // call auth0 with a login request //
@@ -620,75 +718,11 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
    /*******************************************************************************
-    ** create a new auth0 access token
+    ** make http request to Auth0 for a new access token for an application - e.g.,
+    ** with a clientId and clientSecret as params
     **
     *******************************************************************************/
-   public String createAccessToken(QAuthenticationMetaData metaData, String clientId, String clientSecret) throws AccessTokenException
-   {
-      QSession                    sessionBefore = QContext.getQSession();
-      Auth0AuthenticationMetaData auth0MetaData = (Auth0AuthenticationMetaData) metaData;
-
-      try
-      {
-         QContext.setQSession(getChickenAndEggSession());
-
-         ///////////////////////////////////////////////////////////////////////////////////////
-         // fetch the application from database, will throw accesstokenexception if not found //
-         ///////////////////////////////////////////////////////////////////////////////////////
-         QRecord clientAuth0Application = getClientAuth0Application(auth0MetaData, clientId);
-
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         // request access token from auth0 if exception is not thrown, that means 200OK, we want to    //
-         // store the actual access token in the database, and return a unique value                    //
-         // back to the user which will be what they use on subseqeunt requests (because token too big) //
-         /////////////////////////////////////////////////////////////////////////////////////////////////
-         JSONObject accessTokenData = requestAccessTokenFromAuth0(auth0MetaData, clientId, clientSecret);
-
-         Integer expiresInSeconds = accessTokenData.getInt("expires_in");
-         String  accessToken      = accessTokenData.getString("access_token");
-         String  uuid             = UUID.randomUUID().toString();
-
-         /////////////////////////////////
-         // store the details in the db //
-         /////////////////////////////////
-         QRecord accessTokenRecord = new QRecord()
-            .withValue(auth0MetaData.getClientAuth0ApplicationIdField(), clientAuth0Application.getValue("id"))
-            .withValue(auth0MetaData.getAuth0AccessTokenField(), accessToken)
-            .withValue(auth0MetaData.getQqqAccessTokenField(), uuid)
-            .withValue(auth0MetaData.getExpiresInSecondsField(), expiresInSeconds);
-         InsertInput input = new InsertInput();
-         input.setTableName(auth0MetaData.getAccessTokenTableName());
-         input.setRecords(List.of(accessTokenRecord));
-         new InsertAction().execute(input);
-
-         //////////////////////////////////
-         // update and send the response //
-         //////////////////////////////////
-         accessTokenData.put("access_token", uuid);
-         accessTokenData.remove("scope");
-         return (accessTokenData.toString());
-      }
-      catch(AccessTokenException ate)
-      {
-         throw (ate);
-      }
-      catch(Exception e)
-      {
-         throw (new AccessTokenException(e.getMessage(), e));
-      }
-      finally
-      {
-         QContext.setQSession(sessionBefore);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** make http request to Auth0 for a new access token
-    **
-    *******************************************************************************/
-   public JSONObject requestAccessTokenFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
+   public JSONObject requestAccessTokenForClientIdAndSecretFromAuth0(Auth0AuthenticationMetaData auth0MetaData, String clientId, String clientSecret) throws AccessTokenException
    {
       ///////////////////////////////////////////////////////////////////
       // make a request to Auth0 using the client_id and client_secret //
@@ -776,6 +810,63 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
 
+   /*******************************************************************************
+    ** Look up access_token from session UUID
+    **
+    *******************************************************************************/
+   private String getAccessTokenFromSessionUUID(Auth0AuthenticationMetaData metaData, String sessionUUID) throws QAuthenticationException
+   {
+      String   accessToken   = null;
+      QSession beforeSession = QContext.getQSession();
+
+      try
+      {
+         QContext.setQSession(getChickenAndEggSession());
+
+         ///////////////////////////////////////
+         // query for the user session record //
+         ///////////////////////////////////////
+         QRecord userSessionRecord = new GetAction().executeForRecord(new GetInput(UserSession.TABLE_NAME)
+            .withUniqueKey(Map.of("uuid", sessionUUID))
+            .withShouldMaskPasswords(false)
+            .withShouldOmitHiddenFields(false));
+
+         if(userSessionRecord != null)
+         {
+            accessToken = userSessionRecord.getValueString("accessToken");
+
+            ////////////////////////////////////////////////////////////
+            // decode the accessToken and make sure it is not expired //
+            ////////////////////////////////////////////////////////////
+            if(accessToken != null)
+            {
+               DecodedJWT jwt = JWT.decode(accessToken);
+               if(jwt.getExpiresAtAsInstant().isBefore(Instant.now()))
+               {
+                  throw (new QAuthenticationException("accessToken is expired"));
+               }
+            }
+         }
+      }
+      catch(QAuthenticationException qae)
+      {
+         throw (qae);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error looking up userSession by sessionUUID", e);
+         throw (new QAuthenticationException("Error looking up userSession by sessionUUID", e));
+      }
+      finally
+      {
+         QContext.setQSession(beforeSession);
+      }
+
+      return (accessToken);
+   }
+
+
+
    /*******************************************************************************
     ** Look up access_token from api key
     **
@@ -841,7 +932,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
                // store the actual access token in the database, and return a unique value                    //
                // back to the user which will be what they use on subsequent requests (because token too big) //
                /////////////////////////////////////////////////////////////////////////////////////////////////
-               JSONObject accessTokenData = requestAccessTokenFromAuth0(metaData, clientId, clientSecret);
+               JSONObject accessTokenData = requestAccessTokenForClientIdAndSecretFromAuth0(metaData, clientId, clientSecret);
 
                Integer expiresInSeconds = accessTokenData.getInt("expires_in");
                accessToken = accessTokenData.getString("access_token");
@@ -872,25 +963,23 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface
 
 
    /*******************************************************************************
-    ** Look up client_auth0_application record, return if found.
     **
     *******************************************************************************/
-   QRecord getClientAuth0Application(Auth0AuthenticationMetaData metaData, String clientId) throws QException
+   static String maskForLog(String input)
    {
-      //////////////////////////////////////////////////////////////////////////////////////
-      // try to look up existing auth0 application from database, insert one if not found //
-      //////////////////////////////////////////////////////////////////////////////////////
-      QueryInput queryInput = new QueryInput();
-      queryInput.setTableName(metaData.getClientAuth0ApplicationTableName());
-      queryInput.setFilter(new QQueryFilter(new QFilterCriteria(metaData.getAuth0ClientIdField(), QCriteriaOperator.EQUALS, clientId)));
-      QueryOutput queryOutput = new QueryAction().execute(queryInput);
-
-      if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords()))
+      if(input == null)
       {
-         return (queryOutput.getRecords().get(0));
+         return (null);
       }
 
-      throw (new AccessTokenException("This client has not been configured to use the API.", HttpStatus.SC_UNAUTHORIZED));
+      if(input.length() < 8)
+      {
+         return ("******");
+      }
+      else
+      {
+         return (input.substring(0, 6) + "******");
+      }
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/metadata/UserSessionMetaDataProducer.java

@@ -0,0 +1,73 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.modules.authentication.implementations.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.modules.authentication.implementations.model.UserSession;
+
+
+/*******************************************************************************
+ ** Meta Data Producer for UserSession
+ *******************************************************************************/
+public class UserSessionMetaDataProducer extends MetaDataProducer<QTableMetaData>
+{
+   private final String backendName;
+
+
+
+   /*******************************************************************************
+    ** Constructor
+    **
+    *******************************************************************************/
+   public UserSessionMetaDataProducer(String backendName)
+   {
+      this.backendName = backendName;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      QTableMetaData tableMetaData = new QTableMetaData()
+         .withName(UserSession.TABLE_NAME)
+         .withBackendName(backendName)
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields("id")
+         .withPrimaryKeyField("id")
+         .withUniqueKey(new UniqueKey("uuid"))
+         .withFieldsFromEntity(UserSession.class)
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE));
+      return tableMetaData;
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/model/UserSession.java

@@ -0,0 +1,262 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.modules.authentication.implementations.model;
+
+
+import java.time.Instant;
+import com.kingsrook.qqq.backend.core.model.data.QField;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
+
+
+/*******************************************************************************
+ ** QRecord Entity for UserSession table
+ *******************************************************************************/
+public class UserSession extends QRecordEntity
+{
+   public static final String TABLE_NAME = "userSession";
+
+   @QField(isEditable = false)
+   private Integer id;
+
+   @QField(isEditable = false)
+   private Instant createDate;
+
+   @QField(isEditable = false)
+   private Instant modifyDate;
+
+   @QField(isEditable = false, isHidden = true, maxLength = 40, valueTooLongBehavior = ValueTooLongBehavior.ERROR)
+   private String uuid;
+
+   @QField(isEditable = false, isHidden = true)
+   private String accessToken;
+
+   @QField(isEditable = false, maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS)
+   private String userId;
+
+
+
+   /*******************************************************************************
+    ** Default constructor
+    *******************************************************************************/
+   public UserSession()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor that takes a QRecord
+    *******************************************************************************/
+   public UserSession(QRecord record)
+   {
+      populateFromQRecord(record);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for id
+    *******************************************************************************/
+   public Integer getId()
+   {
+      return (this.id);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for id
+    *******************************************************************************/
+   public void setId(Integer id)
+   {
+      this.id = id;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for id
+    *******************************************************************************/
+   public UserSession withId(Integer id)
+   {
+      this.id = id;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDate
+    *******************************************************************************/
+   public Instant getCreateDate()
+   {
+      return (this.createDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDate
+    *******************************************************************************/
+   public void setCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDate
+    *******************************************************************************/
+   public UserSession withCreateDate(Instant createDate)
+   {
+      this.createDate = createDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDate
+    *******************************************************************************/
+   public Instant getModifyDate()
+   {
+      return (this.modifyDate);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDate
+    *******************************************************************************/
+   public void setModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDate
+    *******************************************************************************/
+   public UserSession withModifyDate(Instant modifyDate)
+   {
+      this.modifyDate = modifyDate;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for uuid
+    *******************************************************************************/
+   public String getUuid()
+   {
+      return (this.uuid);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for uuid
+    *******************************************************************************/
+   public void setUuid(String uuid)
+   {
+      this.uuid = uuid;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for uuid
+    *******************************************************************************/
+   public UserSession withUuid(String uuid)
+   {
+      this.uuid = uuid;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for accessToken
+    *******************************************************************************/
+   public String getAccessToken()
+   {
+      return (this.accessToken);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for accessToken
+    *******************************************************************************/
+   public void setAccessToken(String accessToken)
+   {
+      this.accessToken = accessToken;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for accessToken
+    *******************************************************************************/
+   public UserSession withAccessToken(String accessToken)
+   {
+      this.accessToken = accessToken;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for userId
+    *******************************************************************************/
+   public String getUserId()
+   {
+      return (this.userId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for userId
+    *******************************************************************************/
+   public void setUserId(String userId)
+   {
+      this.userId = userId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for userId
+    *******************************************************************************/
+   public UserSession withUserId(String userId)
+   {
+      this.userId = userId;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java

@@ -92,6 +92,7 @@ public class StreamedETLBackendStep implements BackendStep
          ////////////////////////////////////////////////////////////////////////////////
          // rollback the work, then re-throw the error for up-stream to catch & report //
          ////////////////////////////////////////////////////////////////////////////////
+         LOG.warn("Caught top-level process exception - rolling back transaction", e);
          transaction.rollback();
          throw (e);
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java

@@ -198,8 +198,13 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          ////////////////////////////////////////////////////////////////////////////////
          if(transaction.isPresent())
          {
+            LOG.warn("Caught top-level process exception - rolling back transaction", e);
             transaction.get().rollback();
          }
+         else
+         {
+            LOG.warn("Caught top-level process exception - would roll back transaction, but none is present", e);
+         }
          throw (e);
       }
       finally
@@ -302,6 +307,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
       {
          if(doPageLevelTransaction && transaction.isPresent())
          {
+            LOG.warn("Caught page-level process exception - rolling back transaction", e);
             transaction.get().rollback();
          }
          throw (e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/QuerySavedFilterProcess.java

@@ -44,7 +44,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 
 
 /*******************************************************************************
@@ -101,7 +100,7 @@ public class QuerySavedFilterProcess implements BackendStep
             QueryInput input = new QueryInput();
             input.setTableName(SavedFilter.TABLE_NAME);
             input.setFilter(new QQueryFilter()
-               .withCriteria(new QFilterCriteria("tableId", QCriteriaOperator.EQUALS, QQQTableAccessor.getTableId(tableName)))
+               .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName))
                .withOrderBy(new QFilterOrderBy("label")));
 
             QueryOutput output = new QueryAction().execute(input);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/StoreSavedFilterProcess.java

@@ -42,7 +42,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFilter;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 
 
 /*******************************************************************************
@@ -83,7 +82,7 @@ public class StoreSavedFilterProcess implements BackendStep
          QRecord qRecord = new QRecord()
             .withValue("id", runBackendStepInput.getValueInteger("id"))
             .withValue("label", runBackendStepInput.getValueString("label"))
-            .withValue("tableId", QQQTableAccessor.getTableId(runBackendStepInput.getValueString("tableName")))
+            .withValue("tableName", runBackendStepInput.getValueString("tableName"))
             .withValue("filterJson", runBackendStepInput.getValueString("filterJson"))
             .withValue("userId", runBackendStepInput.getSession().getUser().getIdReference());
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptExtractStep.java

@@ -30,7 +30,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
@@ -59,11 +58,6 @@ public class RunRecordScriptExtractStep extends ExtractViaQueryStep
 
       runBackendStepInput.addValue(FIELD_SOURCE_TABLE, tableName);
 
-      /////////////////////////////////////////////////////////////////
-      // set this value, for the select-script possible-value filter //
-      /////////////////////////////////////////////////////////////////
-      runBackendStepInput.addValue("tableId", QQQTableAccessor.getTableId(tableName));
-
       Integer  scriptId = runBackendStepInput.getValueInteger("scriptId");
       GetInput getInput = new GetInput();
       getInput.setTableName(Script.TABLE_NAME);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptPreStep.java

@@ -1,64 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.scripts;
-
-
-import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-
-
-/*******************************************************************************
- ** pre-step for the run-record-script process.  Help deal with this being
- ** a generic process (e.g., no table name defined in the meta data).
- *******************************************************************************/
-public class RunRecordScriptPreStep implements BackendStep
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Override
-   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
-   {
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      // this is a generic (e.g., not table-specific) process - so we must be sure to set the tableName field in the expected slot. //
-      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      String tableName = runBackendStepInput.getValueString("tableName");
-      if(!StringUtils.hasContent(tableName))
-      {
-         throw (new QException("Table name was not specified as input value"));
-      }
-
-      runBackendStepInput.addValue(StreamedETLProcess.FIELD_SOURCE_TABLE, tableName);
-
-      /////////////////////////////////////////////////////////////////
-      // set this value, for the select-script possible-value filter //
-      /////////////////////////////////////////////////////////////////
-      runBackendStepInput.addValue("tableId", QQQTableAccessor.getTableId(tableName));
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java

@@ -31,7 +31,10 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.helpers.ValidateRecordSecurityLockHelper;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
@@ -46,6 +49,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.scripts.Script;
+import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -69,7 +74,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
    {
       InsertAction insertAction = new InsertAction();
       InsertInput  insertInput  = new InsertInput();
-      insertInput.setTableName("scriptRevision");
+      insertInput.setTableName(ScriptRevision.TABLE_NAME);
       QBackendTransaction transaction = insertAction.openTransaction(insertInput);
       insertInput.setTransaction(transaction);
 
@@ -87,14 +92,23 @@ public class StoreScriptRevisionProcessStep implements BackendStep
          // get the existing script, to update //
          ////////////////////////////////////////
          GetInput getInput = new GetInput();
-         getInput.setTableName("script");
+         getInput.setTableName(Script.TABLE_NAME);
          getInput.setPrimaryKey(scriptId);
          getInput.setTransaction(transaction);
          GetOutput getOutput = new GetAction().execute(getInput);
          QRecord   script    = getOutput.getRecord();
 
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // in case the app added a security field to the scripts table, make sure the user is allowed to edit the script //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE);
+         if(CollectionUtils.nullSafeHasContents(script.getErrors()))
+         {
+            throw (new QPermissionDeniedException(script.getErrors().get(0).getMessage()));
+         }
+
          QueryInput queryInput = new QueryInput();
-         queryInput.setTableName("scriptRevision");
+         queryInput.setTableName(ScriptRevision.TABLE_NAME);
          queryInput.setFilter(new QQueryFilter()
             .withCriteria(new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, List.of(script.getValue("id"))))
             .withOrderBy(new QFilterOrderBy("sequenceNo", false))
@@ -183,7 +197,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
          ////////////////////////////////////////////////////
          script.setValue("currentScriptRevisionId", scriptRevision.getValue("id"));
          UpdateInput updateInput = new UpdateInput();
-         updateInput.setTableName("script");
+         updateInput.setTableName(Script.TABLE_NAME);
          updateInput.setRecords(List.of(script));
          updateInput.setTransaction(transaction);
          new UpdateAction().execute(updateInput);
@@ -198,6 +212,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep
       catch(Exception e)
       {
          transaction.rollback();
+         throw (e);
       }
       finally
       {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/CollectionUtils.java

@@ -663,4 +663,19 @@ public class CollectionUtils
       return (output);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static <K> boolean containsKeyWithNonNullValue(Map<K, ?> map, K key)
+   {
+      if(map == null)
+      {
+         return (false);
+      }
+
+      return (map.containsKey(key) && map.get(key) != null);
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/AuditActionTest.java

@@ -36,7 +36,6 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.processes.utils.GeneralProcessUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
@@ -60,7 +59,6 @@ class AuditActionTest extends BaseTest
    {
       QInstance qInstance = TestUtils.defineInstance();
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
       String userName = "John Doe";
       QContext.init(qInstance, new QSession().withUser(new QUser().withFullName(userName)));
@@ -71,6 +69,7 @@ class AuditActionTest extends BaseTest
       /////////////////////////////////////
       // make sure things can be fetched //
       /////////////////////////////////////
+      GeneralProcessUtils.getRecordByFieldOrElseThrow("auditTable", "name", TestUtils.TABLE_NAME_PERSON_MEMORY);
       GeneralProcessUtils.getRecordByFieldOrElseThrow("auditUser", "name", userName);
       QRecord auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId);
       assertEquals("Test Audit", auditRecord.getValueString("message"));
@@ -86,7 +85,6 @@ class AuditActionTest extends BaseTest
    {
       QInstance qInstance = TestUtils.defineInstance();
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
       String userName = "John Doe";
       QContext.init(qInstance, new QSession().withUser(new QUser().withFullName(userName)));
@@ -125,7 +123,6 @@ class AuditActionTest extends BaseTest
    {
       QInstance qInstance = TestUtils.defineInstance();
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
       String userName = "John Doe";
       QContext.init(qInstance, new QSession().withUser(new QUser().withFullName(userName)));
@@ -140,6 +137,7 @@ class AuditActionTest extends BaseTest
       /////////////////////////////////////
       // make sure things can be fetched //
       /////////////////////////////////////
+      GeneralProcessUtils.getRecordByFieldOrElseThrow("auditTable", "name", TestUtils.TABLE_NAME_PERSON_MEMORY);
       GeneralProcessUtils.getRecordByFieldOrElseThrow("auditUser", "name", userName);
       QRecord auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId1);
       assertEquals("Test Audit", auditRecord.getValueString("message"));
@@ -159,7 +157,6 @@ class AuditActionTest extends BaseTest
    {
       QInstance qInstance = TestUtils.defineInstance();
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
       String userName = "John Doe";
       QContext.init(qInstance, new QSession().withUser(new QUser().withFullName(userName)));
@@ -176,6 +173,7 @@ class AuditActionTest extends BaseTest
       /////////////////////////////////////
       // make sure things can be fetched //
       /////////////////////////////////////
+      GeneralProcessUtils.getRecordByFieldOrElseThrow("auditTable", "name", TestUtils.TABLE_NAME_PERSON_MEMORY);
       GeneralProcessUtils.getRecordByFieldOrElseThrow("auditUser", "name", userName);
       QRecord auditRecord = GeneralProcessUtils.getRecordByFieldOrElseThrow("audit", "recordId", recordId1);
       assertEquals("Test Audit", auditRecord.getValueString("message"));

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditActionTest.java

@@ -37,7 +37,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
 import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
@@ -61,7 +60,6 @@ class DMLAuditActionTest extends BaseTest
    {
       QInstance qInstance = QContext.getQInstance();
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
       UpdateInput updateInput = new UpdateInput();
       updateInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteActionTest.java

@@ -49,11 +49,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -125,7 +125,6 @@ class DeleteActionTest extends BaseTest
       new InsertAction().execute(insertInput);
 
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
       qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).setAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD));
 
       DeleteInput deleteInput = new DeleteInput();
@@ -156,7 +155,6 @@ class DeleteActionTest extends BaseTest
       new InsertAction().execute(insertInput);
 
       new AuditsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
       qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).setAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD));
 
       DeleteInput deleteInput = new DeleteInput();
@@ -368,6 +366,36 @@ class DeleteActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityLockWriteScope() throws QException
+   {
+      TestUtils.updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords();
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // try to delete 1, 2, and 3.  2 should be blocked, because it has a writable-By that isn't in our session //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe")));
+      DeleteInput deleteInput = new DeleteInput();
+      deleteInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      deleteInput.setPrimaryKeys(List.of(1, 2, 3));
+      DeleteOutput deleteOutput = new DeleteAction().execute(deleteInput);
+
+      assertEquals(1, deleteOutput.getRecordsWithErrors().size());
+      assertThat(deleteOutput.getRecordsWithErrors().get(0).getErrors().get(0).getMessage())
+         .contains("You do not have permission")
+         .contains("kmarsh")
+         .contains("Only Writable By");
+
+      assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)).getCount());
+      assertEquals(1, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)
+         .withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 2)))).getCount());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java

@@ -35,9 +35,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -692,4 +695,60 @@ class InsertActionTest extends BaseTest
       assertEquals(3, TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_ORDER).size());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityLockWriteScope() throws QException
+   {
+      TestUtils.updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords();
+
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("hsimpson")));
+
+      /////////////////////////////////////////////////////////////////////////////////////////
+      // with only hsimpson in our key, make sure we can't insert a row w/ a different value //
+      /////////////////////////////////////////////////////////////////////////////////////////
+      {
+         InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 100).withValue("firstName", "Jean-Luc").withValue("onlyWritableBy", "jkirk")
+         )));
+         List<QErrorMessage> errors = insertOutput.getRecords().get(0).getErrors();
+         assertEquals(1, errors.size());
+         assertThat(errors.get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("jkirk")
+            .contains("Only Writable By");
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure we can insert w/ a null in onlyWritableBy (because key (from test utils) was set to allow null) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 101).withValue("firstName", "Benajamin").withValue("onlyWritableBy", null)
+         )));
+         List<QErrorMessage> errors = insertOutput.getRecords().get(0).getErrors();
+         assertEquals(0, errors.size());
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////
+      // change the null behavior to deny, and try above again, expecting an error //
+      ///////////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY);
+      {
+         InsertOutput insertOutput = new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 102).withValue("firstName", "Katherine").withValue("onlyWritableBy", null)
+         )));
+         List<QErrorMessage> errors = insertOutput.getRecords().get(0).getErrors();
+         assertEquals(1, errors.size());
+         assertThat(errors.get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("without a value")
+            .contains("Only Writable By");
+      }
+
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateActionTest.java

@@ -29,12 +29,18 @@ import java.util.Objects;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
@@ -492,7 +498,7 @@ class UpdateActionTest extends BaseTest
          updateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM);
          updateInput.setRecords(List.of(new QRecord().withValue("id", 20).withValue("sku", "BASIC3")));
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-         assertEquals("No record was found to update for Id = 20", updateOutput.getRecords().get(0).getErrors().get(0).getMessage());
+         assertTrue(updateOutput.getRecords().get(0).getErrors().stream().anyMatch(em -> em.getMessage().equals("No record was found to update for Id = 20")));
       }
 
       ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -504,7 +510,7 @@ class UpdateActionTest extends BaseTest
          updateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM);
          updateInput.setRecords(List.of(new QRecord().withValue("id", 10).withValue("orderId", 2).withValue("sku", "BASIC3")));
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-         assertEquals("You do not have permission to update this record - the referenced Order was not found.", updateOutput.getRecords().get(0).getErrors().get(0).getMessage());
+         assertTrue(updateOutput.getRecords().get(0).getErrors().stream().anyMatch(em -> em.getMessage().equals("You do not have permission to update this record - the referenced Order was not found.")));
       }
 
       ///////////////////////////////////////////////////////////
@@ -528,7 +534,7 @@ class UpdateActionTest extends BaseTest
          updateInput.setTableName(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC);
          updateInput.setRecords(List.of(new QRecord().withValue("id", 200).withValue("key", "updatedKey")));
          UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-         assertEquals("No record was found to update for Id = 200", updateOutput.getRecords().get(0).getErrors().get(0).getMessage());
+         assertTrue(updateOutput.getRecords().get(0).getErrors().stream().anyMatch(em -> em.getMessage().equals("No record was found to update for Id = 200")));
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -706,4 +712,79 @@ class UpdateActionTest extends BaseTest
       assertEquals(1, TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER).stream().filter(r -> r.getValue("storeId") == null).count());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSecurityLockWriteScope() throws QException
+   {
+      TestUtils.updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords();
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // try to update them all 1, 2, and 3.  2 should be blocked, because it has a writable-By that isn't in our session //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe")));
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("lastName", "Kelkhoff"),
+            new QRecord().withValue("id", 2).withValue("lastName", "Chamberlain"),
+            new QRecord().withValue("id", 3).withValue("lastName", "Maes")
+         )));
+
+         List<QRecord> errorRecords = updateOutput.getRecords().stream().filter(r -> CollectionUtils.nullSafeHasContents(r.getErrors())).toList();
+         assertEquals(1, errorRecords.size());
+         assertEquals(2, errorRecords.get(0).getValueInteger("id"));
+         assertThat(errorRecords.get(0).getErrors().get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("kmarsh")
+            .contains("Only Writable By");
+
+         assertEquals(2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)
+            .withFilter(new QQueryFilter(new QFilterCriteria("lastName", QCriteriaOperator.IS_NOT_BLANK)))).getCount());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // now try to change one of the records to have a different value in the lock-field.  Should fail (as it's not in our session) //
+      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      {
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("onlyWritableBy", "ecartman"))));
+
+         List<QRecord> errorRecords = updateOutput.getRecords().stream().filter(r -> CollectionUtils.nullSafeHasContents(r.getErrors())).toList();
+         assertEquals(1, errorRecords.size());
+         assertThat(errorRecords.get(0).getErrors().get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("ecartman")
+            .contains("Only Writable By");
+      }
+
+      ///////////////////////////////////////////////////////////////
+      // add that to our session and confirm we can do that update //
+      ///////////////////////////////////////////////////////////////
+      {
+         QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe", "ecartman")));
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("onlyWritableBy", "ecartman"))));
+         List<QRecord> errorRecords = updateOutput.getRecords().stream().filter(r -> CollectionUtils.nullSafeHasContents(r.getErrors())).toList();
+         assertEquals(0, errorRecords.size());
+      }
+
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      // change the null behavior to deny, then try to udpate a record and remove its onlyWritableBy //
+      /////////////////////////////////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY);
+      {
+         UpdateOutput updateOutput = new UpdateAction().execute(new UpdateInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+            new QRecord().withValue("id", 1).withValue("onlyWritableBy", null))));
+         List<QErrorMessage> errors = updateOutput.getRecords().get(0).getErrors();
+         assertEquals(1, errors.size());
+         assertThat(errors.get(0).getMessage())
+            .contains("You do not have permission")
+            .contains("without a value")
+            .contains("Only Writable By");
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -822,6 +822,58 @@ class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSectionsWithJoinFields()
+   {
+      Consumer<QTableMetaData> putAllFieldsInASection = table -> table.addSection(new QFieldSection()
+         .withName("section0")
+         .withTier(Tier.T1)
+         .withFieldNames(new ArrayList<>(table.getFields().keySet())));
+
+      assertValidationFailureReasons(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_ORDER);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_LINE_ITEM + ".sku");
+      }, "orderLine.sku references an is-many join, which is not supported");
+
+      assertValidationSuccess(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_ORDER + ".orderNo");
+      });
+
+      assertValidationFailureReasons(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_ORDER + ".asdf");
+      }, "order.asdf specifies a fieldName [asdf] which does not exist in that table [order].");
+
+      /////////////////////////////////////////////////////////////////////////////
+      // this is aactually allowed, well, just not considered as a join-field... //
+      /////////////////////////////////////////////////////////////////////////////
+      // assertValidationFailureReasons(qInstance ->
+      // {
+      //    QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+      //    putAllFieldsInASection.accept(table);
+      //    table.getSections().get(0).getFieldNames().add("foo.bar");
+      // }, "unrecognized table name [foo]");
+
+      assertValidationFailureReasons(qInstance ->
+      {
+         QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_LINE_ITEM);
+         putAllFieldsInASection.accept(table);
+         table.getSections().get(0).getFieldNames().add(TestUtils.TABLE_NAME_SHAPE + ".id");
+      }, "[shape] which is not an exposed join on this table");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java

@@ -273,6 +273,52 @@ class QMetaDataVariableInterpreterTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetIntegerFromPropertyOrEnvironment()
+   {
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+
+      //////////////////////////////////////////////////////////
+      // if neither prop nor env is set, get back the default //
+      //////////////////////////////////////////////////////////
+      assertEquals(1, interpreter.getIntegerFromPropertyOrEnvironment("notSet", "NOT_SET", 1));
+      assertEquals(2, interpreter.getIntegerFromPropertyOrEnvironment("notSet", "NOT_SET", 2));
+
+      /////////////////////////////////////////////
+      // unrecognized values are same as not set //
+      /////////////////////////////////////////////
+      System.setProperty("unrecognized", "asdf");
+      interpreter.setEnvironmentOverrides(Map.of("UNRECOGNIZED", "qwerty"));
+      assertEquals(3, interpreter.getIntegerFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", 3));
+      assertEquals(4, interpreter.getIntegerFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", 4));
+
+      /////////////////////////////////
+      // if only prop is set, get it //
+      /////////////////////////////////
+      assertEquals(5, interpreter.getIntegerFromPropertyOrEnvironment("foo.size", "FOO_SIZE", 5));
+      System.setProperty("foo.size", "6");
+      assertEquals(6, interpreter.getIntegerFromPropertyOrEnvironment("foo.size", "FOO_SIZE", 7));
+
+      ////////////////////////////////
+      // if only env is set, get it //
+      ////////////////////////////////
+      assertEquals(8, interpreter.getIntegerFromPropertyOrEnvironment("bar.size", "BAR_SIZE", 8));
+      interpreter.setEnvironmentOverrides(Map.of("BAR_SIZE", "9"));
+      assertEquals(9, interpreter.getIntegerFromPropertyOrEnvironment("bar.size", "BAR_SIZE", 10));
+
+      ///////////////////////////////////
+      // if both are set, get the prop //
+      ///////////////////////////////////
+      System.setProperty("baz.size", "11");
+      interpreter.setEnvironmentOverrides(Map.of("BAZ_SIZE", "12"));
+      assertEquals(11, interpreter.getIntegerFromPropertyOrEnvironment("baz.size", "BAZ_SIZE", 13));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModuleTest.java

@@ -42,15 +42,17 @@ import com.kingsrook.qqq.backend.core.state.SimpleStateKey;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
-import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.AUTH0_ACCESS_TOKEN_KEY;
+import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.ACCESS_TOKEN_KEY;
 import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.BASIC_AUTH_KEY;
 import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.COULD_NOT_DECODE_ERROR;
 import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.EXPIRED_TOKEN_ERROR;
 import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.INVALID_TOKEN_ERROR;
 import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.TOKEN_NOT_PROVIDED_ERROR;
+import static com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule.maskForLog;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
@@ -143,7 +145,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
    public void testInvalidToken()
    {
       Map<String, String> context = new HashMap<>();
-      context.put(AUTH0_ACCESS_TOKEN_KEY, INVALID_TOKEN);
+      context.put(ACCESS_TOKEN_KEY, INVALID_TOKEN);
 
       try
       {
@@ -167,7 +169,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
    public void testUndecodableToken()
    {
       Map<String, String> context = new HashMap<>();
-      context.put(AUTH0_ACCESS_TOKEN_KEY, UNDECODABLE_TOKEN);
+      context.put(ACCESS_TOKEN_KEY, UNDECODABLE_TOKEN);
 
       try
       {
@@ -191,7 +193,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
    public void testProperlyFormattedButExpiredToken()
    {
       Map<String, String> context = new HashMap<>();
-      context.put(AUTH0_ACCESS_TOKEN_KEY, EXPIRED_TOKEN);
+      context.put(ACCESS_TOKEN_KEY, EXPIRED_TOKEN);
 
       try
       {
@@ -236,7 +238,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
    public void testNullToken()
    {
       Map<String, String> context = new HashMap<>();
-      context.put(AUTH0_ACCESS_TOKEN_KEY, null);
+      context.put(ACCESS_TOKEN_KEY, null);
 
       try
       {
@@ -267,7 +269,7 @@ public class Auth0AuthenticationModuleTest extends BaseTest
       auth0Spy.createSession(qInstance, context);
       auth0Spy.createSession(qInstance, context);
       auth0Spy.createSession(qInstance, context);
-      verify(auth0Spy, times(1)).getAccessTokenFromAuth0(any(), any(), any());
+      verify(auth0Spy, times(1)).getAccessTokenForUsernameAndPasswordFromAuth0(any(), any(), any());
    }
 
 
@@ -467,4 +469,26 @@ public class Auth0AuthenticationModuleTest extends BaseTest
       return (encoder.encodeToString(originalString.getBytes()));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMask()
+   {
+      assertNull(maskForLog(null));
+      assertEquals("******", maskForLog("1"));
+      assertEquals("******", maskForLog("12"));
+      assertEquals("******", maskForLog("123"));
+      assertEquals("******", maskForLog("1234"));
+      assertEquals("******", maskForLog("12345"));
+      assertEquals("******", maskForLog("12345"));
+      assertEquals("******", maskForLog("123456"));
+      assertEquals("******", maskForLog("1234567"));
+      assertEquals("123456******", maskForLog("12345678"));
+      assertEquals("123456******", maskForLog("123456789"));
+      assertEquals("123456******", maskForLog("1234567890"));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedfilters/SavedFilterProcessTests.java

@@ -35,7 +35,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFiltersMetaDataProvider;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
@@ -57,8 +56,6 @@ class SavedFilterProcessTests extends BaseTest
    {
       QInstance qInstance = QContext.getQInstance();
       new SavedFiltersMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
-
       String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY;
 
       {

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/RunRecordScriptTest.java

@@ -35,7 +35,6 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.junit.jupiter.api.Test;
@@ -56,7 +55,6 @@ class RunRecordScriptTest extends BaseTest
    {
       QInstance qInstance = QContext.getQInstance();
       new ScriptsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
 
       TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(new QRecord().withValue("id", 1)));
       TestUtils.insertRecords(qInstance, qInstance.getTable(Script.TABLE_NAME), List.of(new QRecord().withValue("id", 1).withTableName(TestUtils.TABLE_NAME_PERSON_MEMORY)));

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/TestScriptProcessStepTest.java

@@ -37,8 +37,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.scripts.Script;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptType;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTableAccessor;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -60,8 +58,6 @@ class TestScriptProcessStepTest extends BaseTest
    {
       QInstance qInstance = QContext.getQInstance();
       new ScriptsMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null);
-      new QQQTablesMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
-
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(ScriptType.TABLE_NAME);
       insertInput.setRecords(List.of(new ScriptType()
@@ -75,7 +71,7 @@ class TestScriptProcessStepTest extends BaseTest
       insertInput.setRecords(List.of(new Script()
          .withName("TestScript")
          .withScriptTypeId(insertOutput.getRecords().get(0).getValueInteger("id"))
-         .withTableId(QQQTableAccessor.getTableId(TestUtils.TABLE_NAME_SHAPE))
+         .withTableName(TestUtils.TABLE_NAME_SHAPE)
          .toQRecord()));
       insertOutput = new InsertAction().execute(insertInput);
 
@@ -93,10 +89,7 @@ class TestScriptProcessStepTest extends BaseTest
       // expect an error because the javascript module isn't available //
       //////////////////////////////////////////////////////////////////
       assertNotNull(output.getValue("exception"));
-      assertThat((Exception) output.getValue("exception"))
-         .hasRootCauseInstanceOf(ClassNotFoundException.class)
-         .rootCause()
-         .hasMessageContaining("QJavaScriptExecutor");
+      assertThat((Exception) output.getValue("exception")).hasRootCauseInstanceOf(ClassNotFoundException.class);
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -33,15 +33,18 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarCh
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
 import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.GetAgeStatistics;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
 import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -110,6 +113,9 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicE
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess;
 import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.reports.RunReportForRecordProcess;
+import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -590,6 +596,7 @@ public class TestUtils
             .withFieldName("order.storeId")
             .withJoinNameChain(List.of("orderLineItem")))
          .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_LINE_ITEM_EXTRINSIC).withJoinName("lineItemLineItemExtrinsic"))
+         .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER))
          .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false))
          .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false))
          .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false))
@@ -1393,4 +1400,39 @@ public class TestUtils
             ))
          );
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void updatePersonMemoryTableInContextWithWritableByWriteLockAndInsert3TestRecords() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      qInstance.addSecurityKeyType(new QSecurityKeyType()
+         .withName("writableBy"));
+
+      QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
+      table.withField(new QFieldMetaData("onlyWritableBy", QFieldType.STRING).withLabel("Only Writable By"));
+      table.withRecordSecurityLock(new RecordSecurityLock()
+         .withSecurityKeyType("writableBy")
+         .withFieldName("onlyWritableBy")
+         .withNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW)
+         .withLockScope(RecordSecurityLock.LockScope.WRITE));
+
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe", "kmarsh")));
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(List.of(
+         new QRecord().withValue("id", 1).withValue("firstName", "Darin"),
+         new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("onlyWritableBy", "kmarsh"),
+         new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("onlyWritableBy", "jdoe")
+      )));
+
+      //////////////////////////////////////////////
+      // make sure we can query for all 3 records //
+      //////////////////////////////////////////////
+      QContext.getQSession().setSecurityKeyValues(MapBuilder.of("writableBy", ListBuilder.of("jdoe")));
+      assertEquals(3, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_PERSON_MEMORY)).getCount());
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/YamlUtilsTest.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.utils;
 
 import java.util.Map;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import org.junit.jupiter.api.Test;
 
@@ -31,7 +32,7 @@ import org.junit.jupiter.api.Test;
 /*******************************************************************************
  ** Unit test for YamlUtils
  *******************************************************************************/
-class YamlUtilsTest
+class YamlUtilsTest extends BaseTest
 {
 
    /*******************************************************************************

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -519,7 +519,7 @@ public class BaseAPIActionUtil
          {
             String wrapperObjectName = getBackendDetails(table).getTableWrapperObjectName();
             jsonObject = JsonUtils.toJSONObject(resultString);
-            if(jsonObject.has(wrapperObjectName))
+            if(jsonObject.has(wrapperObjectName) && !jsonObject.isNull(wrapperObjectName))
             {
                Object o = jsonObject.get(wrapperObjectName);
                if(o instanceof JSONArray jsonArray)
@@ -750,6 +750,10 @@ public class BaseAPIActionUtil
                throw (new QException("Error setting authorization query parameter", e));
             }
          }
+         case CUSTOM ->
+         {
+            handleCustomAuthorization(request);
+         }
          default -> throw new IllegalArgumentException("Unexpected authorization type: " + backendMetaData.getAuthorizationType());
       }
    }
@@ -1398,4 +1402,16 @@ public class BaseAPIActionUtil
       ///////////////////////////////////////////////////////////////////////////////////////////////////
       return (7 * 1024);
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected void handleCustomAuthorization(HttpRequestBase request) throws QException
+   {
+      ///////////////////////////////////////////////////////////////////////
+      // nothing to do at this layer, meant to be overridden by subclasses //
+      ///////////////////////////////////////////////////////////////////////
+   }
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/AuthorizationType.java

@@ -31,6 +31,7 @@ public enum AuthorizationType
    API_TOKEN,
    BASIC_AUTH_API_KEY,
    BASIC_AUTH_USERNAME_PASSWORD,
+   CUSTOM,
    OAUTH2,
    API_KEY_QUERY_PARAM,
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java

@@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -387,7 +388,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
       QQueryFilter securityFilter = new QQueryFilter();
       securityFilter.setBooleanOperator(QQueryFilter.BooleanOperator.AND);
 
-      for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+      for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
       {
          // todo - uh, if it's a RIGHT (or FULL) join, then, this should be isOuter = true, right?
          boolean isOuter = false;
@@ -407,7 +408,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
          }
 
          QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
-         for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks()))
+         for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(joinTable.getRecordSecurityLocks())))
          {
             boolean isOuter = queryJoin.getType().equals(QueryJoin.Type.LEFT); // todo full?
             addSubFilterForRecordSecurityLock(instance, session, joinTable, securityFilter, recordSecurityLock, joinsContext, queryJoin.getJoinTableOrItsAlias(), isOuter);
@@ -1035,7 +1036,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
    {
       if(table != null)
       {
-         for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+         for(RecordSecurityLock recordSecurityLock : RecordSecurityLockFilters.filterForReadLocks(CollectionUtils.nonNullList(table.getRecordSecurityLocks())))
          {
             for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
             {

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java

@@ -83,7 +83,6 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
          boolean    needToCloseConnection = false;
          if(deleteInput.getTransaction() != null && deleteInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -96,7 +96,6 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
          boolean    needToCloseConnection = false;
          if(insertInput.getTransaction() != null && insertInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from insertInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -111,7 +111,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          boolean    needToCloseConnection = false;
          if(queryInput.getTransaction() != null && queryInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from queryInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java

@@ -131,7 +131,6 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
          boolean    needToCloseConnection = false;
          if(updateInput.getTransaction() != null && updateInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
          {
-            LOG.debug("Using connection from updateInput [" + rdbmsTransaction.getConnection() + "]");
             connection = rdbmsTransaction.getConnection();
          }
          else

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java

@@ -66,6 +66,7 @@ public class TestUtils
    public static final String TABLE_NAME_PERSONAL_ID_CARD    = "personalIdCard";
    public static final String TABLE_NAME_STORE               = "store";
    public static final String TABLE_NAME_ORDER               = "order";
+   public static final String TABLE_NAME_ORDER_INSTRUCTIONS  = "orderInstructions";
    public static final String TABLE_NAME_ITEM                = "item";
    public static final String TABLE_NAME_ORDER_LINE          = "orderLine";
    public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic";
@@ -245,6 +246,16 @@ public class TestUtils
          .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE))
          .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
          .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
+         .withField(new QFieldMetaData("currentOrderInstructionsId", QFieldType.INTEGER).withBackendName("current_order_instructions_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_INSTRUCTIONS, "order_instructions")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName("order.storeId")
+            .withJoinNameChain(List.of("orderInstructionsJoinOrder")))
+         .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id"))
+         .withField(new QFieldMetaData("instructions", QFieldType.STRING))
       );
 
       qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item")
@@ -357,6 +368,22 @@ public class TestUtils
          .withJoinOn(new JoinOn("id", "orderLineId"))
       );
 
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderJoinCurrentOrderInstructions")
+         .withLeftTable(TABLE_NAME_ORDER)
+         .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS)
+         .withType(JoinType.ONE_TO_ONE)
+         .withJoinOn(new JoinOn("currentOrderInstructionsId", "id"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withName("orderInstructionsJoinOrder")
+         .withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS)
+         .withRightTable(TABLE_NAME_ORDER)
+         .withType(JoinType.MANY_TO_ONE)
+         .withJoinOn(new JoinOn("orderId", "id"))
+      );
+
       qInstance.addPossibleValueSource(new QPossibleValueSource()
          .withName("store")
          .withType(QPossibleValueSourceType.TABLE)

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -32,10 +32,12 @@ import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -1695,4 +1697,51 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
 
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException
+   {
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+
+      {
+         /////////////////////////////////////////////////////////
+         // assert a failure if the join to use isn't specified //
+         /////////////////////////////////////////////////////////
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS));
+         assertThatThrownBy(() -> new QueryAction().execute(queryInput)).rootCause().hasMessageContaining("More than 1 join was found");
+      }
+
+      Integer noOfOrders            = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER)).getCount();
+      Integer noOfOrderInstructions = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS)).getCount();
+
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure we can join on order.current_order_instruction_id = order_instruction.id -- and that we get back 1 row per order //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderJoinCurrentOrderInstructions")));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         assertEquals(noOfOrders, queryOutput.getRecords().size());
+      }
+
+      {
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure we can join on order.id = order_instruction.order_id -- and that we get back 1 row per order instruction //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(TestUtils.TABLE_NAME_ORDER);
+         queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS).withJoinMetaData(QContext.getQInstance().getJoin("orderInstructionsJoinOrder")));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         assertEquals(noOfOrderInstructions, queryOutput.getRecords().size());
+      }
+
+   }
+
 }

qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql

@@ -84,6 +84,7 @@ DROP TABLE IF EXISTS line_item_extrinsic;
 DROP TABLE IF EXISTS order_line;
 DROP TABLE IF EXISTS item;
 DROP TABLE IF EXISTS `order`;
+DROP TABLE IF EXISTS order_instructions;
 DROP TABLE IF EXISTS warehouse_store_int;
 DROP TABLE IF EXISTS store;
 DROP TABLE IF EXISTS warehouse;
@@ -123,7 +124,8 @@ CREATE TABLE `order`
    id INT AUTO_INCREMENT PRIMARY KEY,
    store_id INT REFERENCES store,
    bill_to_person_id INT,
-   ship_to_person_id INT
+   ship_to_person_id INT,
+   current_order_instructions_id INT -- f-key to order_instructions, which also has an f-key back here!
 );
 
 -- variable orders
@@ -136,6 +138,27 @@ INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES
 INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (7, 3, null, 5);
 INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (8, 3, null, 5);
 
+CREATE TABLE order_instructions
+(
+   id INT AUTO_INCREMENT PRIMARY KEY,
+   order_id INT,
+   instructions VARCHAR(250)
+);
+
+-- give orders 1 & 2 multiple versions of the instruction record
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (1, 1, 'order 1 v1');
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (2, 1, 'order 1 v2');
+UPDATE `order` SET current_order_instructions_id = 2 WHERE id=1;
+
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (3, 2, 'order 2 v1');
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (4, 2, 'order 2 v2');
+INSERT INTO order_instructions (id, order_id, instructions) VALUES (5, 2, 'order 2 v3');
+UPDATE `order` SET current_order_instructions_id = 5 WHERE id=2;
+
+-- give all other orders just 1 instruction
+INSERT INTO order_instructions (order_id, instructions) SELECT id, concat('order ', id, ' v1') FROM `order` WHERE current_order_instructions_id IS NULL;
+UPDATE `order` SET current_order_instructions_id = (SELECT MIN(id) FROM order_instructions WHERE order_id = `order`.id) WHERE current_order_instructions_id is null;
+
 CREATE TABLE order_line
 (
    id INT AUTO_INCREMENT PRIMARY KEY,

qqq-dev-tools/CURRENT-SNAPSHOT-VERSION

@@ -1 +1 @@
-0.18.0
+0.19.0

qqq-dev-tools/bin/xbar-circleci-latest.sh

@@ -47,10 +47,10 @@ checkBuild()
    shortRepo="$repo"
    case $repo in
     qqq)                             shortRepo="qqq";;
-    qqq-frontend-core)               shortRepo="f'core";;
-    qqq-frontend-material-dashboard) shortRepo="m-db";;
+    qqq-frontend-core)               shortRepo="fc";;
+    qqq-frontend-material-dashboard) shortRepo="qfmd";;
     ColdTrack-Live)                  shortRepo="ctl";;
-    ColdTrack-Live-Scripts)          shortRepo="ct1-scr";;
+    ColdTrack-Live-Scripts)          shortRepo="cls";;
    esac
 
    timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" $(echo "$startDate" | sed 's/\....Z/+0000/') +%s)

qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/ExecuteCodeActionTest.java

@@ -30,6 +30,7 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.scripts.ExecuteCodeAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.QCodeExecutor;
 import com.kingsrook.qqq.backend.core.actions.scripts.QCodeExecutorAware;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
@@ -307,6 +308,32 @@ class ExecuteCodeActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDeploymentModeIsInContext() throws QException
+   {
+      String scriptSource = """
+         return (deploymentMode);
+         """;
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // first, with no deployment mode in the qInstance, make sure we can run, but get a null output //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      OneTestOutput oneTestOutput = testOne(null, scriptSource, new HashMap<>());
+      assertNull(oneTestOutput.executeCodeOutput.getOutput());
+
+      /////////////////////////////////////////////////////////////////////
+      // next, set a deploymentMode, and assert that we get it back out. //
+      /////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().setDeploymentMode("unit-test");
+      oneTestOutput = testOne(null, scriptSource, new HashMap<>());
+      assertEquals("unit-test", oneTestOutput.executeCodeOutput.getOutput());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java

@@ -36,9 +36,12 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.APIVersion;
+import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
 import com.kingsrook.qqq.api.model.actions.HttpApiResponse;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiOperation;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessCustomizers;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInput;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInputFieldsContainer;
@@ -104,6 +107,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -150,6 +154,7 @@ public class ApiImplementation
 
       QTableMetaData table     = validateTableAndVersion(apiInstanceMetaData, version, tableApiName, ApiOperation.QUERY_BY_QUERY_STRING);
       String         tableName = table.getName();
+      String         apiName   = apiInstanceMetaData.getName();
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
@@ -231,6 +236,8 @@ public class ApiImplementation
          badRequestMessages.add("includeCount must be either true or false");
       }
 
+      Map<String, QFieldMetaData> tableApiFields = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, version, tableName));
+
       if(StringUtils.hasContent(orderBy))
       {
          for(String orderByPart : orderBy.split(","))
@@ -238,6 +245,7 @@ public class ApiImplementation
             orderByPart = orderByPart.trim();
             String[] orderByNameDirection = orderByPart.split(" +");
             boolean  asc                  = true;
+            String   apiFieldName         = orderByNameDirection[0];
             if(orderByNameDirection.length == 2)
             {
                if("asc".equalsIgnoreCase(orderByNameDirection[1]))
@@ -250,7 +258,7 @@ public class ApiImplementation
                }
                else
                {
-                  badRequestMessages.add("orderBy direction for field " + orderByNameDirection[0] + " must be either ASC or DESC.");
+                  badRequestMessages.add("orderBy direction for field " + apiFieldName + " must be either ASC or DESC.");
                }
             }
             else if(orderByNameDirection.length > 2)
@@ -258,14 +266,27 @@ public class ApiImplementation
                badRequestMessages.add("Unrecognized format for orderBy clause: " + orderByPart + ".  Expected:  fieldName [ASC|DESC].");
             }
 
-            try
+            QFieldMetaData field = tableApiFields.get(apiFieldName);
+            if(field == null)
             {
-               QFieldMetaData field = table.getField(orderByNameDirection[0]);
-               filter.withOrderBy(new QFilterOrderBy(field.getName(), asc));
+               badRequestMessages.add("Unrecognized orderBy field name: " + apiFieldName + ".");
             }
-            catch(Exception e)
+            else
             {
-               badRequestMessages.add("Unrecognized orderBy field name: " + orderByNameDirection[0] + ".");
+               QFilterOrderBy filterOrderBy = new QFilterOrderBy(field.getName(), asc);
+
+               ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
+               if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
+               {
+                  filterOrderBy.setFieldName(apiFieldMetaData.getReplacedByFieldName());
+               }
+               else if(apiFieldMetaData.getCustomValueMapper() != null)
+               {
+                  ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
+                  customValueMapper.customizeFilterOrderBy(queryInput, filterOrderBy, apiFieldName, apiFieldMetaData);
+               }
+
+               filter.withOrderBy(filterOrderBy);
             }
          }
       }
@@ -289,20 +310,36 @@ public class ApiImplementation
             continue;
          }
 
-         try
+         QFieldMetaData field = tableApiFields.get(name);
+         if(field == null)
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-            // todo - deal with removed fields; fields w/ custom value mappers (need new method(s) there) //
-            ////////////////////////////////////////////////////////////////////////////////////////////////
-
-            QFieldMetaData field = table.getField(name);
+            badRequestMessages.add("Unrecognized filter criteria field: " + name);
+         }
+         else
+         {
+            ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
             for(String value : values)
             {
                if(StringUtils.hasContent(value))
                {
                   try
                   {
-                     filter.addCriteria(parseQueryParamToCriteria(field, name, value));
+                     QFilterCriteria criteria = parseQueryParamToCriteria(field, name, value);
+
+                     /////////////////////////////////////////////
+                     // deal with replaced or customized fields //
+                     /////////////////////////////////////////////
+                     if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
+                     {
+                        criteria.setFieldName(apiFieldMetaData.getReplacedByFieldName());
+                     }
+                     else if(apiFieldMetaData.getCustomValueMapper() != null)
+                     {
+                        ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
+                        customValueMapper.customizeFilterCriteria(queryInput, filter, criteria, name, apiFieldMetaData);
+                     }
+
+                     filter.addCriteria(criteria);
                   }
                   catch(Exception e)
                   {
@@ -311,10 +348,6 @@ public class ApiImplementation
                }
             }
          }
-         catch(Exception e)
-         {
-            badRequestMessages.add("Unrecognized filter criteria field: " + name);
-         }
       }
 
       //////////////////////////////////////////
@@ -350,7 +383,7 @@ public class ApiImplementation
       ArrayList<Map<String, Serializable>> records = new ArrayList<>();
       for(QRecord record : queryOutput.getRecords())
       {
-         records.add(QRecordApiAdapter.qRecordToApiMap(record, tableName, apiInstanceMetaData.getName(), version));
+         records.add(QRecordApiAdapter.qRecordToApiMap(record, tableName, apiName, version));
       }
 
       /////////////////////////////

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -485,7 +485,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
                   .withDescription("How the results of the query should be sorted. SQL-style, comma-separated list of field names, each optionally followed by ASC or DESC (defaults to ASC).")
                   .withIn("query")
                   .withSchema(new Schema().withType("string"))
-                  .withExamples(buildOrderByExamples(primaryKeyApiName, tableApiFields)),
+                  .withExamples(buildOrderByExamples(apiName, primaryKeyApiName, tableApiFields)),
                new Parameter()
                   .withName("booleanOperator")
                   .withDescription("Whether to combine query field as an AND or an OR.  Default is AND.")
@@ -500,10 +500,12 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
          for(QFieldMetaData tableApiField : tableApiFields)
          {
-            String label = tableApiField.getLabel();
+            String fieldName = ApiFieldMetaData.getEffectiveApiFieldName(apiName, tableApiField);
+            String label     = tableApiField.getLabel();
+
             if(!StringUtils.hasContent(label))
             {
-               label = QInstanceEnricher.nameToLabel(tableApiField.getName());
+               label = QInstanceEnricher.nameToLabel(fieldName);
             }
 
             StringBuilder description = new StringBuilder("Query on the " + label + " field.  ");
@@ -517,7 +519,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
             }
 
             queryGet.getParameters().add(new Parameter()
-               .withName(tableApiField.getName())
+               .withName(fieldName)
                .withDescription(description.toString())
                .withIn("query")
                .withExplode(true)
@@ -892,6 +894,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       ////////////////////////////////
       List<Parameter> parameters      = new ArrayList<>();
       ApiProcessInput apiProcessInput = apiProcessMetaData.getInput();
+      String          apiName         = apiInstanceMetaData.getName();
       if(apiProcessInput != null)
       {
          ApiProcessInputFieldsContainer queryStringParams = apiProcessInput.getQueryStringParams();
@@ -912,12 +915,13 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
          if(bodyField != null)
          {
             ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(bodyField);
-            ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiInstanceMetaData.getName());
+            ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiName);
 
             String fieldLabel = bodyField.getLabel();
+            String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiName, bodyField);
             if(!StringUtils.hasContent(fieldLabel))
             {
-               fieldLabel = QInstanceEnricher.nameToLabel(bodyField.getName());
+               fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
             }
 
             String bodyDescription = "Value for the " + fieldLabel;
@@ -979,7 +983,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       ApiProcessOutputInterface output = apiProcessMetaData.getOutput();
       if(!ApiProcessMetaData.AsyncMode.ALWAYS.equals(apiProcessMetaData.getAsyncMode()))
       {
-         responses.putAll(output.getSpecResponses(apiInstanceMetaData.getName()));
+         responses.putAll(output.getSpecResponses(apiName));
       }
       if(!ApiProcessMetaData.AsyncMode.NEVER.equals(apiProcessMetaData.getAsyncMode()))
       {
@@ -1074,13 +1078,16 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
     *******************************************************************************/
    private Parameter processFieldToParameter(ApiInstanceMetaData apiInstanceMetaData, QFieldMetaData field)
    {
-      ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(field);
-      ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiInstanceMetaData.getName());
+      String apiName = apiInstanceMetaData.getName();
 
+      ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(field);
+      ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiName);
+
+      String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiName, field);
       String fieldLabel = field.getLabel();
       if(!StringUtils.hasContent(fieldLabel))
       {
-         fieldLabel = QInstanceEnricher.nameToLabel(field.getName());
+         fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
       }
 
       String description = "Value for the " + fieldLabel + " field.";
@@ -1097,7 +1104,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       Schema fieldSchema = getFieldSchema(field, description, apiInstanceMetaData);
 
       Parameter parameter = new Parameter()
-         .withName(field.getName())
+         .withName(fieldName)
          .withDescription(description)
          .withRequired(field.getIsRequired())
          .withSchema(fieldSchema);
@@ -1213,14 +1220,15 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       for(QFieldMetaData field : tableApiFields)
       {
          String fieldLabel = field.getLabel();
+         String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiInstanceMetaData.getName(), field);
          if(!StringUtils.hasContent(fieldLabel))
          {
-            fieldLabel = QInstanceEnricher.nameToLabel(field.getName());
+            fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
          }
 
          String defaultDescription = fieldLabel + " for the " + table.getLabel() + ".";
          Schema fieldSchema        = getFieldSchema(field, defaultDescription, apiInstanceMetaData);
-         tableFields.put(ApiFieldMetaData.getEffectiveApiFieldName(apiInstanceMetaData.getName(), field), fieldSchema);
+         tableFields.put(fieldName, fieldSchema);
       }
 
       //////////////////////////////////
@@ -1561,7 +1569,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Example> buildOrderByExamples(String primaryKeyApiName, List<? extends QFieldMetaData> tableApiFields)
+   private Map<String, Example> buildOrderByExamples(String apiName, String primaryKeyApiName, List<? extends QFieldMetaData> tableApiFields)
    {
       Map<String, Example> rs = new LinkedHashMap<>();
 
@@ -1569,7 +1577,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       List<String> fieldsForExample5 = new ArrayList<>();
       for(QFieldMetaData tableApiField : tableApiFields)
       {
-         String name = tableApiField.getName();
+         String name = ApiFieldMetaData.getEffectiveApiFieldName(apiName, tableApiField);
          if(primaryKeyApiName.equals(name) || fieldsForExample4.contains(name) || fieldsForExample5.contains(name))
          {
             continue;

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GetTableApiFieldsAction.java

@@ -24,7 +24,10 @@ package com.kingsrook.qqq.api.actions;
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
@@ -50,6 +53,65 @@ import org.apache.commons.lang.BooleanUtils;
  *******************************************************************************/
 public class GetTableApiFieldsAction extends AbstractQActionFunction<GetTableApiFieldsInput, GetTableApiFieldsOutput>
 {
+   private static Map<ApiNameVersionAndTableName, List<QFieldMetaData>>        fieldListCache = new HashMap<>();
+   private static Map<ApiNameVersionAndTableName, Map<String, QFieldMetaData>> fieldMapCache  = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Allow tests (that manipulate meta-data) to clear field caches.
+    *******************************************************************************/
+   public static void clearCaches()
+   {
+      fieldListCache.clear();
+      fieldMapCache.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** convenience (and caching) wrapper
+    *******************************************************************************/
+   public static Map<String, QFieldMetaData> getTableApiFieldMap(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
+   {
+      if(!fieldMapCache.containsKey(apiNameVersionAndTableName))
+      {
+         Map<String, QFieldMetaData> map = getTableApiFieldList(apiNameVersionAndTableName).stream().collect(Collectors.toMap(f -> (ApiFieldMetaData.getEffectiveApiFieldName(apiNameVersionAndTableName.apiName(), f)), f -> f));
+         fieldMapCache.put(apiNameVersionAndTableName, map);
+      }
+
+      return (fieldMapCache.get(apiNameVersionAndTableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** convenience (and caching) wrapper
+    *******************************************************************************/
+   public static List<QFieldMetaData> getTableApiFieldList(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
+   {
+      if(!fieldListCache.containsKey(apiNameVersionAndTableName))
+      {
+         List<QFieldMetaData> value = new GetTableApiFieldsAction().execute(new GetTableApiFieldsInput()
+            .withTableName(apiNameVersionAndTableName.tableName())
+            .withVersion(apiNameVersionAndTableName.apiVersion())
+            .withApiName(apiNameVersionAndTableName.apiName())).getFields();
+         fieldListCache.put(apiNameVersionAndTableName, value);
+      }
+      return (fieldListCache.get(apiNameVersionAndTableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Input-record for convenience methods
+    *******************************************************************************/
+   public record ApiNameVersionAndTableName(String apiName, String apiVersion, String tableName)
+   {
+
+   }
+
+
 
    /*******************************************************************************
     **

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java

@@ -29,12 +29,10 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
-import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
@@ -66,20 +64,6 @@ public class QRecordApiAdapter
 {
    private static final QLogger LOG = QLogger.getLogger(QRecordApiAdapter.class);
 
-   private static Map<ApiNameVersionAndTableName, List<QFieldMetaData>>        fieldListCache = new HashMap<>();
-   private static Map<ApiNameVersionAndTableName, Map<String, QFieldMetaData>> fieldMapCache  = new HashMap<>();
-
-
-
-   /*******************************************************************************
-    ** Allow tests (that manipulate meta-data) to clear field caches.
-    *******************************************************************************/
-   public static void clearCaches()
-   {
-      fieldListCache.clear();
-      fieldMapCache.clear();
-   }
-
 
 
    /*******************************************************************************
@@ -92,7 +76,7 @@ public class QRecordApiAdapter
          return (null);
       }
 
-      List<QFieldMetaData>                tableApiFields = getTableApiFieldList(new ApiNameVersionAndTableName(apiName, apiVersion, tableName));
+      List<QFieldMetaData>                tableApiFields = GetTableApiFieldsAction.getTableApiFieldList(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, apiVersion, tableName));
       LinkedHashMap<String, Serializable> outputRecord   = new LinkedHashMap<>();
 
       /////////////////////////////////////////
@@ -111,7 +95,7 @@ public class QRecordApiAdapter
          else if(apiFieldMetaData.getCustomValueMapper() != null)
          {
             ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
-            value = customValueMapper.produceApiValue(record);
+            value = customValueMapper.produceApiValue(record, apiFieldName);
          }
          else
          {
@@ -157,7 +141,7 @@ public class QRecordApiAdapter
     *******************************************************************************/
    private static boolean isAssociationOmitted(String apiName, String apiVersion, QTableMetaData table, Association association)
    {
-      ApiTableMetaData       thisApiTableMetaData   = ObjectUtils.tryElse(() -> ApiTableMetaDataContainer.of(table).getApiTableMetaData(apiName), new ApiTableMetaData());
+      ApiTableMetaData       thisApiTableMetaData   = ObjectUtils.tryAndRequireNonNullElse(() -> ApiTableMetaDataContainer.of(table).getApiTableMetaData(apiName), new ApiTableMetaData());
       ApiAssociationMetaData apiAssociationMetaData = thisApiTableMetaData.getApiAssociationMetaData().get(association.getName());
       if(apiAssociationMetaData != null)
       {
@@ -185,7 +169,7 @@ public class QRecordApiAdapter
       ////////////////////////////////////////////////////////////////////////////////
       // make map of apiFieldNames (e.g., names as api uses them) to QFieldMetaData //
       ////////////////////////////////////////////////////////////////////////////////
-      Map<String, QFieldMetaData> apiFieldsMap           = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, apiVersion, tableName));
+      Map<String, QFieldMetaData> apiFieldsMap           = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, apiVersion, tableName));
       List<String>                unrecognizedFieldNames = new ArrayList<>();
       QRecord                     qRecord                = new QRecord();
 
@@ -241,7 +225,7 @@ public class QRecordApiAdapter
             else if(apiFieldMetaData.getCustomValueMapper() != null)
             {
                ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
-               customValueMapper.consumeApiValue(qRecord, value, jsonObject);
+               customValueMapper.consumeApiValue(qRecord, value, jsonObject, jsonKey);
             }
             else
             {
@@ -332,7 +316,7 @@ public class QRecordApiAdapter
       {
          if(!supportedVersion.toString().equals(apiVersion))
          {
-            Map<String, QFieldMetaData> versionFields = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName));
+            Map<String, QFieldMetaData> versionFields = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName));
             if(versionFields.containsKey(unrecognizedFieldName))
             {
                versionsWithThisField.add(supportedVersion.toString());
@@ -348,47 +332,4 @@ public class QRecordApiAdapter
       return (null);
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, QFieldMetaData> getTableApiFieldMap(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
-   {
-      if(!fieldMapCache.containsKey(apiNameVersionAndTableName))
-      {
-         Map<String, QFieldMetaData> map = getTableApiFieldList(apiNameVersionAndTableName).stream().collect(Collectors.toMap(f -> (ApiFieldMetaData.getEffectiveApiFieldName(apiNameVersionAndTableName.apiName(), f)), f -> f));
-         fieldMapCache.put(apiNameVersionAndTableName, map);
-      }
-
-      return (fieldMapCache.get(apiNameVersionAndTableName));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QFieldMetaData> getTableApiFieldList(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
-   {
-      if(!fieldListCache.containsKey(apiNameVersionAndTableName))
-      {
-         List<QFieldMetaData> value = new GetTableApiFieldsAction().execute(new GetTableApiFieldsInput()
-            .withTableName(apiNameVersionAndTableName.tableName())
-            .withVersion(apiNameVersionAndTableName.apiVersion())
-            .withApiName(apiNameVersionAndTableName.apiName())).getFields();
-         fieldListCache.put(apiNameVersionAndTableName, value);
-      }
-      return (fieldListCache.get(apiNameVersionAndTableName));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private record ApiNameVersionAndTableName(String apiName, String apiVersion, String tableName)
-   {
-
-   }
 }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java

@@ -27,7 +27,6 @@ import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -58,7 +57,6 @@ import com.kingsrook.qqq.api.model.openapi.HttpMethod;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.AccessTokenException;
 import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
@@ -75,15 +73,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.authentication.Auth0AuthenticationMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.session.QUser;
-import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
-import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
@@ -137,11 +132,6 @@ public class QJavalinApiHandler
    {
       return (() ->
       {
-         /////////////////////////////
-         // authentication endpoint //
-         /////////////////////////////
-         ApiBuilder.post("/api/oauth/token", QJavalinApiHandler::handleAuthorization);
-
          ///////////////////////////////////////////////
          // static endpoints to support rapidoc pages //
          ///////////////////////////////////////////////
@@ -583,101 +573,6 @@ public class QJavalinApiHandler
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static void handleAuthorization(Context context)
-   {
-      try
-      {
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////
-         // clientId & clientSecret may either be provided as formParams, or in an Authorization: Basic header //
-         ////////////////////////////////////////////////////////////////////////////////////////////////////////
-         String clientId;
-         String clientSecret;
-         String authorizationHeader = context.header("Authorization");
-         if(authorizationHeader != null && authorizationHeader.startsWith("Basic "))
-         {
-            try
-            {
-               byte[]   credDecoded = Base64.getDecoder().decode(authorizationHeader.replace("Basic ", ""));
-               String   credentials = new String(credDecoded, StandardCharsets.UTF_8);
-               String[] parts       = credentials.split(":", 2);
-               clientId = parts[0];
-               clientSecret = parts[1];
-            }
-            catch(Exception e)
-            {
-               context.status(HttpStatus.BAD_REQUEST_400);
-               context.result("Could not parse client_id and client_secret from Basic Authorization header.");
-               return;
-            }
-         }
-         else
-         {
-            clientId = context.formParam("client_id");
-            if(clientId == null)
-            {
-               context.status(HttpStatus.BAD_REQUEST_400);
-               context.result("'client_id' must be provided.");
-               return;
-            }
-            clientSecret = context.formParam("client_secret");
-            if(clientSecret == null)
-            {
-               context.status(HttpStatus.BAD_REQUEST_400);
-               context.result("'client_secret' must be provided.");
-               return;
-            }
-         }
-
-         ////////////////////////////////////////////////////////
-         // get the auth0 authentication module from qInstance //
-         ////////////////////////////////////////////////////////
-         Auth0AuthenticationMetaData     metaData                        = (Auth0AuthenticationMetaData) qInstance.getAuthentication();
-         QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
-         QAuthenticationModuleInterface  authenticationModule            = qAuthenticationModuleDispatcher.getQModule(qInstance.getAuthentication());
-
-         try
-         {
-            //////////////////////////////////////////////////////////////////////////////////////////
-            // make call to get access token data, if no exception thrown, assume 200 OK and return //
-            //////////////////////////////////////////////////////////////////////////////////////////
-            QContext.init(qInstance, null); // hmm...
-            String accessToken = authenticationModule.createAccessToken(metaData, clientId, clientSecret);
-            context.status(HttpStatus.Code.OK.getCode());
-            context.result(accessToken);
-            QJavalinAccessLogger.logEndSuccess();
-         }
-         catch(AccessTokenException aae)
-         {
-            LOG.info("Error getting api access token", aae, logPair("clientId", clientId));
-
-            ///////////////////////////////////////////////////////////////////////////
-            // if the exception has a status code, then return that code and message //
-            ///////////////////////////////////////////////////////////////////////////
-            if(aae.getStatusCode() != null)
-            {
-               context.status(aae.getStatusCode());
-               context.result(aae.getMessage());
-               QJavalinAccessLogger.logEndSuccess();
-            }
-
-            ////////////////////////////////////////////////////////
-            // if no code, throw and handle like other exceptions //
-            ////////////////////////////////////////////////////////
-            throw (aae);
-         }
-      }
-      catch(Exception e)
-      {
-         handleException(context, e);
-         QJavalinAccessLogger.logEndFail(e);
-      }
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/ApiFieldCustomValueMapper.java

@@ -23,6 +23,11 @@ package com.kingsrook.qqq.api.model.actions;
 
 
 import java.io.Serializable;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import org.json.JSONObject;
 
@@ -34,9 +39,11 @@ public abstract class ApiFieldCustomValueMapper
 {
 
    /*******************************************************************************
-    **
+    ** When producing a JSON Object to send over the API (e.g., for a GET), this method
+    ** can run to customize the value that is produced, for the input QRecord's specified
+    ** fieldName
     *******************************************************************************/
-   public Serializable produceApiValue(QRecord record)
+   public Serializable produceApiValue(QRecord record, String apiFieldName)
    {
       /////////////////////
       // null by default //
@@ -46,10 +53,36 @@ public abstract class ApiFieldCustomValueMapper
 
 
 
+   /*******************************************************************************
+    ** When producing a QRecord (the first parameter) from a JSON Object that was
+    ** received from the API (e.g., a POST or PATCH) - this method can run to
+    ** allow customization of the incoming value.
+    *******************************************************************************/
+   public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject, String apiFieldName)
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject)
+   public void customizeFilterCriteria(QueryInput queryInput, QQueryFilter filter, QFilterCriteria criteria, String apiFieldName, ApiFieldMetaData apiFieldMetaData)
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void customizeFilterOrderBy(QueryInput queryInput, QFilterOrderBy orderBy, String apiFieldName, ApiFieldMetaData apiFieldMetaData)
    {
       /////////////////////
       // noop by default //

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/ApiInstanceMetaData.java

@@ -125,7 +125,10 @@ public class ApiInstanceMetaData implements ApiOperation.EnabledOperationsProvid
             {
                if(BooleanUtils.isNotTrue(apiTableMetaData.getIsExcluded()))
                {
-                  validator.assertCondition(allVersions.contains(new APIVersion(apiTableMetaData.getInitialVersion())), "Table " + table.getName() + "'s initial API version is not a recognized version for api " + apiName);
+                  if(StringUtils.hasContent(apiTableMetaData.getInitialVersion()))
+                  {
+                     validator.assertCondition(allVersions.contains(new APIVersion(apiTableMetaData.getInitialVersion())), "Table " + table.getName() + "'s initial API version is not a recognized version for api " + apiName);
+                  }
                }
             }
          }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/tables/ApiTableMetaData.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.api.model.metadata.ApiOperation;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -80,7 +81,7 @@ public class ApiTableMetaData implements ApiOperation.EnabledOperationsProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void enrich(String apiName, QTableMetaData table)
+   public void enrich(QInstance qInstance, String apiName, QTableMetaData table)
    {
       if(initialVersion != null)
       {
@@ -95,7 +96,7 @@ public class ApiTableMetaData implements ApiOperation.EnabledOperationsProvider
 
          for(QFieldMetaData field : CollectionUtils.nonNullList(removedApiFields))
          {
-            new QInstanceEnricher(null).enrichField(field);
+            new QInstanceEnricher(qInstance).enrichField(field);
             ApiFieldMetaData apiFieldMetaData = ensureFieldHasApiSupplementalMetaData(apiName, field);
             if(apiFieldMetaData.getInitialVersion() == null)
             {

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/tables/ApiTableMetaDataContainer.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.api.model.metadata.tables;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import com.kingsrook.qqq.api.ApiSupplementType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -80,13 +81,13 @@ public class ApiTableMetaDataContainer extends QSupplementalTableMetaData
     **
     *******************************************************************************/
    @Override
-   public void enrich(QTableMetaData table)
+   public void enrich(QInstance qInstance, QTableMetaData table)
    {
-      super.enrich(table);
+      super.enrich(qInstance, table);
 
       for(Map.Entry<String, ApiTableMetaData> entry : CollectionUtils.nonNullMap(apis).entrySet())
       {
-         entry.getValue().enrich(entry.getKey(), table);
+         entry.getValue().enrich(qInstance, entry.getKey(), table);
       }
    }
 

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/ApiImplementationTest.java

@@ -23,9 +23,14 @@ package com.kingsrook.qqq.api.actions;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.Month;
+import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.api.BaseTest;
 import com.kingsrook.qqq.api.TestUtils;
+import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
@@ -35,19 +40,23 @@ import com.kingsrook.qqq.api.model.metadata.tables.ApiAssociationMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaDataContainer;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.json.JSONObject;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -66,7 +75,7 @@ class ApiImplementationTest extends BaseTest
    @AfterEach
    void beforeAndAfterEach()
    {
-      QRecordApiAdapter.clearCaches();
+      GetTableApiFieldsAction.clearCaches();
    }
 
 
@@ -188,6 +197,43 @@ class ApiImplementationTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testQueryWithRemovedFields() throws QException
+   {
+      QInstance           qInstance           = QContext.getQInstance();
+      ApiInstanceMetaData apiInstanceMetaData = ApiInstanceMetaDataContainer.of(qInstance).getApiInstanceMetaData(TestUtils.API_NAME);
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecord(new QRecord()
+         .withValue("firstName", "Tim")
+         .withValue("noOfShoes", 2)
+         .withValue("birthDate", LocalDate.of(1980, Month.MAY, 31))
+         .withValue("cost", new BigDecimal("3.50"))
+         .withValue("price", new BigDecimal("9.99"))
+         .withValue("photo", "ABCD".getBytes())));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // query by a field that wasn't in an old api version, but is in the table now - should fail //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+
+      assertThatThrownBy(() ->
+         ApiImplementation.query(apiInstanceMetaData, TestUtils.V2022_Q4, TestUtils.TABLE_NAME_PERSON, MapBuilder.of("noOfShoes", List.of("2"))))
+         .isInstanceOf(QBadRequestException.class)
+         .hasMessageContaining("Unrecognized filter criteria field");
+
+      {
+         /////////////////////////////////////////////
+         // query by a removed field (was replaced) //
+         /////////////////////////////////////////////
+         Map<String, Serializable> queryResult = ApiImplementation.query(apiInstanceMetaData, TestUtils.V2022_Q4, TestUtils.TABLE_NAME_PERSON, MapBuilder.of("shoeCount", List.of("2")));
+         assertEquals(1, queryResult.get("count"));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -198,7 +244,7 @@ class ApiImplementationTest extends BaseTest
        **
        *******************************************************************************/
       @Override
-      public Serializable produceApiValue(QRecord record)
+      public Serializable produceApiValue(QRecord record, String apiFieldName)
       {
          return ("customValue-" + record.getValueString("lastName"));
       }
@@ -209,7 +255,7 @@ class ApiImplementationTest extends BaseTest
        **
        *******************************************************************************/
       @Override
-      public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject)
+      public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject, String apiFieldName)
       {
          String valueString = ValueUtils.getValueAsString(value);
          valueString = valueString.replaceFirst("^stripThisAway-", "");

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java

@@ -51,7 +51,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.modules.authentication.implementations.FullyAnonymousAuthenticationModule;
 import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.javalin.QJavalinImplementation;
@@ -1386,56 +1385,6 @@ class QJavalinApiHandlerTest extends BaseTest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testAuthorizeNoParams()
-   {
-      ///////////////
-      // no params //
-      ///////////////
-      HttpResponse<String> response = Unirest.post(BASE_URL + "/api/oauth/token").asString();
-      assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
-      assertThat(response.getBody()).contains("client_id");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testAuthorizeOneParam()
-   {
-      ///////////////
-      // no params //
-      ///////////////
-      HttpResponse<String> response = Unirest.post(BASE_URL + "/api/oauth/token")
-         .body("client_id=XXXXXXXXXX").asString();
-      assertEquals(HttpStatus.BAD_REQUEST_400, response.getStatus());
-      assertThat(response.getBody()).contains("client_secret");
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testAuthorizeAllParams()
-   {
-      ///////////////
-      // no params //
-      ///////////////
-      HttpResponse<String> response = Unirest.post(BASE_URL + "/api/oauth/token")
-         .body("client_id=XXXXXXXXXX&client_secret=YYYYYYYYYYYY").asString();
-      assertEquals(HttpStatus.OK_200, response.getStatus());
-      assertThat(response.getBody()).isEqualTo(FullyAnonymousAuthenticationModule.TEST_ACCESS_TOKEN);
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -113,6 +113,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.QStatusMessage;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleInterface;
+import com.kingsrook.qqq.backend.core.modules.authentication.implementations.Auth0AuthenticationModule;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
@@ -148,10 +149,10 @@ public class QJavalinImplementation
 {
    private static final QLogger LOG = QLogger.getLogger(QJavalinImplementation.class);
 
-   public static final int    SESSION_COOKIE_AGE     = 60 * 60 * 24;
-   public static final String SESSION_ID_COOKIE_NAME = "sessionId";
-   public static final String BASIC_AUTH_NAME        = "basicAuthString";
-   public static final String API_KEY_NAME           = "apiKey";
+   public static final int    SESSION_COOKIE_AGE       = 60 * 60 * 24;
+   public static final String SESSION_ID_COOKIE_NAME   = "sessionId";
+   public static final String SESSION_UUID_COOKIE_NAME = "sessionUUID";
+   public static final String API_KEY_NAME             = "apiKey";
 
    static QInstance        qInstance;
    static QJavalinMetaData javalinMetaData;
@@ -159,8 +160,8 @@ public class QJavalinImplementation
    private static Supplier<QInstance> qInstanceHotSwapSupplier;
    private static long                lastQInstanceHotSwapMillis;
 
-   private static final long MILLIS_BETWEEN_HOT_SWAPS = 2500;
-   public static final  long SLOW_LOG_THRESHOLD_MS    = 1000;
+   private static      long MILLIS_BETWEEN_HOT_SWAPS = 2500;
+   public static final long SLOW_LOG_THRESHOLD_MS    = 1000;
 
    private static final Integer DEFAULT_COUNT_TIMEOUT_SECONDS = 60;
    private static final Integer DEFAULT_QUERY_TIMEOUT_SECONDS = 60;
@@ -329,6 +330,8 @@ public class QJavalinImplementation
    {
       return (() ->
       {
+         post("/manageSession", QJavalinImplementation::manageSession);
+
          /////////////////////
          // metadata routes //
          /////////////////////
@@ -400,6 +403,36 @@ public class QJavalinImplementation
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void manageSession(Context context)
+   {
+      try
+      {
+         Map<?, ?> map = context.bodyAsClass(Map.class);
+
+         QAuthenticationModuleDispatcher qAuthenticationModuleDispatcher = new QAuthenticationModuleDispatcher();
+         QAuthenticationModuleInterface  authenticationModule            = qAuthenticationModuleDispatcher.getQModule(qInstance.getAuthentication());
+
+         Map<String, String> authContext = new HashMap<>();
+         //? authContext.put("uuid", ValueUtils.getValueAsString(map.get("uuid")));
+         authContext.put(Auth0AuthenticationModule.ACCESS_TOKEN_KEY, ValueUtils.getValueAsString(map.get("accessToken")));
+         authContext.put(Auth0AuthenticationModule.DO_STORE_USER_SESSION_KEY, "true");
+
+         QSession session = authenticationModule.createSession(qInstance, authContext);
+
+         context.cookie(SESSION_UUID_COOKIE_NAME, session.getUuid(), SESSION_COOKIE_AGE);
+         context.result(JsonUtils.toJson(MapBuilder.of("uuid", session.getUuid())));
+      }
+      catch(Exception e)
+      {
+         handleException(context, e);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -443,16 +476,24 @@ public class QJavalinImplementation
          Map<String, String> authenticationContext = new HashMap<>();
 
          String sessionIdCookieValue     = context.cookie(SESSION_ID_COOKIE_NAME);
+         String sessionUuidCookieValue   = context.cookie(Auth0AuthenticationModule.SESSION_UUID_KEY);
          String authorizationHeaderValue = context.header("Authorization");
          String apiKeyHeaderValue        = context.header("x-api-key");
 
          if(StringUtils.hasContent(sessionIdCookieValue))
          {
-            ////////////////////////////////////////
-            // first, look for a sessionId cookie //
-            ////////////////////////////////////////
+            ///////////////////////////////////////////////////////
+            // sessionId - maybe used by table-based auth module //
+            ///////////////////////////////////////////////////////
             authenticationContext.put(SESSION_ID_COOKIE_NAME, sessionIdCookieValue);
          }
+         else if(StringUtils.hasContent(sessionUuidCookieValue))
+         {
+            ///////////////////////////////////////////////////////////////////////////
+            // session UUID - known to be used by auth0 module (in aug. 2023 update) //
+            ///////////////////////////////////////////////////////////////////////////
+            authenticationContext.put(Auth0AuthenticationModule.SESSION_UUID_KEY, sessionUuidCookieValue);
+         }
          else if(apiKeyHeaderValue != null)
          {
             /////////////////////////////////////////////////////////////////
@@ -533,12 +574,12 @@ public class QJavalinImplementation
       if(authorizationHeaderValue.startsWith(basicPrefix))
       {
          authorizationHeaderValue = authorizationHeaderValue.replaceFirst(basicPrefix, "");
-         authenticationContext.put(BASIC_AUTH_NAME, authorizationHeaderValue);
+         authenticationContext.put(Auth0AuthenticationModule.BASIC_AUTH_KEY, authorizationHeaderValue);
       }
       else if(authorizationHeaderValue.startsWith(bearerPrefix))
       {
          authorizationHeaderValue = authorizationHeaderValue.replaceFirst(bearerPrefix, "");
-         authenticationContext.put(SESSION_ID_COOKIE_NAME, authorizationHeaderValue);
+         authenticationContext.put(Auth0AuthenticationModule.ACCESS_TOKEN_KEY, authorizationHeaderValue);
       }
       else
       {
@@ -871,6 +912,8 @@ public class QJavalinImplementation
          getInput.setShouldTranslatePossibleValues(true);
          getInput.setShouldFetchHeavyFields(true);
 
+         getInput.setQueryJoins(processQueryJoinsParam(context));
+
          if("true".equals(context.queryParam("includeAssociations")))
          {
             getInput.setIncludeAssociations(true);
@@ -1816,4 +1859,14 @@ public class QJavalinImplementation
       return StringUtils.joinWithCommasAndAnd(errors.stream().map(QStatusMessage::getMessage).toList());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void setMillisBetweenHotSwaps(long millisBetweenHotSwaps)
+   {
+      MILLIS_BETWEEN_HOT_SWAPS = millisBetweenHotSwaps;
+   }
+
 }

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinAccessLoggerTest.java

@@ -26,7 +26,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
@@ -52,7 +52,7 @@ class QJavalinAccessLoggerTest
     **
     *******************************************************************************/
    @Test
-   void testDefaultOn() throws QException
+   void testDefaultOn() throws QInstanceValidationException
    {
       QInstance qInstance = TestUtils.defineInstance();
       new QJavalinImplementation(qInstance, new QJavalinMetaData());
@@ -74,7 +74,7 @@ class QJavalinAccessLoggerTest
     **
     *******************************************************************************/
    @Test
-   void testTurnedOffByCode() throws QException
+   void testTurnedOffByCode() throws QInstanceValidationException
    {
       QInstance qInstance = TestUtils.defineInstance();
       new QJavalinImplementation(qInstance, new QJavalinMetaData()
@@ -97,7 +97,7 @@ class QJavalinAccessLoggerTest
     **
     *******************************************************************************/
    @Test
-   void testTurnedOffBySystemPropertyWithJavalinMetaData() throws QException
+   void testTurnedOffBySystemPropertyWithJavalinMetaData() throws QInstanceValidationException
    {
       System.setProperty(DISABLED_PROPERTY, "true");
       QInstance qInstance = TestUtils.defineInstance();
@@ -114,7 +114,7 @@ class QJavalinAccessLoggerTest
     **
     *******************************************************************************/
    @Test
-   void testTurnedOffBySystemPropertyWithoutJavalinMetaData() throws QException
+   void testTurnedOffBySystemPropertyWithoutJavalinMetaData() throws QInstanceValidationException
    {
       System.setProperty(DISABLED_PROPERTY, "true");
       QInstance qInstance = TestUtils.defineInstance();
@@ -131,7 +131,7 @@ class QJavalinAccessLoggerTest
     **
     *******************************************************************************/
    @Test
-   void testFilter() throws QException
+   void testFilter() throws QInstanceValidationException
    {
       QInstance qInstance = TestUtils.defineInstance();
       new QJavalinImplementation(qInstance, new QJavalinMetaData()

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationAuthenticationTest.java

@@ -27,7 +27,7 @@ import java.util.Base64;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.authentication.TableBasedAuthenticationMetaData;
@@ -61,7 +61,7 @@ public class QJavalinImplementationAuthenticationTest extends QJavalinTestBase
     **
     *******************************************************************************/
    @BeforeEach
-   public void beforeEach() throws QException
+   public void beforeEach() throws QInstanceValidationException
    {
       Unirest.config().reset().enableCookieManagement(false);
       setupTableBasedAuthenticationInstance();
@@ -188,7 +188,7 @@ public class QJavalinImplementationAuthenticationTest extends QJavalinTestBase
    /*******************************************************************************
     **
     *******************************************************************************/
-   static void setupTableBasedAuthenticationInstance() throws QException
+   static void setupTableBasedAuthenticationInstance() throws QInstanceValidationException
    {
       QInstance                        qInstance                        = TestUtils.defineInstance();
       TableBasedAuthenticationMetaData tableBasedAuthenticationMetaData = new TableBasedAuthenticationMetaData();

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java

@@ -29,9 +29,18 @@ import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.SleepUtils;
 import kong.unirest.HttpResponse;
 import kong.unirest.Unirest;
 import org.eclipse.jetty.http.HttpStatus;
@@ -43,6 +52,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 
@@ -635,7 +645,8 @@ class QJavalinImplementationTest extends QJavalinTestBase
       JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
       assertNotNull(jsonObject);
       assertEquals(1, jsonObject.getInt("deletedRecordCount"));
-      TestUtils.runTestSql("SELECT id FROM person", (rs -> {
+      TestUtils.runTestSql("SELECT id FROM person", (rs ->
+      {
          int rowsFound = 0;
          while(rs.next())
          {
@@ -832,4 +843,130 @@ class QJavalinImplementationTest extends QJavalinTestBase
       assertTrue(jsonObject.has("type"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testManageSession()
+   {
+      String body = """
+         {
+            "accessToken": "abcd",
+            "doStoreUserSession": true
+         }
+         """;
+      HttpResponse<String> response = Unirest.post(BASE_URL + "/manageSession")
+         .header("Content-Type", "application/json")
+         .body(body)
+         .asString();
+
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      assertTrue(jsonObject.has("uuid"));
+      response.getHeaders().get("Set-Cookie").stream().anyMatch(s -> s.contains("sessionUUID"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testHotSwap() throws QInstanceValidationException
+   {
+      try
+      {
+         Function<String, QInstance> makeNewInstanceWithBackendName = (backendName) ->
+         {
+            QInstance newInstance = new QInstance();
+            newInstance.addBackend(new QBackendMetaData().withName(backendName).withBackendType("mock"));
+
+            if(!"invalid".equals(backendName))
+            {
+               newInstance.addTable(new QTableMetaData()
+                  .withName("newTable")
+                  .withBackendName(backendName)
+                  .withField(new QFieldMetaData("newField", QFieldType.INTEGER))
+                  .withPrimaryKeyField("newField")
+               );
+            }
+
+            return (newInstance);
+         };
+
+         QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("newBackend"));
+
+         /////////////////////////////////////////////////////////////////////////////////
+         // make sure before a hot-swap, that the instance doesn't have our new backend //
+         /////////////////////////////////////////////////////////////////////////////////
+         assertNull(QJavalinImplementation.qInstance.getBackend("newBackend"));
+
+         ///////////////////////////////////////////////////////
+         // do a hot-swap, make sure the new backend is there //
+         ///////////////////////////////////////////////////////
+         QJavalinImplementation.hotSwapQInstance(null);
+         assertNotNull(QJavalinImplementation.qInstance.getBackend("newBackend"));
+
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // now change to make a different backend - try to swap again - but the newer backend shouldn't be there, //
+         // because the millis-between-hot-swaps won't have passed                                                 //
+         ////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("newerBackend"));
+         QJavalinImplementation.hotSwapQInstance(null);
+         assertNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
+
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         // set the sleep threshold to 1 milli, sleep for 2, and then assert that we do swap again //
+         ////////////////////////////////////////////////////////////////////////////////////////////
+         QJavalinImplementation.setMillisBetweenHotSwaps(1);
+         SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
+
+         QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("newerBackend"));
+         QJavalinImplementation.hotSwapQInstance(null);
+         assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
+
+         ////////////////////////////////////////////////////////////
+         // assert that an invalid instance doesn't get swapped in //
+         // e.g., "newerBackend" still exists                      //
+         ////////////////////////////////////////////////////////////
+         SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
+         QJavalinImplementation.setQInstanceHotSwapSupplier(() -> makeNewInstanceWithBackendName.apply("invalid"));
+         QJavalinImplementation.hotSwapQInstance(null);
+         assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
+
+         ///////////////////////////////////////////////////////
+         // assert that if the supplier throws, we don't swap //
+         // e.g., "newerBackend" still exists                 //
+         ///////////////////////////////////////////////////////
+         SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
+         QJavalinImplementation.setQInstanceHotSwapSupplier(() ->
+         {
+            throw new RuntimeException("oops");
+         });
+         QJavalinImplementation.hotSwapQInstance(null);
+         assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
+
+         /////////////////////////////////////////////////////////////
+         // assert that if the supplier returns null, we don't swap //
+         // e.g., "newerBackend" still exists                       //
+         /////////////////////////////////////////////////////////////
+         SleepUtils.sleep(2, TimeUnit.MILLISECONDS);
+         QJavalinImplementation.setQInstanceHotSwapSupplier(() -> null);
+         QJavalinImplementation.hotSwapQInstance(null);
+         assertNotNull(QJavalinImplementation.qInstance.getBackend("newerBackend"));
+      }
+      finally
+      {
+         ////////////////////////////////////////////////////////////
+         // restore things to how they used to be, for other tests //
+         ////////////////////////////////////////////////////////////
+         QInstance qInstance = TestUtils.defineInstance();
+         QJavalinImplementation.setQInstanceHotSwapSupplier(null);
+         restartServerWithInstance(qInstance);
+      }
+   }
+
 }

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinTestBase.java

@@ -22,7 +22,6 @@
 package com.kingsrook.qqq.backend.javalin;
 
 
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
@@ -61,7 +60,7 @@ public class QJavalinTestBase
     **
     *******************************************************************************/
    @BeforeAll
-   public static void beforeAll() throws QException
+   public static void beforeAll() throws QInstanceValidationException
    {
       qJavalinImplementation = new QJavalinImplementation(TestUtils.defineInstance());
       QJavalinProcessHandler.setAsyncStepTimeoutMillis(250);

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java

@@ -68,7 +68,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.AssociatedScript;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.savedfilters.SavedFiltersMetaDataProvider;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
-import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
 import com.kingsrook.qqq.backend.core.processes.implementations.mock.MockBackendStep;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
@@ -139,7 +138,7 @@ public class TestUtils
     ** Define the q-instance for testing (h2 rdbms and 'person' table)
     **
     *******************************************************************************/
-   public static QInstance defineInstance() throws QException
+   public static QInstance defineInstance()
    {
       QInstance qInstance = new QInstance();
       qInstance.setAuthentication(defineAuthentication());
@@ -155,8 +154,6 @@ public class TestUtils
       qInstance.addPossibleValueSource(definePossibleValueSourcePerson());
       defineWidgets(qInstance);
 
-      new QQQTablesMetaDataProvider().defineAll(qInstance, BACKEND_NAME_MEMORY, BACKEND_NAME_MEMORY, null);
-
       qInstance.addBackend(defineMemoryBackend());
       try
       {

qqq-middleware-picocli/src/main/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementation.java

@@ -90,8 +90,6 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import io.github.cdimascio.dotenv.Dotenv;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.core.config.Configurator;
-import org.jline.reader.LineReader;
-import org.jline.reader.LineReaderBuilder;
 import org.jline.utils.Log;
 import picocli.CommandLine;
 import picocli.CommandLine.Model.CommandSpec;
@@ -292,18 +290,7 @@ public class QPicoCliImplementation
          }
 
          Map<String, String> authenticationContext = new HashMap<>();
-         if(sessionId == null && authenticationModule instanceof Auth0AuthenticationModule)
-         {
-            LineReader lr      = LineReaderBuilder.builder().build();
-            String     tokenId = lr.readLine("Create a .env file with the contents of the Auth0 JWT Id Token in the variable 'SESSION_ID': \nPress enter once complete...");
-            dotenv = loadDotEnv();
-            if(dotenv.isPresent())
-            {
-               sessionId = dotenv.get().get("SESSION_ID");
-            }
-         }
-
-         authenticationContext.put("sessionId", sessionId);
+         authenticationContext.put(Auth0AuthenticationModule.ACCESS_TOKEN_KEY, sessionId);
 
          // todo - does this need some per-provider logic actually?  mmm...
          session = authenticationModule.createSession(qInstance, authenticationContext);