Merged feature/sftp-and-headless-bulk-load into dev

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java

@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
@@ -160,4 +161,18 @@ public interface RecordCustomizerUtilityInterface
       return (oldRecordMap);
    }
 
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   static <T extends Serializable> T getValueFromRecordOrOldRecord(String fieldName, QRecord record, Serializable primaryKey, Optional<Map<Serializable, QRecord>> oldRecordMap)
+   {
+      T value = (T) record.getValue(fieldName);
+      if(value == null && primaryKey != null && oldRecordMap.isPresent() && oldRecordMap.get().containsKey(primaryKey))
+      {
+         value = (T) oldRecordMap.get().get(primaryKey).getValue(fieldName);
+      }
+      return value;
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java

@@ -0,0 +1,251 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.net.URLEncoder;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Generic widget to display a list of records.
+ **
+ ** Note, closely related to (and copied from ChildRecordListRenderer.
+ ** opportunity to share more code with that in the future??
+ *******************************************************************************/
+public class RecordListWidgetRenderer extends AbstractWidgetRenderer
+{
+   private static final QLogger LOG = QLogger.getLogger(RecordListWidgetRenderer.class);
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static Builder widgetMetaDataBuilder(String widgetName)
+   {
+      return (new Builder(new QWidgetMetaData()
+         .withName(widgetName)
+         .withIsCard(true)
+         .withCodeReference(new QCodeReference(RecordListWidgetRenderer.class))
+         .withType(WidgetType.CHILD_RECORD_LIST.getType())
+         .withValidatorPlugin(new RecordListWidgetValidator())
+      ));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static class Builder extends AbstractWidgetMetaDataBuilder
+   {
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public Builder(QWidgetMetaData widgetMetaData)
+      {
+         super(widgetMetaData);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withLabel(String label)
+      {
+         widgetMetaData.setLabel(label);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withMaxRows(Integer maxRows)
+      {
+         widgetMetaData.withDefaultValue("maxRows", maxRows);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withTableName(String tableName)
+      {
+         widgetMetaData.withDefaultValue("tableName", tableName);
+         return (this);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      public Builder withFilter(QQueryFilter filter)
+      {
+         widgetMetaData.withDefaultValue("filter", filter);
+         return (this);
+      }
+
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      try
+      {
+         Integer maxRows = null;
+         if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
+         {
+            maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows"));
+         }
+         else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows"))
+         {
+            maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows"));
+         }
+
+         QQueryFilter filter = ((QQueryFilter) input.getWidgetMetaData().getDefaultValues().get("filter")).clone();
+         filter.interpretValues(new HashMap<>(input.getQueryParams()), FilterUseCase.DEFAULT);
+         filter.setLimit(maxRows);
+
+         String         tableName = ValueUtils.getValueAsString(input.getWidgetMetaData().getDefaultValues().get("tableName"));
+         QTableMetaData table     = QContext.getQInstance().getTable(tableName);
+
+         QueryInput queryInput = new QueryInput();
+         queryInput.setTableName(tableName);
+         queryInput.setShouldTranslatePossibleValues(true);
+         queryInput.setShouldGenerateDisplayValues(true);
+         queryInput.setFilter(filter);
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+
+         QValueFormatter.setBlobValuesToDownloadUrls(table, queryOutput.getRecords());
+
+         int totalRows = queryOutput.getRecords().size();
+         if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
+         {
+            /////////////////////////////////////////////////////////////////////////////////////
+            // if the input said to only do some max, and the # of results we got is that max, //
+            // then do a count query, for displaying 1-n of <count>                            //
+            /////////////////////////////////////////////////////////////////////////////////////
+            CountInput countInput = new CountInput();
+            countInput.setTableName(tableName);
+            countInput.setFilter(filter);
+            totalRows = new CountAction().execute(countInput).getCount();
+         }
+
+         String tablePath   = QContext.getQInstance().getTablePath(tableName);
+         String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+
+         ChildRecordListData widgetData = new ChildRecordListData(input.getQueryParams().get("widgetLabel"), queryOutput, table, tablePath, viewAllLink, totalRows);
+
+         return (new RenderWidgetOutput(widgetData));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error rendering record list widget", e, logPair("widgetName", () -> input.getWidgetMetaData().getName()));
+         throw (e);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static class RecordListWidgetValidator implements QInstanceValidatorPluginInterface<QWidgetMetaDataInterface>
+   {
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator)
+      {
+         String prefix = "Widget " + widgetMetaData.getName() + ": ";
+
+         //////////////////////////////////////////////
+         // make sure table name is given and exists //
+         //////////////////////////////////////////////
+         QTableMetaData table = null;
+         String tableName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("tableName"));
+         if(qInstanceValidator.assertCondition(StringUtils.hasContent(tableName), prefix + "defaultValue for tableName must be given"))
+         {
+            ////////////////////////////
+            // make sure table exists //
+            ////////////////////////////
+            table = qInstance.getTable(tableName);
+            qInstanceValidator.assertCondition(table != null, prefix + "No table named " + tableName + " exists in the instance");
+         }
+
+         ////////////////////////////////////////////////////////////////////////////////////
+         // make sure filter is given and is valid (only check that if table is given too) //
+         ////////////////////////////////////////////////////////////////////////////////////
+         QQueryFilter filter = ((QQueryFilter) widgetMetaData.getDefaultValues().get("filter"));
+         if(qInstanceValidator.assertCondition(filter != null, prefix + "defaultValue for filter must be given") && table != null)
+         {
+            qInstanceValidator.validateQueryFilter(qInstance, prefix, table, filter, null);
+         }
+      }
+
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java

@@ -815,13 +815,14 @@ public class RunProcessAction
       {
          QSession         session         = QContext.getQSession();
          QBackendMetaData backendMetaData = QContext.getQInstance().getBackend(process.getVariantBackend());
-         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
+         String           variantTypeKey  = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
+         if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
          {
             LOG.warn("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'");
          }
          else
          {
-            basepullKeyValue += "-" + session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
+            basepullKeyValue += "-" + session.getBackendVariants().get(variantTypeKey);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java

@@ -108,7 +108,7 @@ public class CsvExportStreamer implements ExportStreamerInterface
       }
       catch(Exception e)
       {
-         throw (new QReportingException("Error starting CSV report"));
+         throw (new QReportingException("Error starting CSV report", e));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java

@@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
@@ -82,6 +83,22 @@ public class CountAction
 
 
 
+   /*******************************************************************************
+    ** shorthand way to call for the most common use-case, when you just want the
+    ** count to be returned, and you just want to pass in a table name and filter.
+    *******************************************************************************/
+   public static Integer execute(String tableName, QQueryFilter filter) throws QException
+   {
+      CountAction countAction = new CountAction();
+      CountInput  countInput  = new CountInput();
+      countInput.setTableName(tableName);
+      countInput.setFilter(filter);
+      CountOutput countOutput = countAction.execute(countInput);
+      return (countOutput.getCount());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java

@@ -82,6 +82,11 @@ public class DeleteAction
    {
       ActionHelper.validateSession(deleteInput);
 
+      if(deleteInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in delete input"));
+      }
+
       QTableMetaData table               = deleteInput.getTable();
       String         primaryKeyFieldName = table.getPrimaryKeyField();
       QFieldMetaData primaryKeyField     = table.getField(primaryKeyFieldName);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -238,6 +238,11 @@ public class GetAction
     *******************************************************************************/
    public static QRecord execute(String tableName, Serializable primaryKey) throws QException
    {
+      if(primaryKey instanceof QQueryFilter)
+      {
+         LOG.warn("Unexpected use of QQueryFilter instead of primary key in GetAction call");
+      }
+
       GetAction getAction = new GetAction();
       GetInput  getInput  = new GetInput(tableName).withPrimaryKey(primaryKey);
       return getAction.executeForRecord(getInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
       ActionHelper.validateSession(insertInput);
+
+      if(!StringUtils.hasContent(insertInput.getTableName()))
+      {
+         throw (new QException("Table name was not specified in insert input"));
+      }
+
       QTableMetaData table = insertInput.getTable();
 
       if(table == null)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -47,7 +47,8 @@ public class StorageAction
 {
 
    /*******************************************************************************
-    **
+    ** create an output stream in the storage backend - that can be written to,
+    ** for the purpose of inserting or writing a file into storage.
     *******************************************************************************/
    public OutputStream createOutputStream(StorageInput storageInput) throws QException
    {
@@ -59,7 +60,8 @@ public class StorageAction
 
 
    /*******************************************************************************
-    **
+    ** create an input stream in the storage backend - that can be read from,
+    ** for the purpose of getting or reading a file from storage.
     *******************************************************************************/
    public InputStream getInputStream(StorageInput storageInput) throws QException
    {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -74,6 +74,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -118,6 +119,11 @@ public class UpdateAction
    {
       ActionHelper.validateSession(updateInput);
 
+      if(!StringUtils.hasContent(updateInput.getTableName()))
+      {
+         throw (new QException("Table name was not specified in update input"));
+      }
+
       QTableMetaData table = updateInput.getTable();
 
       //////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.values;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
@@ -74,6 +75,31 @@ public interface QCustomPossibleValueProvider<T extends Serializable>
    }
 
 
+   /***************************************************************************
+    ** meant to be protected (but interface...) - for a custom PVS implementation
+    ** to complete its search (e.g., after it generates the list of PVS objects,
+    ** let this method do the filtering).
+    ***************************************************************************/
+   default List<QPossibleValue<T>> completeCustomPVSSearch(SearchPossibleValueSourceInput input, List<QPossibleValue<T>> possibleValues)
+   {
+      SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input);
+
+      List<QPossibleValue<T>> rs = new ArrayList<>();
+
+      for(QPossibleValue<T> possibleValue : possibleValues)
+      {
+         if(possibleValue != null && SearchPossibleValueSourceAction.doesPossibleValueMatchSearchInput(possibleValue, preparedInput))
+         {
+            rs.add(possibleValue);
+         }
+      }
+
+      rs.sort(Comparator.nullsLast(Comparator.comparing((QPossibleValue<T> pv) -> pv.getLabel())));
+
+      return (rs);
+   }
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,6 +28,7 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -44,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import org.apache.commons.lang3.BooleanUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -484,6 +486,8 @@ public class QValueFormatter
             String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
             String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
 
+            Boolean downloadUrlDynamic = ValueUtils.getValueAsBoolean(adornmentValues.get(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC));
+
             for(QRecord record : records)
             {
                if(!doesFieldHaveValue(field, record))
@@ -491,6 +495,11 @@ public class QValueFormatter
                   continue;
                }
 
+               if(BooleanUtils.isTrue(downloadUrlDynamic))
+               {
+                  continue;
+               }
+
                Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
                String       fileName   = null;
 
@@ -508,7 +517,7 @@ public class QValueFormatter
                   {
                      @SuppressWarnings("unchecked") // instance validation should make this safe!
                      List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
-                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
+                     List<String> values = CollectionUtils.nullSafeHasContents(fileNameFormatFields) ? fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList() : Collections.emptyList();
                      fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
                   }
                }
@@ -531,7 +540,7 @@ public class QValueFormatter
 
                ////////////////////////////////////////////////////////////////////////////////////////////////
                // if field type is blob OR if there's a supplemental process or code-ref that needs to run - //
-               // then update its value to be a callback-url that'll give access to the bytes to download    //
+               // then update its value to be a callback-url that'll give access to the bytes to download.   //
                // implied here is that a String value (w/o supplemental code/proc) has its value stay as a   //
                // URL, which is where the file is directly downloaded from.  And in the case of a String     //
                // with code-to-run, then the code should run, followed by a redirect to the value URL.       //
@@ -540,7 +549,7 @@ public class QValueFormatter
                   || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE)
                   || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME))
                {
-                  record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+                  record.setValue(field.getName(), AdornmentType.FileDownloadValues.makeFieldDownloadUrl(table.getName(), primaryKey, field.getName(), fileName));
                }
                record.setDisplayValue(field.getName(), fileName);
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -26,6 +26,7 @@ import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -51,7 +52,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -108,60 +108,54 @@ public class SearchPossibleValueSourceAction
 
 
 
+   /***************************************************************************
+    ** record to store "computed" values as part of a possible-value search -
+    ** e.g., ids type-convered, and lower-cased labels.
+    ***************************************************************************/
+   public record PreparedSearchPossibleValueSourceInput(Collection<?> inputIdsAsCorrectType, Collection<String> lowerCaseLabels, String searchTerm) {}
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static PreparedSearchPossibleValueSourceInput prepareSearchPossibleValueSourceInput(SearchPossibleValueSourceInput input)
+   {
+      QPossibleValueSource possibleValueSource   = QContext.getQInstance().getPossibleValueSource(input.getPossibleValueSourceName());
+      List<?>              inputIdsAsCorrectType = convertInputIdsToPossibleValueSourceIdType(possibleValueSource, input.getIdList());
+
+      Set<String> lowerCaseLabels = null;
+      if(input.getLabelList() != null)
+      {
+         lowerCaseLabels = input.getLabelList().stream()
+            .filter(Objects::nonNull)
+            .map(l -> l.toLowerCase())
+            .collect(Collectors.toSet());
+      }
+
+      return (new PreparedSearchPossibleValueSourceInput(inputIdsAsCorrectType, lowerCaseLabels, input.getSearchTerm()));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private SearchPossibleValueSourceOutput searchPossibleValueEnum(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource)
    {
+      PreparedSearchPossibleValueSourceInput preparedSearchPossibleValueSourceInput = prepareSearchPossibleValueSourceInput(input);
+
       SearchPossibleValueSourceOutput output      = new SearchPossibleValueSourceOutput();
       List<Serializable>              matchingIds = new ArrayList<>();
 
-      List<?> inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList());
-      Set<String> labels = null;
-
       for(QPossibleValue<?> possibleValue : possibleValueSource.getEnumValues())
       {
-         boolean match = false;
-
-         if(input.getIdList() != null)
-         {
-            if(inputIdsAsCorrectType.contains(possibleValue.getId()))
-            {
-               match = true;
-            }
-         }
-         else if(input.getLabelList() != null)
-         {
-            if(labels == null)
-            {
-               labels = input.getLabelList().stream().filter(Objects::nonNull).map(l -> l.toLowerCase()).collect(Collectors.toSet());
-            }
-
-            if(labels.contains(possibleValue.getLabel().toLowerCase()))
-            {
-               match = true;
-            }
-         }
-         else
-         {
-            if(StringUtils.hasContent(input.getSearchTerm()))
-            {
-               match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase())
-                        || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase()));
-            }
-            else
-            {
-               match = true;
-            }
-         }
+         boolean match = doesPossibleValueMatchSearchInput(possibleValue, preparedSearchPossibleValueSourceInput);
 
          if(match)
          {
-            matchingIds.add((Serializable) possibleValue.getId());
+            matchingIds.add(possibleValue.getId());
          }
-
-         // todo - skip & limit?
-         // todo - default filter
       }
 
       List<QPossibleValue<?>> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, matchingIds);
@@ -172,42 +166,84 @@ public class SearchPossibleValueSourceAction
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean doesPossibleValueMatchSearchInput(QPossibleValue<?> possibleValue, PreparedSearchPossibleValueSourceInput input)
+   {
+      boolean match = false;
+
+      if(input.inputIdsAsCorrectType() != null)
+      {
+         if(input.inputIdsAsCorrectType().contains(possibleValue.getId()))
+         {
+            match = true;
+         }
+      }
+      else if(input.lowerCaseLabels() != null)
+      {
+         if(input.lowerCaseLabels().contains(possibleValue.getLabel().toLowerCase()))
+         {
+            match = true;
+         }
+      }
+      else
+      {
+         if(StringUtils.hasContent(input.searchTerm()))
+         {
+            match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.searchTerm().toLowerCase())
+               || possibleValue.getLabel().toLowerCase().startsWith(input.searchTerm().toLowerCase()));
+         }
+         else
+         {
+            match = true;
+         }
+      }
+      return match;
+   }
+
+
+
    /*******************************************************************************
     ** The input list of ids might come through as a type that isn't the same as
     ** the type of the ids in the enum (e.g., strings from a frontend, integers
-    ** in an enum).  So, this method looks at the first id in the enum, and then
-    ** maps all the inputIds to be of the same type.
+    ** in an enum).  So, this method type-converts them.
     *******************************************************************************/
-   private List<Object> convertInputIdsToEnumIdType(QPossibleValueSource possibleValueSource, List<Serializable> inputIdList)
+   private static List<Object> convertInputIdsToPossibleValueSourceIdType(QPossibleValueSource possibleValueSource, List<Serializable> inputIdList)
    {
       List<Object> rs = new ArrayList<>();
-      if(CollectionUtils.nullSafeIsEmpty(inputIdList))
+
+      if(inputIdList == null)
+      {
+         return (null);
+      }
+      else if(inputIdList.isEmpty())
       {
          return (rs);
       }
 
-      Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId();
+      QFieldType type = possibleValueSource.getIdType();
 
       for(Serializable inputId : inputIdList)
       {
          Object properlyTypedId = null;
          try
          {
-            if(anIdFromTheEnum instanceof Integer)
+            if(type.equals(QFieldType.INTEGER))
             {
                properlyTypedId = ValueUtils.getValueAsInteger(inputId);
             }
-            else if(anIdFromTheEnum instanceof String)
+            else if(type.isStringLike())
             {
                properlyTypedId = ValueUtils.getValueAsString(inputId);
             }
-            else if(anIdFromTheEnum instanceof Boolean)
+            else if(type.equals(QFieldType.BOOLEAN))
             {
                properlyTypedId = ValueUtils.getValueAsBoolean(inputId);
             }
             else
             {
-               LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName());
+               LOG.warn("Unexpected type [" + type + "] for ids in enum: " + possibleValueSource.getName());
             }
          }
          catch(Exception e)
@@ -397,9 +433,9 @@ public class SearchPossibleValueSourceAction
       }
       catch(Exception e)
       {
-         String message = "Error sending searching custom possible value source [" + input.getPossibleValueSourceName() + "]";
+         String message = "Error searching custom possible value source [" + input.getPossibleValueSourceName() + "]";
          LOG.warn(message, e);
-         throw (new QException(message));
+         throw (new QException(message, e));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -95,6 +95,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager;
+import com.kingsrook.qqq.backend.core.utils.ClassPathUtils;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -1482,6 +1483,31 @@ public class QInstanceEnricher
 
 
 
+   /***************************************************************************
+    ** scan the classpath for classes in the specified package name which
+    ** implement the QInstanceEnricherPluginInterface - any found get added
+    ***************************************************************************/
+   public static void discoverAndAddPluginsInPackage(String packageName) throws QException
+   {
+      try
+      {
+         for(Class<?> aClass : ClassPathUtils.getClassesInPackage(packageName))
+         {
+            if(QInstanceEnricherPluginInterface.class.isAssignableFrom(aClass))
+            {
+               QInstanceEnricherPluginInterface<?> plugin = (QInstanceEnricherPluginInterface<?>) aClass.getConstructor().newInstance();
+               addEnricherPlugin(plugin);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         throw (new QException("Error discovering and adding enricher plugins in package [" + packageName + "]", e));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -37,7 +37,9 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TimeZone;
+import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
 import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
@@ -108,12 +110,16 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.Automatio
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ListingHash;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda;
+import org.apache.commons.lang.BooleanUtils;
 import org.quartz.CronExpression;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
@@ -136,6 +142,8 @@ public class QInstanceValidator
 
    private static ListingHash<Class<?>, QInstanceValidatorPluginInterface<?>> validatorPlugins = new ListingHash<>();
 
+   private JoinGraph joinGraph = null;
+
    private List<String> errors = new ArrayList<>();
 
 
@@ -163,7 +171,6 @@ public class QInstanceValidator
       // the enricher will build a join graph (if there are any joins).  we'd like to only do that       //
       // once, during the enrichment/validation work, so, capture it, and store it back in the instance. //
       /////////////////////////////////////////////////////////////////////////////////////////////////////
-      JoinGraph joinGraph = null;
       long start = System.currentTimeMillis();
       try
       {
@@ -173,7 +180,7 @@ public class QInstanceValidator
          // TODO - possible point of customization (use a different enricher, or none, or pass it options).
          QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance);
          qInstanceEnricher.enrich();
-         joinGraph = qInstanceEnricher.getJoinGraph();
+         this.joinGraph = qInstanceEnricher.getJoinGraph();
       }
       catch(Exception e)
       {
@@ -543,6 +550,60 @@ public class QInstanceValidator
          {
             assertCondition(Objects.equals(backendName, backend.getName()), "Inconsistent naming for backend: " + backendName + "/" + backend.getName() + ".");
 
+            ///////////////////////
+            // validate variants //
+            ///////////////////////
+            BackendVariantsConfig backendVariantsConfig = backend.getBackendVariantsConfig();
+            if(BooleanUtils.isTrue(backend.getUsesVariants()))
+            {
+               if(assertCondition(backendVariantsConfig != null, "Missing backendVariantsConfig in backend [" + backendName + "] which is marked as usesVariants"))
+               {
+                  assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]");
+
+                  String         optionsTableName = backendVariantsConfig.getOptionsTableName();
+                  QTableMetaData optionsTable     = qInstance.getTable(optionsTableName);
+                  if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]"))
+                  {
+                     if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]"))
+                     {
+                        QQueryFilter optionsFilter = backendVariantsConfig.getOptionsFilter();
+                        if(optionsFilter != null)
+                        {
+                           validateQueryFilter(qInstance, "optionsFilter in backendVariantsConfig in backend [" + backendName + "]: ", optionsTable, optionsFilter, null);
+                        }
+                     }
+                  }
+
+                  Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap();
+                  if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]"))
+                  {
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     // only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier //
+                     // (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... //
+                     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+                     if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null)
+                     {
+                        for(Map.Entry<BackendVariantSetting, String> entry : backendSettingSourceFieldNameMap.entrySet())
+                        {
+                           assertCondition(optionsTable.getFields().containsKey(entry.getValue()), "Unrecognized fieldName [" + entry.getValue() + "] in backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]");
+                        }
+                     }
+                  }
+
+                  if(backendVariantsConfig.getVariantRecordLookupFunction() != null)
+                  {
+                     validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class);
+                  }
+               }
+            }
+            else
+            {
+               assertCondition(backendVariantsConfig == null, "Should not have a backendVariantsConfig in backend [" + backendName + "] which is not marked as usesVariants");
+            }
+
+            ///////////////////////////////////////////
+            // let the backend do its own validation //
+            ///////////////////////////////////////////
             backend.performValidation(this);
 
             runPlugins(QBackendMetaData.class, backend, qInstance);
@@ -1356,7 +1417,7 @@ public class QInstanceValidator
                ////////////////////////////////////////////////////////////////////////
                if(customizerInstance != null && tableCustomizer.getExpectedType() != null)
                {
-                  assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance);
+                  assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType());
                }
             }
          }
@@ -1368,18 +1429,31 @@ public class QInstanceValidator
    /*******************************************************************************
     ** Make sure that a given object can be casted to an expected type.
     *******************************************************************************/
-   private <T> T assertObjectCanBeCasted(String errorPrefix, Class<T> expectedType, Object object)
+   private void assertObjectCanBeCasted(String errorPrefix, Object object, Class<?>... anyOfExpectedClasses)
+   {
+      for(Class<?> expectedClass : anyOfExpectedClasses)
       {
-      T castedObject = null;
          try
          {
-         castedObject = expectedType.cast(object);
+            expectedClass.cast(object);
+            return;
          }
          catch(ClassCastException e)
          {
-         errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType);
+            /////////////////////////////////////
+            // try next type (if there is one) //
+            /////////////////////////////////////
+         }
+      }
+
+      if(anyOfExpectedClasses.length == 1)
+      {
+         errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]);
+      }
+      else
+      {
+         errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", ")));
       }
-      return castedObject;
    }
 
 
@@ -1616,12 +1690,12 @@ public class QInstanceValidator
 
             for(QFieldMetaData field : process.getInputFields())
             {
-               validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName());
+               validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", input field " + field.getName() + " ");
             }
 
             for(QFieldMetaData field : process.getOutputFields())
             {
-               validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName());
+               validateFieldPossibleValueSourceAttributes(qInstance, field, "Process " + processName + ", output field " + field.getName() + " ");
             }
 
             if(process.getCancelStep() != null)
@@ -1832,7 +1906,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List<QueryJoin> queryJoins)
+   public void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List<QueryJoin> queryJoins)
    {
       for(QFilterCriteria criterion : CollectionUtils.nonNullList(queryFilter.getCriteria()))
       {
@@ -1877,6 +1951,7 @@ public class QInstanceValidator
          if(fieldName.contains("."))
          {
             String fieldNameAfterDot  = fieldName.substring(fieldName.lastIndexOf(".") + 1);
+            String tableNameBeforeDot = fieldName.substring(0, fieldName.lastIndexOf("."));
 
             if(CollectionUtils.nullSafeHasContents(queryJoins))
             {
@@ -1900,11 +1975,32 @@ public class QInstanceValidator
             }
             else
             {
-               errors.add("QInstanceValidator does not yet support finding a field that looks like a join field, but isn't associated with a query.");
+               if(this.joinGraph != null)
+               {
+                  Set<JoinGraph.JoinConnectionList> joinConnections = joinGraph.getJoinConnections(table.getName());
+                  for(JoinGraph.JoinConnectionList joinConnectionList : joinConnections)
+                  {
+                     JoinGraph.JoinConnection joinConnection = joinConnectionList.list().get(joinConnectionList.list().size() - 1);
+                     if(tableNameBeforeDot.equals(joinConnection.joinTable()))
+                     {
+                        QTableMetaData joinTable = qInstance.getTable(tableNameBeforeDot);
+                        if(joinTable.getFields().containsKey(fieldNameAfterDot))
+                        {
+                           /////////////////////////
+                           // mmm, looks valid... //
+                           /////////////////////////
                            return (true);
-               // todo! for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values())
-               // {
-               // }
+                        }
+                     }
+                  }
+               }
+
+               //////////////////////////////////////////////////////////////////////////////////////
+               // todo - not sure how vulnerable we are to ongoing issues here...                  //
+               // idea:  let a filter (or any object?) be opted out of validation, some version of //
+               // a static map of objects we can check at the top of various validate methods...   //
+               //////////////////////////////////////////////////////////////////////////////////////
+               errors.add("Failed to find field named: " + fieldName);
             }
          }
       }
@@ -2123,7 +2219,8 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?> expectedClass)
+   @SafeVarargs
+   private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class<?>... anyOfExpectedClasses)
    {
       if(!preAssertionsForCodeReference(codeReference, prefix))
       {
@@ -2151,7 +2248,7 @@ public class QInstanceValidator
             ////////////////////////////////////////////////////////////////////////
             if(classInstance != null)
             {
-               assertObjectCanBeCasted(prefix, expectedClass, classInstance);
+               assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses);
             }
          }
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/audits/AuditSingleInput.java

@@ -326,6 +326,20 @@ public class AuditSingleInput implements Serializable
 
 
 
+   /*******************************************************************************
+    ** Fluent setter for details
+    *******************************************************************************/
+   public AuditSingleInput withDetailMessages(List<String> details)
+   {
+      for(String detail : details)
+      {
+         addDetail(message);
+      }
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.processes;
 import java.io.Serializable;
 import java.time.Instant;
 import java.time.LocalDate;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -34,9 +35,12 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.async.NonPersistedAsyncJobCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface;
@@ -247,6 +251,26 @@ public class RunBackendStepInput extends AbstractActionInput
 
 
 
+   /*******************************************************************************
+    ** Getter for records converted to entities of a given type.
+    **
+    *******************************************************************************/
+   public <E extends QRecordEntity> List<E> getRecordsAsEntities(Class<E> entityClass) throws QException
+   {
+      List<E> rs = new ArrayList<>();
+
+      ///////////////////////////////////////////////////////////////////////////////////
+      // note - important to call getRecords here, which is overwritten in subclasses! //
+      ///////////////////////////////////////////////////////////////////////////////////
+      for(QRecord record : getRecords())
+      {
+         rs.add(QRecordEntity.fromQRecord(entityClass, record));
+      }
+      return (rs);
+   }
+
+
+
    /*******************************************************************************
     ** Setter for records
     **
@@ -582,7 +606,7 @@ public class RunBackendStepInput extends AbstractActionInput
     ***************************************************************************/
    public void traceMessage(ProcessTracerMessage message)
    {
-      if(processTracer != null)
+      if(processTracer != null && message != null)
       {
          try
          {
@@ -594,4 +618,14 @@ public class RunBackendStepInput extends AbstractActionInput
          }
       }
    }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QProcessMetaData getProcess()
+   {
+      return (QContext.getQInstance().getProcess(getProcessName()));
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -258,7 +259,7 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
 
 
    /*******************************************************************************
-    **
+    ** add a record to the step output, e.g., for going through to the next step.
     *******************************************************************************/
    public void addRecord(QRecord record)
    {
@@ -271,6 +272,16 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
 
 
 
+   /***************************************************************************
+    ** add a RecordEntity to the step output, e.g., for going through to the next step.
+    ***************************************************************************/
+   public void addRecordEntity(QRecordEntity recordEntity)
+   {
+      addRecord(recordEntity.toQRecord());
+   }
+
+
+
    /*******************************************************************************
     ** Getter for auditInputList
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutput.java

@@ -0,0 +1,47 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+
+
+/*******************************************************************************
+ ** for use-cases where a metaDataProducer directly adds its objects to the
+ ** qInstance, then this empty object can be returned.
+ *******************************************************************************/
+public class EmptyMetaDataProducerOutput implements MetaDataProducerOutput
+{
+   private static final QLogger LOG = QLogger.getLogger(EmptyMetaDataProducerOutput.class);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance instance)
+   {
+      /////////////////////////////////
+      // noop - this output is empty //
+      /////////////////////////////////
+      LOG.trace("empty meta data producer has nothing to add.");
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java

@@ -29,5 +29,40 @@ package com.kingsrook.qqq.backend.core.model.metadata;
  *******************************************************************************/
 public abstract class MetaDataProducer<T extends MetaDataProducerOutput> implements MetaDataProducerInterface<T>
 {
+   private Class<?> sourceClass;
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   @Override
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   @Override
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public MetaDataProducer<T> withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java

@@ -106,14 +106,10 @@ public class MetaDataProducerHelper
    }
 
 
-
-   /*******************************************************************************
-    ** Recursively find all classes in the given package, that implement MetaDataProducerInterface
-    ** run them, and add their output to the given qInstance.
+   /***************************************************************************
     **
-    ** Note - they'll be sorted by the sortOrder they provide.
-    *******************************************************************************/
-   public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException
+    ***************************************************************************/
+   public static List<MetaDataProducerInterface<?>> findProducers(String packageName) throws QException
    {
       List<Class<?>> classesInPackage;
       try
@@ -196,6 +192,20 @@ public class MetaDataProducerHelper
             }
          }));
 
+      return (producers);
+   }
+
+
+   /*******************************************************************************
+    ** Recursively find all classes in the given package, that implement MetaDataProducerInterface
+    ** run them, and add their output to the given qInstance.
+    **
+    ** Note - they'll be sorted by the sortOrder they provide.
+    *******************************************************************************/
+   public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException
+   {
+      List<MetaDataProducerInterface<?>> producers = findProducers(packageName);
+
       ///////////////////////////////////////////////////////////////////////////
       // execute each one (if enabled), adding their meta data to the instance //
       ///////////////////////////////////////////////////////////////////////////
@@ -229,17 +239,19 @@ public class MetaDataProducerHelper
     **
     ***************************************************************************/
    @SuppressWarnings("unchecked")
-   private static <T extends Serializable & PossibleValueEnum<T>> MetaDataProducerInterface<?> processMetaDataProducingPossibleValueEnum(Class<?> aClass)
+   private static <T extends Serializable & PossibleValueEnum<T>> MetaDataProducerInterface<?> processMetaDataProducingPossibleValueEnum(Class<?> sourceClass)
    {
       String warningPrefix = "Found a class annotated as @" + QMetaDataProducingPossibleValueEnum.class.getSimpleName();
-      if(!PossibleValueEnum.class.isAssignableFrom(aClass))
+      if(!PossibleValueEnum.class.isAssignableFrom(sourceClass))
       {
-         LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName()));
+         LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName()));
          return null;
       }
 
-      PossibleValueEnum<?>[] values = (PossibleValueEnum<?>[]) aClass.getEnumConstants();
-      return (new PossibleValueSourceOfEnumGenericMetaDataProducer<T>(aClass.getSimpleName(), (PossibleValueEnum<T>[]) values));
+      PossibleValueEnum<?>[] values = (PossibleValueEnum<?>[]) sourceClass.getEnumConstants();
+      PossibleValueSourceOfEnumGenericMetaDataProducer<T> producer = new PossibleValueSourceOfEnumGenericMetaDataProducer<>(sourceClass.getSimpleName(), (PossibleValueEnum<T>[]) values);
+      producer.setSourceClass(sourceClass);
+      return producer;
    }
 
 
@@ -247,32 +259,32 @@ public class MetaDataProducerHelper
    /***************************************************************************
     **
     ***************************************************************************/
-   private static List<MetaDataProducerInterface<?>> processMetaDataProducingEntity(Class<?> aClass) throws Exception
+   private static List<MetaDataProducerInterface<?>> processMetaDataProducingEntity(Class<?> sourceClass) throws Exception
    {
       List<MetaDataProducerInterface<?>> rs = new ArrayList<>();
 
-      QMetaDataProducingEntity qMetaDataProducingEntity = aClass.getAnnotation(QMetaDataProducingEntity.class);
+      QMetaDataProducingEntity qMetaDataProducingEntity = sourceClass.getAnnotation(QMetaDataProducingEntity.class);
       String                   warningPrefix            = "Found a class annotated as @" + QMetaDataProducingEntity.class.getSimpleName();
 
       ///////////////////////////////////////////////////////////
       // make sures class is QRecordEntity and cast it as such //
       ///////////////////////////////////////////////////////////
-      if(!QRecordEntity.class.isAssignableFrom(aClass))
+      if(!QRecordEntity.class.isAssignableFrom(sourceClass))
       {
-         LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName()));
+         LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName()));
          return (rs);
       }
 
       @SuppressWarnings("unchecked") // safe per the check above.
-      Class<? extends QRecordEntity> recordEntityClass = (Class<? extends QRecordEntity>) aClass;
+      Class<? extends QRecordEntity> recordEntityClass = (Class<? extends QRecordEntity>) sourceClass;
 
       ////////////////////////////////////////////////
       // get TABLE_NAME static field from the class //
       ////////////////////////////////////////////////
-      Field tableNameField = aClass.getDeclaredField("TABLE_NAME");
+      Field tableNameField = recordEntityClass.getDeclaredField("TABLE_NAME");
       if(!tableNameField.getType().equals(String.class))
       {
-         LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", aClass.getSimpleName()));
+         LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", recordEntityClass.getSimpleName()));
          return (rs);
       }
 
@@ -293,6 +305,7 @@ public class MetaDataProducerHelper
             }
 
             RecordEntityToTableGenericMetaDataProducer producer = new RecordEntityToTableGenericMetaDataProducer(tableNameValue, recordEntityClass, tableMetaDataProductionCustomizer);
+            producer.setSourceClass(recordEntityClass);
 
             if(tableMetaDataCustomizer != null)
             {
@@ -312,7 +325,9 @@ public class MetaDataProducerHelper
       ////////////////////////////////////////
       if(qMetaDataProducingEntity.producePossibleValueSource())
       {
-         rs.add(new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue));
+         PossibleValueSourceOfTableGenericMetaDataProducer producer = new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue);
+         producer.setSourceClass(recordEntityClass);
+         rs.add(producer);
       }
 
       //////////////////////////
@@ -323,11 +338,11 @@ public class MetaDataProducerHelper
          Class<? extends QRecordEntity> childEntityClass = childTable.childTableEntityClass();
          if(childTable.childJoin().enabled())
          {
-            CollectionUtils.addIfNotNull(rs, processChildJoin(aClass, childTable));
+            CollectionUtils.addIfNotNull(rs, processChildJoin(recordEntityClass, childTable));
 
             if(childTable.childRecordListWidget().enabled())
             {
-               CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(aClass, childTable));
+               CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(recordEntityClass, childTable));
             }
          }
          else
@@ -337,7 +352,7 @@ public class MetaDataProducerHelper
                //////////////////////////////////////////////////////////////////////////
                // if not doing the join, can't do the child-widget, so warn about that //
                //////////////////////////////////////////////////////////////////////////
-               LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", aClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName()));
+               LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", recordEntityClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName()));
             }
          }
       }
@@ -350,14 +365,16 @@ public class MetaDataProducerHelper
    /***************************************************************************
     **
     ***************************************************************************/
-   private static MetaDataProducerInterface<?> processChildRecordListWidget(Class<?> aClass, ChildTable childTable) throws Exception
+   private static MetaDataProducerInterface<?> processChildRecordListWidget(Class<? extends QRecordEntity> sourceClass, ChildTable childTable) throws Exception
    {
       Class<? extends QRecordEntity> childEntityClass = childTable.childTableEntityClass();
-      String                         parentTableName  = getTableNameStaticFieldValue(aClass);
+      String                         parentTableName  = getTableNameStaticFieldValue(sourceClass);
       String                         childTableName   = getTableNameStaticFieldValue(childEntityClass);
 
       ChildRecordListWidget childRecordListWidget = childTable.childRecordListWidget();
-      return (new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget));
+      ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer producer = new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget);
+      producer.setSourceClass(sourceClass);
+      return producer;
    }
 
 
@@ -387,20 +404,22 @@ public class MetaDataProducerHelper
    /***************************************************************************
     **
     ***************************************************************************/
-   private static MetaDataProducerInterface<?> processChildJoin(Class<?> aClass, ChildTable childTable) throws Exception
+   private static MetaDataProducerInterface<?> processChildJoin(Class<? extends QRecordEntity> entityClass, ChildTable childTable) throws Exception
    {
       Class<? extends QRecordEntity> childEntityClass = childTable.childTableEntityClass();
 
-      String parentTableName        = getTableNameStaticFieldValue(aClass);
+      String parentTableName        = getTableNameStaticFieldValue(entityClass);
       String childTableName         = getTableNameStaticFieldValue(childEntityClass);
       String possibleValueFieldName = findPossibleValueField(childEntityClass, parentTableName);
       if(!StringUtils.hasContent(possibleValueFieldName))
       {
-         LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + aClass.getSimpleName() + "]");
+         LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + entityClass.getSimpleName() + "]");
          return (null);
       }
 
-      return (new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName));
+      ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName);
+      producer.setSourceClass(entityClass);
+      return producer;
    }
 
 
@@ -408,18 +427,20 @@ public class MetaDataProducerHelper
    /***************************************************************************
     **
     ***************************************************************************/
-   private static MetaDataProducerInterface<?> processMetaDataProducer(Class<?> aClass) throws Exception
+   private static MetaDataProducerInterface<?> processMetaDataProducer(Class<?> sourceCClass) throws Exception
    {
-      for(Constructor<?> constructor : aClass.getConstructors())
+      for(Constructor<?> constructor : sourceCClass.getConstructors())
       {
          if(constructor.getParameterCount() == 0)
          {
             Object o = constructor.newInstance();
-            return (MetaDataProducerInterface<?>) o;
+            MetaDataProducerInterface<?> producer = (MetaDataProducerInterface<?>) o;
+            producer.setSourceClass(sourceCClass);
+            return producer;
          }
       }
 
-      LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName()));
+      LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", sourceCClass.getSimpleName()));
       return null;
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java

@@ -73,4 +73,23 @@ public interface MetaDataProducerInterface<T extends MetaDataProducerOutput>
       return (true);
    }
 
+
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   default void setSourceClass(Class<?> sourceClass)
+   {
+      //////////
+      // noop //
+      //////////
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default Class<?> getSourceClass()
+   {
+      return null;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata;
 
 import java.util.ArrayList;
 import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
@@ -31,10 +32,12 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  ** Output object for a MetaDataProducer, which contains multiple meta-data
  ** objects.
  *******************************************************************************/
-public class MetaDataProducerMultiOutput implements MetaDataProducerOutput
+public class MetaDataProducerMultiOutput implements MetaDataProducerOutput, SourceQBitAware
 {
    private List<MetaDataProducerOutput> contents;
 
+   private String sourceQBitName;
+
 
 
    /*******************************************************************************
@@ -98,4 +101,48 @@ public class MetaDataProducerMultiOutput implements MetaDataProducerOutput
       return (rs);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public String getSourceQBitName()
+   {
+      return (this.sourceQBitName);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void setSourceQBitName(String sourceQBitName)
+   {
+      this.sourceQBitName = sourceQBitName;
+
+      /////////////////////////////////////////////
+      // propagate the name down to the children //
+      /////////////////////////////////////////////
+      for(MetaDataProducerOutput content : contents)
+      {
+         if(content instanceof SourceQBitAware aware)
+         {
+            aware.setSourceQBitName(sourceQBitName);
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput withSourceQBitName(String sourceQBitName)
+   {
+      setSourceQBitName(sourceQBitName);
+      return this;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java

@@ -26,8 +26,13 @@ import java.util.HashSet;
 import java.util.Set;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.serialization.QBackendMetaDataDeserializer;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 
 
@@ -46,20 +51,17 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    private Set<Capability> disabledCapabilities = new HashSet<>();
 
    private Boolean               usesVariants = false;
-   private String  variantOptionsTableIdField;
-   private String  variantOptionsTableNameField;
-   private String  variantOptionsTableTypeField;
-   private String  variantOptionsTableTypeValue;
-   private String  variantOptionsTableUsernameField;
-   private String  variantOptionsTablePasswordField;
-   private String  variantOptionsTableApiKeyField;
-   private String  variantOptionsTableClientIdField;
-   private String  variantOptionsTableClientSecretField;
-   private String  variantOptionsTableName;
+   private BackendVariantsConfig backendVariantsConfig;
 
    // todo - at some point, we may want to apply this to secret properties on subclasses?
    // @JsonFilter("secretsFilter")
 
+   @Deprecated(since = "Replaced by filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
+   private String variantOptionsTableTypeField; // a field on which to filter the variant-options table, to limit which records in it are available as variants
+
+   @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
+   private String variantOptionsTableTypeValue; // value for the type-field, to limit which records in it are available as variants; but also, the key in the session.backendVariants map!
+
 
 
    /*******************************************************************************
@@ -394,22 +396,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableIdField
-    *******************************************************************************/
-   public String getVariantOptionsTableIdField()
-   {
-      return (this.variantOptionsTableIdField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableIdField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
    public void setVariantOptionsTableIdField(String variantOptionsTableIdField)
    {
-      this.variantOptionsTableIdField = variantOptionsTableIdField;
+      /////////////////////////////////////////////////
+      // noop as we migrate to backendVariantsConfig //
+      /////////////////////////////////////////////////
    }
 
 
@@ -417,30 +412,24 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableIdField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's primary key")
    public QBackendMetaData withVariantOptionsTableIdField(String variantOptionsTableIdField)
    {
-      this.variantOptionsTableIdField = variantOptionsTableIdField;
+      this.setVariantOptionsTableIdField(variantOptionsTableIdField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableNameField
-    *******************************************************************************/
-   public String getVariantOptionsTableNameField()
-   {
-      return (this.variantOptionsTableNameField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableNameField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
    public void setVariantOptionsTableNameField(String variantOptionsTableNameField)
    {
-      this.variantOptionsTableNameField = variantOptionsTableNameField;
+      /////////////////////////////////////////////////
+      // noop as we migrate to backendVariantsConfig //
+      /////////////////////////////////////////////////
    }
 
 
@@ -448,30 +437,26 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableNameField
     *******************************************************************************/
+   @Deprecated(since = "backendVariantsConfig will infer this from the variant options table's recordLabel")
    public QBackendMetaData withVariantOptionsTableNameField(String variantOptionsTableNameField)
    {
-      this.variantOptionsTableNameField = variantOptionsTableNameField;
+      this.setVariantOptionsTableNameField(variantOptionsTableNameField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableTypeField
-    *******************************************************************************/
-   public String getVariantOptionsTableTypeField()
-   {
-      return (this.variantOptionsTableTypeField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableTypeField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
    public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField)
    {
       this.variantOptionsTableTypeField = variantOptionsTableTypeField;
+      if(this.variantOptionsTableTypeValue != null)
+      {
+         this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
+      }
    }
 
 
@@ -479,30 +464,28 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableTypeField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter")
    public QBackendMetaData withVariantOptionsTableTypeField(String variantOptionsTableTypeField)
    {
-      this.variantOptionsTableTypeField = variantOptionsTableTypeField;
+      this.setVariantOptionsTableTypeField(variantOptionsTableTypeField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableTypeValue
-    *******************************************************************************/
-   public String getVariantOptionsTableTypeValue()
-   {
-      return (this.variantOptionsTableTypeValue);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableTypeValue
     *******************************************************************************/
+   @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
    public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
    {
+      this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue);
+
       this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
+      if(this.variantOptionsTableTypeField != null)
+      {
+         this.getOrWithNewBackendVariantsConfig().setOptionsFilter(new QQueryFilter(new QFilterCriteria(variantOptionsTableTypeField, QCriteriaOperator.EQUALS, variantOptionsTableTypeValue)));
+      }
    }
 
 
@@ -510,30 +493,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableTypeValue
     *******************************************************************************/
+   @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter")
    public QBackendMetaData withVariantOptionsTableTypeValue(String variantOptionsTableTypeValue)
    {
-      this.variantOptionsTableTypeValue = variantOptionsTableTypeValue;
+      this.setVariantOptionsTableTypeValue(variantOptionsTableTypeValue);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableUsernameField
-    *******************************************************************************/
-   public String getVariantOptionsTableUsernameField()
-   {
-      return (this.variantOptionsTableUsernameField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableUsernameField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
    {
-      this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.USERNAME, variantOptionsTableUsernameField);
    }
 
 
@@ -541,30 +516,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableUsernameField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableUsernameField(String variantOptionsTableUsernameField)
    {
-      this.variantOptionsTableUsernameField = variantOptionsTableUsernameField;
+      this.setVariantOptionsTableUsernameField(variantOptionsTableUsernameField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTablePasswordField
-    *******************************************************************************/
-   public String getVariantOptionsTablePasswordField()
-   {
-      return (this.variantOptionsTablePasswordField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTablePasswordField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
    {
-      this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.PASSWORD, variantOptionsTablePasswordField);
    }
 
 
@@ -572,30 +539,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTablePasswordField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTablePasswordField(String variantOptionsTablePasswordField)
    {
-      this.variantOptionsTablePasswordField = variantOptionsTablePasswordField;
+      this.setVariantOptionsTablePasswordField(variantOptionsTablePasswordField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableApiKeyField
-    *******************************************************************************/
-   public String getVariantOptionsTableApiKeyField()
-   {
-      return (this.variantOptionsTableApiKeyField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableApiKeyField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
    {
-      this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.API_KEY, variantOptionsTableApiKeyField);
    }
 
 
@@ -603,30 +562,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableApiKeyField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableApiKeyField(String variantOptionsTableApiKeyField)
    {
-      this.variantOptionsTableApiKeyField = variantOptionsTableApiKeyField;
+      this.setVariantOptionsTableApiKeyField(variantOptionsTableApiKeyField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableName
-    *******************************************************************************/
-   public String getVariantOptionsTableName()
-   {
-      return (this.variantOptionsTableName);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableName
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.tableName")
    public void setVariantOptionsTableName(String variantOptionsTableName)
    {
-      this.variantOptionsTableName = variantOptionsTableName;
+      this.getOrWithNewBackendVariantsConfig().withOptionsTableName(variantOptionsTableName);
    }
 
 
@@ -634,9 +585,10 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableName
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.tableName")
    public QBackendMetaData withVariantOptionsTableName(String variantOptionsTableName)
    {
-      this.variantOptionsTableName = variantOptionsTableName;
+      this.setVariantOptionsTableName(variantOptionsTableName);
       return (this);
    }
 
@@ -651,22 +603,15 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
       qInstance.addBackend(this);
    }
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientIdField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientIdField()
-   {
-      return (this.variantOptionsTableClientIdField);
-   }
-
 
 
    /*******************************************************************************
     ** Setter for variantOptionsTableClientIdField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
    {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_ID, variantOptionsTableClientIdField);
    }
 
 
@@ -674,30 +619,22 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableClientIdField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableClientIdField(String variantOptionsTableClientIdField)
    {
-      this.variantOptionsTableClientIdField = variantOptionsTableClientIdField;
+      this.setVariantOptionsTableClientIdField(variantOptionsTableClientIdField);
       return (this);
    }
 
 
 
-   /*******************************************************************************
-    ** Getter for variantOptionsTableClientSecretField
-    *******************************************************************************/
-   public String getVariantOptionsTableClientSecretField()
-   {
-      return (this.variantOptionsTableClientSecretField);
-   }
-
-
-
    /*******************************************************************************
     ** Setter for variantOptionsTableClientSecretField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public void setVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
    {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+      this.getOrWithNewBackendVariantsConfig().withBackendSettingSourceFieldName(LegacyBackendVariantSetting.CLIENT_SECRET, variantOptionsTableClientSecretField);
    }
 
 
@@ -705,11 +642,55 @@ public class QBackendMetaData implements TopLevelMetaDataInterface
    /*******************************************************************************
     ** Fluent setter for variantOptionsTableClientSecretField
     *******************************************************************************/
+   @Deprecated(since = "Replaced by backendVariantsConfig.backendSettingSourceFieldNameMap")
    public QBackendMetaData withVariantOptionsTableClientSecretField(String variantOptionsTableClientSecretField)
    {
-      this.variantOptionsTableClientSecretField = variantOptionsTableClientSecretField;
+      this.setVariantOptionsTableClientSecretField(variantOptionsTableClientSecretField);
       return (this);
    }
 
 
+
+   /*******************************************************************************
+    ** Getter for backendVariantsConfig
+    *******************************************************************************/
+   public BackendVariantsConfig getBackendVariantsConfig()
+   {
+      return (this.backendVariantsConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backendVariantsConfig
+    *******************************************************************************/
+   public void setBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
+   {
+      this.backendVariantsConfig = backendVariantsConfig;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backendVariantsConfig
+    *******************************************************************************/
+   public QBackendMetaData withBackendVariantsConfig(BackendVariantsConfig backendVariantsConfig)
+   {
+      this.backendVariantsConfig = backendVariantsConfig;
+      return (this);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private BackendVariantsConfig getOrWithNewBackendVariantsConfig()
+   {
+      if(backendVariantsConfig == null)
+      {
+         setBackendVariantsConfig(new BackendVariantsConfig());
+      }
+      return backendVariantsConfig;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -56,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData;
@@ -89,6 +90,7 @@ public class QInstance
    ////////////////////////////////////////////////////////////////////////////////////////////
    // Important to use LinkedHashmap here, to preserve the order in which entries are added. //
    ////////////////////////////////////////////////////////////////////////////////////////////
+   private Map<String, QBitMetaData>             qBits                = new LinkedHashMap<>();
    private Map<String, QTableMetaData>           tables               = new LinkedHashMap<>();
    private Map<String, QJoinMetaData>            joins                = new LinkedHashMap<>();
    private Map<String, QPossibleValueSource>     possibleValueSources = new LinkedHashMap<>();
@@ -1489,6 +1491,7 @@ public class QInstance
    }
 
 
+
    /*******************************************************************************
     ** Getter for metaDataFilter
     *******************************************************************************/
@@ -1519,4 +1522,68 @@ public class QInstance
    }
 
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addQBit(QBitMetaData qBitMetaData)
+   {
+      List<String> missingParts = new ArrayList<>();
+      if(!StringUtils.hasContent(qBitMetaData.getGroupId()))
+      {
+         missingParts.add("groupId");
+      }
+      if(!StringUtils.hasContent(qBitMetaData.getArtifactId()))
+      {
+         missingParts.add("artifactId");
+      }
+      if(!StringUtils.hasContent(qBitMetaData.getVersion()))
+      {
+         missingParts.add("version");
+
+      }
+      if(!missingParts.isEmpty())
+      {
+         throw (new IllegalArgumentException("Attempted to add a qBit without a " + StringUtils.joinWithCommasAndAnd(missingParts)));
+      }
+
+      String name = qBitMetaData.getName();
+      if(this.qBits.containsKey(name))
+      {
+         throw (new IllegalArgumentException("Attempted to add a second qBit with name (formed from 'groupId:artifactId:version[:namespace]'): " + name));
+      }
+      this.qBits.put(name, qBitMetaData);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for qBits
+    *******************************************************************************/
+   public Map<String, QBitMetaData> getQBits()
+   {
+      return (this.qBits);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for qBits
+    *******************************************************************************/
+   public void setQBits(Map<String, QBitMetaData> qBits)
+   {
+      this.qBits = qBits;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for qBits
+    *******************************************************************************/
+   public QInstance withQBits(Map<String, QBitMetaData> qBits)
+   {
+      this.qBits = qBits;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/audits/AuditLevel.java

@@ -30,4 +30,5 @@ public enum AuditLevel
    NONE,
    RECORD,
    FIELD
+   // idea:  only audit changes to fields, e.g., on edit.  though, is that a different dimension than this?
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java

@@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 
 
 import java.io.Serializable;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
 import com.kingsrook.qqq.backend.core.utils.Pair;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 
 
@@ -42,13 +46,13 @@ public enum AdornmentType
    REVEAL,
    FILE_DOWNLOAD,
    FILE_UPLOAD,
+   TOOLTIP,
    ERROR;
    //////////////////////////////////////////////////////////////////////////
    // keep these values in sync with AdornmentType.ts in qqq-frontend-core //
    //////////////////////////////////////////////////////////////////////////
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -56,6 +60,7 @@ public enum AdornmentType
    {
       String TARGET                       = "target";
       String TO_RECORD_FROM_TABLE         = "toRecordFromTable";
+      String TO_RECORD_FROM_TABLE_DYNAMIC = "toRecordFromTableDynamic";
    }
 
 
@@ -72,6 +77,8 @@ public enum AdornmentType
       String SUPPLEMENTAL_PROCESS_NAME   = "supplementalProcessName";
       String SUPPLEMENTAL_CODE_REFERENCE = "supplementalCodeReference";
 
+      String DOWNLOAD_URL_DYNAMIC = "downloadUrlDynamic";
+
       ////////////////////////////////////////////////////
       // use these two together, as in:                 //
       // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" //
@@ -79,6 +86,17 @@ public enum AdornmentType
       ////////////////////////////////////////////////////
       String FILE_NAME_FORMAT        = "fileNameFormat";
       String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields";
+
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName)
+      {
+         return ("/data/" + tableName + "/"
+            + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8).replace("+", "%20") + "/"
+            + fieldName + "/"
+            + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8).replace("+", "%20"));
+      }
    }
 
 
@@ -229,4 +247,15 @@ public enum AdornmentType
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public interface TooltipValues
+   {
+      String STATIC_TEXT = "staticText";
+      String TOOLTIP_DYNAMIC = "tooltipDynamic";
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendTableMetaData.java

@@ -86,7 +86,6 @@ public class QFrontendTableMetaData
    //////////////////////////////////////////////////////////////////////////////////
 
 
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -170,7 +169,7 @@ public class QFrontendTableMetaData
       if(backend != null && backend.getUsesVariants())
       {
          usesVariants = true;
-         variantTableLabel = QContext.getQInstance().getTable(backend.getVariantOptionsTableName()).getLabel();
+         variantTableLabel = QContext.getQInstance().getTable(backend.getBackendVariantsConfig().getOptionsTableName()).getLabel();
       }
 
       this.helpContents = tableMetaData.getHelpContent();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java

@@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware;
 import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -46,11 +47,14 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
  ** Meta-Data to define a process in a QQQ instance.
  **
  *******************************************************************************/
-public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface
+public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware
 {
    private String name;
    private String label;
    private String tableName;
+
+   private String sourceQBitName;
+
    private boolean               isHidden = false;
    private BasepullConfiguration basepullConfiguration;
    private QPermissionRules      permissionRules;
@@ -870,6 +874,7 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
    }
 
 
+
    /*******************************************************************************
     ** Getter for processTracerCodeReference
     *******************************************************************************/
@@ -900,4 +905,37 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi
    }
 
 
+   /*******************************************************************************
+    ** Getter for sourceQBitName
+    *******************************************************************************/
+   @Override
+   public String getSourceQBitName()
+   {
+      return (this.sourceQBitName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceQBitName
+    *******************************************************************************/
+   @Override
+   public void setSourceQBitName(String sourceQBitName)
+   {
+      this.sourceQBitName = sourceQBitName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceQBitName
+    *******************************************************************************/
+   @Override
+   public QProcessMetaData withSourceQBitName(String sourceQBitName)
+   {
+      this.sourceQBitName = sourceQBitName;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java

@@ -62,6 +62,7 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat
    private String parentTableName; // e.g., order
    private String foreignKeyFieldName; // e.g., orderId
 
+   private Class<?> sourceClass;
 
 
    /***************************************************************************
@@ -102,4 +103,37 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat
       return (join);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public ChildJoinFromRecordEntityGenericMetaDataProducer withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java

@@ -57,6 +57,8 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem
 
    private ChildRecordListWidget childRecordListWidget;
 
+   private Class<?> sourceClass;
+
 
 
    /***************************************************************************
@@ -111,4 +113,36 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem
       return (widget);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java

@@ -40,6 +40,10 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer<T extends Serializ
    private final String                 name;
    private final PossibleValueEnum<T>[] values;
 
+   private Class<?> sourceClass;
+
+
+
 
 
    /*******************************************************************************
@@ -62,4 +66,37 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer<T extends Serializ
    {
       return (QPossibleValueSource.newForEnum(name, values));
    }
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public PossibleValueSourceOfEnumGenericMetaDataProducer<T> withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java

@@ -37,6 +37,7 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa
 {
    private final String tableName;
 
+   private Class<?> sourceClass;
 
 
    /*******************************************************************************
@@ -58,4 +59,38 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa
    {
       return (QPossibleValueSource.newForTable(tableName));
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public PossibleValueSourceOfTableGenericMetaDataProducer withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java

@@ -48,6 +48,7 @@ public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProdu
 
    private static MetaDataCustomizerInterface<QTableMetaData> defaultMetaDataCustomizer = null;
 
+   private Class<?> sourceClass;
 
 
    /*******************************************************************************
@@ -154,4 +155,37 @@ public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProdu
       RecordEntityToTableGenericMetaDataProducer.defaultMetaDataCustomizer = defaultMetaDataCustomizer;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for sourceClass
+    **
+    *******************************************************************************/
+   public Class<?> getSourceClass()
+   {
+      return sourceClass;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceClass
+    **
+    *******************************************************************************/
+   public void setSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+   }
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceClass
+    **
+    *******************************************************************************/
+   public RecordEntityToTableGenericMetaDataProducer withSourceClass(Class<?> sourceClass)
+   {
+      this.sourceClass = sourceClass;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java

@@ -0,0 +1,122 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+/***************************************************************************
+ ** Common (maybe)? qbit config pattern, where the qbit may be able to provide
+ ** a particular table, or, the application may supply it itself.
+ **
+ ** If the qbit provides it, then we need to be told (by the application)
+ ** what backendName to use for the table.
+ **
+ ** Else if the application supplies it, it needs to tell the qBit what the
+ ** tableName is.
+ ***************************************************************************/
+public class ProvidedOrSuppliedTableConfig
+{
+   private boolean doProvideTable;
+   private String  backendName;
+   private String  tableName;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public ProvidedOrSuppliedTableConfig(boolean doProvideTable, String backendName, String tableName)
+   {
+      this.doProvideTable = doProvideTable;
+      this.backendName = backendName;
+      this.tableName = tableName;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static ProvidedOrSuppliedTableConfig provideTableUsingBackendNamed(String backendName)
+   {
+      return (new ProvidedOrSuppliedTableConfig(true, backendName, null));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static ProvidedOrSuppliedTableConfig useSuppliedTaleNamed(String tableName)
+   {
+      return (new ProvidedOrSuppliedTableConfig(false, null, tableName));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public String getEffectiveTableName(String tableNameIfProviding)
+   {
+      if (getDoProvideTable())
+      {
+         return tableNameIfProviding;
+      }
+      else
+      {
+         return getTableName();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    **
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return tableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for doProvideTable
+    **
+    *******************************************************************************/
+   public boolean getDoProvideTable()
+   {
+      return doProvideTable;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backendName
+    **
+    *******************************************************************************/
+   public String getBackendName()
+   {
+      return backendName;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java

@@ -0,0 +1,71 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput;
+
+
+/*******************************************************************************
+ ** extension of MetaDataProducerInterface, designed for producing meta data
+ ** within a (java-defined, at this time) QBit.
+ **
+ ** Specifically exists to accept the QBitConfig as a type parameter and a value,
+ ** easily accessed in the producer's methods as getQBitConfig()
+ *******************************************************************************/
+public abstract class QBitComponentMetaDataProducer<T extends MetaDataProducerOutput, C extends QBitConfig> implements MetaDataProducerInterface<T>
+{
+   private C qBitConfig = null;
+
+
+
+   /*******************************************************************************
+    ** Getter for qBitConfig
+    *******************************************************************************/
+   public C getQBitConfig()
+   {
+      return (this.qBitConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for qBitConfig
+    *******************************************************************************/
+   public void setQBitConfig(C qBitConfig)
+   {
+      this.qBitConfig = qBitConfig;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for qBitConfig
+    *******************************************************************************/
+   public QBitComponentMetaDataProducer<T, C> withQBitConfig(C qBitConfig)
+   {
+      this.qBitConfig = qBitConfig;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java

@@ -0,0 +1,110 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Interface for configuration settings used both in the production of meta-data
+ ** for a QBit, but also at runtime, e.g., to be aware of exactly how the qbit
+ ** has been incorporated into an application.
+ **
+ ** For example:
+ ** - should the QBit define certain tables, or will they be supplied by the application?
+ ** - what other meta-data names should the qbit reference (backends, schedulers)
+ ** - what meta-data-customizer(s) should be used?
+ **
+ ** When implementing a QBit, you'll implement this interface - adding whatever
+ ** (if any) properties you need, and if you have any rules, then overriding
+ ** the validate method (ideally the one that takes the List-of-String errors)
+ **
+ ** When using a QBit, you'll create an instance of the QBit's config object,
+ ** and pass it through to the QBit producer.
+ *******************************************************************************/
+public interface QBitConfig extends Serializable
+{
+   QLogger LOG = QLogger.getLogger(QBitConfig.class);
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default void validate(QInstance qInstance) throws QBitConfigValidationException
+   {
+      List<String> errors = new ArrayList<>();
+
+      try
+      {
+         validate(qInstance, errors);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error validating QBitConfig: " + this.getClass().getName(), e);
+      }
+
+      if(!errors.isEmpty())
+      {
+         throw (new QBitConfigValidationException(this, errors));
+      }
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default void validate(QInstance qInstance, List<String> errors)
+   {
+      /////////////////////////////////////
+      // nothing to validate by default! //
+      /////////////////////////////////////
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default boolean assertCondition(boolean condition, String message, List<String> errors)
+   {
+      if(!condition)
+      {
+         errors.add(message);
+      }
+      return (condition);
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
+   {
+      return (null);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java

@@ -0,0 +1,44 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** thrown by QBitConfig.validate() if there's an issue.
+ *******************************************************************************/
+public class QBitConfigValidationException extends QException
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public QBitConfigValidationException(QBitConfig qBitConfig, List<String> errors)
+   {
+      super("Validation failed for QBitConfig: " + qBitConfig.getClass().getName() + ":\n" + StringUtils.join("\n", errors));
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java

@@ -0,0 +1,237 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Meta-data to define an active QBit in a QQQ Instance.
+ **
+ ** The unique "name" for the QBit is composed of its groupId and artifactId
+ ** (maven style).  There is also a version - but it is not part of the unique
+ ** name.  But - there is also a namespace attribute, which IS part of the
+ ** unique name.  This will (eventually?) allow us to have multiple instances
+ ** of the same qbit in a qInstance at the same time (e.g., 2 versions of some
+ ** table, which should be namespace-prefixed);
+ **
+ ** QBitMetaData also retains the QBitConfig that was used to produce the QBit.
+ **
+ ** Some meta-data objects are aware of the fact that they may have come from a
+ ** QBit - see SourceQBitAware interface.  These objects can get their source
+ ** QBitMetaData (this object) and its config,via that interface.
+ *******************************************************************************/
+public class QBitMetaData implements TopLevelMetaDataInterface
+{
+   private String groupId;
+   private String artifactId;
+   private String version;
+   private String namespace;
+
+   private QBitConfig config;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public String getName()
+   {
+      String name = groupId + ":" + artifactId;
+      if(StringUtils.hasContent(namespace))
+      {
+         name += ":" + namespace;
+      }
+      return name;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void addSelfToInstance(QInstance qInstance)
+   {
+      qInstance.addQBit(this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for config
+    *******************************************************************************/
+   public QBitConfig getConfig()
+   {
+      return (this.config);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for config
+    *******************************************************************************/
+   public void setConfig(QBitConfig config)
+   {
+      this.config = config;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for config
+    *******************************************************************************/
+   public QBitMetaData withConfig(QBitConfig config)
+   {
+      this.config = config;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for groupId
+    *******************************************************************************/
+   public String getGroupId()
+   {
+      return (this.groupId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for groupId
+    *******************************************************************************/
+   public void setGroupId(String groupId)
+   {
+      this.groupId = groupId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for groupId
+    *******************************************************************************/
+   public QBitMetaData withGroupId(String groupId)
+   {
+      this.groupId = groupId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for artifactId
+    *******************************************************************************/
+   public String getArtifactId()
+   {
+      return (this.artifactId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for artifactId
+    *******************************************************************************/
+   public void setArtifactId(String artifactId)
+   {
+      this.artifactId = artifactId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for artifactId
+    *******************************************************************************/
+   public QBitMetaData withArtifactId(String artifactId)
+   {
+      this.artifactId = artifactId;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for version
+    *******************************************************************************/
+   public String getVersion()
+   {
+      return (this.version);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for version
+    *******************************************************************************/
+   public void setVersion(String version)
+   {
+      this.version = version;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for version
+    *******************************************************************************/
+   public QBitMetaData withVersion(String version)
+   {
+      this.version = version;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for namespace
+    *******************************************************************************/
+   public String getNamespace()
+   {
+      return (this.namespace);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for namespace
+    *******************************************************************************/
+   public void setNamespace(String namespace)
+   {
+      this.namespace = namespace;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for namespace
+    *******************************************************************************/
+   public QBitMetaData withNamespace(String namespace)
+   {
+      this.namespace = namespace;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java

@@ -0,0 +1,117 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** interface for how a QBit's meta-data gets produced and added to a QInstance.
+ **
+ ** When implementing a QBit, you'll implement this interface:
+ ** - adding a QBitConfig subclass as a property
+ ** - overriding the produce(qInstance, namespace) method - where you'll:
+ ** -- create and add your QBitMetaData
+ ** -- call MetaDataProducerHelper.findProducers
+ ** -- hand off to finishProducing() in this interface
+ **
+ ** When using a QBit, you'll create an instance of the QBit's config object,
+ ** pass it in to the producer, then call produce, ala:
+ **
+ ** new SomeQBitProducer()
+ **   .withQBitConfig(someQBitConfig)
+ **   .produce(qInstance);
+ **
+ *******************************************************************************/
+public interface QBitProducer
+{
+   QLogger LOG = QLogger.getLogger(QBitProducer.class);
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default void produce(QInstance qInstance) throws QException
+   {
+      produce(qInstance, null);
+   }
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   void produce(QInstance qInstance, String namespace) throws QException;
+
+
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   default <C extends QBitConfig> void finishProducing(QInstance qInstance, QBitMetaData qBitMetaData, C qBitConfig, List<MetaDataProducerInterface<?>> producers) throws QException
+   {
+      qBitConfig.validate(qInstance);
+
+      ///////////////////////////////
+      // todo - move to base class //
+      ///////////////////////////////
+      for(MetaDataProducerInterface<?> producer : producers)
+      {
+         if(producer instanceof QBitComponentMetaDataProducer<?, ?>)
+         {
+            QBitComponentMetaDataProducer<?, C> qBitComponentMetaDataProducer = (QBitComponentMetaDataProducer<?, C>) producer;
+            qBitComponentMetaDataProducer.setQBitConfig(qBitConfig);
+         }
+
+         if(!producer.isEnabled())
+         {
+            LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName()));
+            continue;
+         }
+
+         MetaDataProducerOutput output = producer.produce(qInstance);
+
+         /////////////////////////////////////////
+         // apply table customizer, if provided //
+         /////////////////////////////////////////
+         if(qBitConfig.getTableMetaDataCustomizer() != null && output instanceof QTableMetaData table)
+         {
+            output = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
+         }
+
+         /////////////////////////////////////////////////
+         // set source qbit, if output is aware of such //
+         /////////////////////////////////////////////////
+         if(output instanceof SourceQBitAware sourceQBitAware)
+         {
+            sourceQBitAware.setSourceQBitName(qBitMetaData.getName());
+         }
+
+         output.addSelfToInstance(qInstance);
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java

@@ -0,0 +1,77 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import com.kingsrook.qqq.backend.core.context.QContext;
+
+
+/*******************************************************************************
+ ** interface for meta data objects that may have come from a qbit, and where we
+ ** might want to get data about that qbit (e.g., config or meta-data).
+ *******************************************************************************/
+public interface SourceQBitAware
+{
+   /*******************************************************************************
+    ** Getter for sourceQBitName
+    *******************************************************************************/
+   String getSourceQBitName();
+
+
+   /*******************************************************************************
+    ** Setter for sourceQBitName
+    *******************************************************************************/
+   void setSourceQBitName(String sourceQBitName);
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceQBitName
+    *******************************************************************************/
+   Object withSourceQBitName(String sourceQBitName);
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default QBitMetaData getSourceQBit()
+   {
+      String qbitName = getSourceQBitName();
+      return (QContext.getQInstance().getQBits().get(qbitName));
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   default QBitConfig getSourceQBitConfig()
+   {
+      QBitMetaData sourceQBit = getSourceQBit();
+      if(sourceQBit == null)
+      {
+         return null;
+      }
+      else
+      {
+         return sourceQBit.getConfig();
+      }
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java

@@ -50,10 +50,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
 import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -61,7 +63,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
  ** Meta-Data to define a table in a QQQ instance.
  **
  *******************************************************************************/
-public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface
+public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware
 {
    private static final QLogger LOG = QLogger.getLogger(QTableMetaData.class);
 
@@ -72,6 +74,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
    private String  primaryKeyField;
    private boolean isHidden = false;
 
+   private String sourceQBitName;
+
    private Map<String, QFieldMetaData> fields;
    private List<UniqueKey>             uniqueKeys;
    private List<Association>           associations;
@@ -712,6 +716,25 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
 
 
 
+   /*******************************************************************************
+    ** Getter for sections
+    **
+    *******************************************************************************/
+   public QFieldSection getSection(String name)
+   {
+      for(QFieldSection qFieldSection : CollectionUtils.nonNullList(sections))
+      {
+         if(qFieldSection.getName().equals(name))
+         {
+            return (qFieldSection);
+         }
+      }
+
+      return (null);
+   }
+
+
+
    /*******************************************************************************
     ** Setter for sections
     **
@@ -1036,7 +1059,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
    {
       for(Capability disabledCapability : disabledCapabilities)
       {
-         withCapability(disabledCapability);
+         withoutCapability(disabledCapability);
       }
       return (this);
    }
@@ -1534,4 +1557,38 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData
       QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot);
    }
 
+
+   /*******************************************************************************
+    ** Getter for sourceQBitName
+    *******************************************************************************/
+   @Override
+   public String getSourceQBitName()
+   {
+      return (this.sourceQBitName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sourceQBitName
+    *******************************************************************************/
+   @Override
+   public void setSourceQBitName(String sourceQBitName)
+   {
+      this.sourceQBitName = sourceQBitName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sourceQBitName
+    *******************************************************************************/
+   @Override
+   public QTableMetaData withSourceQBitName(String sourceQBitName)
+   {
+      this.sourceQBitName = sourceQBitName;
+      return (this);
+   }
+
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactory.java

@@ -0,0 +1,221 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.tables;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+
+
+/*******************************************************************************
+ ** Factory class for creating "standard" qfield sections.  e.g., if you want
+ ** the same t1, t2, and t3 section on all your tables, use this class to
+ ** produce them.
+ **
+ ** You can change the default name & iconNames for those sections, but note,
+ ** this is a static/utility style class, so those settings are static fields.
+ **
+ ** The method customT2 is provided as not much of a shortcut over "doing it yourself",
+ ** but to allow all sections for a table to be produced through calls to this factory,
+ ** so they look more similar.
+ *******************************************************************************/
+public class SectionFactory
+{
+   private static String defaultT1name     = "identity";
+   private static String defaultT1iconName = "badge";
+   private static String defaultT2name     = "data";
+   private static String defaultT2iconName = "text_snippet";
+   private static String defaultT3name     = "dates";
+   private static String defaultT3iconName = "calendar_month";
+
+
+   /*******************************************************************************
+    ** private constructor, to enforce static usage, e.g., to make clear the fields
+    ** are static fields.
+    **
+    *******************************************************************************/
+   private SectionFactory()
+   {
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static QFieldSection defaultT1(String... fieldNames)
+   {
+      return new QFieldSection(defaultT1name, new QIcon().withName(defaultT1iconName), Tier.T1, List.of(fieldNames));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static QFieldSection defaultT2(String... fieldNames)
+   {
+      return new QFieldSection(defaultT2name, new QIcon().withName(defaultT2iconName), Tier.T2, List.of(fieldNames));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static QFieldSection customT2(String name, QIcon icon, String... fieldNames)
+   {
+      return new QFieldSection(name, icon, Tier.T2, List.of(fieldNames));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static QFieldSection defaultT3(String... fieldNames)
+   {
+      return new QFieldSection(defaultT3name, new QIcon().withName(defaultT3iconName), Tier.T3, List.of(fieldNames));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultT1name
+    *******************************************************************************/
+   public static String getDefaultT1name()
+   {
+      return (SectionFactory.defaultT1name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultT1name
+    *******************************************************************************/
+   public static void setDefaultT1name(String defaultT1name)
+   {
+      SectionFactory.defaultT1name = defaultT1name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultT1iconName
+    *******************************************************************************/
+   public static String getDefaultT1iconName()
+   {
+      return (SectionFactory.defaultT1iconName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultT1iconName
+    *******************************************************************************/
+   public static void setDefaultT1iconName(String defaultT1iconName)
+   {
+      SectionFactory.defaultT1iconName = defaultT1iconName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultT2name
+    *******************************************************************************/
+   public static String getDefaultT2name()
+   {
+      return (SectionFactory.defaultT2name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultT2name
+    *******************************************************************************/
+   public static void setDefaultT2name(String defaultT2name)
+   {
+      SectionFactory.defaultT2name = defaultT2name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultT2iconName
+    *******************************************************************************/
+   public static String getDefaultT2iconName()
+   {
+      return (SectionFactory.defaultT2iconName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultT2iconName
+    *******************************************************************************/
+   public static void setDefaultT2iconName(String defaultT2iconName)
+   {
+      SectionFactory.defaultT2iconName = defaultT2iconName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultT3name
+    *******************************************************************************/
+   public static String getDefaultT3name()
+   {
+      return (SectionFactory.defaultT3name);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultT3name
+    *******************************************************************************/
+   public static void setDefaultT3name(String defaultT3name)
+   {
+      SectionFactory.defaultT3name = defaultT3name;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for defaultT3iconName
+    *******************************************************************************/
+   public static String getDefaultT3iconName()
+   {
+      return (SectionFactory.defaultT3iconName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for defaultT3iconName
+    *******************************************************************************/
+   public static void setDefaultT3iconName(String defaultT3iconName)
+   {
+      SectionFactory.defaultT3iconName = defaultT3iconName;
+   }
+
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java

@@ -0,0 +1,89 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.tables;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+
+
+/*******************************************************************************
+ ** possible-value source provider for the `Tables` PVS - a list of all tables
+ ** in an application/qInstance.
+ *******************************************************************************/
+public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider<String>
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QPossibleValue<String> getPossibleValue(Serializable idValue)
+   {
+      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue));
+      if(table != null && !table.getIsHidden())
+      {
+         PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
+         if(PermissionCheckResult.ALLOW.equals(permissionCheckResult))
+         {
+            return (new QPossibleValue<>(table.getName(), table.getLabel()));
+         }
+      }
+
+      return null;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+   {
+      /////////////////////////////////////////////////////////////////////////////////////
+      // build all of the possible values (note, will be filtered by user's permissions) //
+      /////////////////////////////////////////////////////////////////////////////////////
+      List<QPossibleValue<String>> allPossibleValues = new ArrayList<>();
+      for(QTableMetaData table : QContext.getQInstance().getTables().values())
+      {
+         QPossibleValue<String> possibleValue = getPossibleValue(table.getName());
+         if(possibleValue != null)
+         {
+            allPossibleValues.add(possibleValue);
+         }
+      }
+
+      return completeCustomPVSSearch(input, allPossibleValues);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java

@@ -22,17 +22,11 @@
 package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
-import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-import org.apache.commons.lang.BooleanUtils;
 
 
 /*******************************************************************************
@@ -51,22 +45,10 @@ public class TablesPossibleValueSourceMetaDataProvider
    {
       QPossibleValueSource possibleValueSource = new QPossibleValueSource()
          .withName(NAME)
-         .withType(QPossibleValueSourceType.ENUM)
+         .withType(QPossibleValueSourceType.CUSTOM)
+         .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class))
          .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
 
-      List<QPossibleValue<?>> enumValues = new ArrayList<>();
-      for(QTableMetaData table : qInstance.getTables().values())
-      {
-         if(BooleanUtils.isNotTrue(table.getIsHidden()))
-         {
-            String label = StringUtils.hasContent(table.getLabel()) ? table.getLabel() : QInstanceEnricher.nameToLabel(table.getName());
-            enumValues.add(new QPossibleValue<>(table.getName(), label));
-         }
-      }
-
-      enumValues.sort(Comparator.comparing(QPossibleValue::getLabel));
-
-      possibleValueSource.withEnumValues(enumValues);
       return (possibleValueSource);
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantSetting.java

@@ -0,0 +1,31 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.variants;
+
+
+/*******************************************************************************
+ ** interface to be implemented by enums (presumably) that define the possible
+ ** settings a particular backend type can get from a variant record.
+ *******************************************************************************/
+public interface BackendVariantSetting
+{
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java

@@ -0,0 +1,225 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.variants;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+
+
+/*******************************************************************************
+ ** Configs for how a backend that uses variants works.  Specifically:
+ **
+ ** - the variant "type key" - e.g., key for variants map in session.
+ ** - what table supplies the variant options (optionsTableName
+ ** - an optional filter to apply to that options table
+ ** - a map of the settings that a backend gets from its variant table to the
+ ** field names in that table that they come from.  e.g., a backend may have a
+ ** username attribute, whose value comes from a field named "theUser" in the
+ ** variant options table.
+ ** - an optional code reference to a variantRecordLookupFunction - to customize
+ ** how the variant record is looked up (such as, adding joined or other custom
+ ** fields).
+ *******************************************************************************/
+public class BackendVariantsConfig
+{
+   private String variantTypeKey;
+
+   private String         optionsTableName;
+   private QQueryFilter   optionsFilter;
+   private QCodeReference variantRecordLookupFunction;
+
+   private Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap;
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getOptionsTableName()
+   {
+      return (this.optionsTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableName
+    *******************************************************************************/
+   public void setOptionsTableName(String optionsTableName)
+   {
+      this.optionsTableName = optionsTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for filter
+    *******************************************************************************/
+   public QQueryFilter getOptionsFilter()
+   {
+      return (this.optionsFilter);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for filter
+    *******************************************************************************/
+   public void setOptionsFilter(QQueryFilter optionsFilter)
+   {
+      this.optionsFilter = optionsFilter;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backendSettingSourceFieldNameMap
+    *******************************************************************************/
+   public Map<BackendVariantSetting, String> getBackendSettingSourceFieldNameMap()
+   {
+      return (this.backendSettingSourceFieldNameMap);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backendSettingSourceFieldNameMap
+    *******************************************************************************/
+   public void setBackendSettingSourceFieldNameMap(Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap)
+   {
+      this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backendSettingSourceFieldNameMap
+    *******************************************************************************/
+   public BackendVariantsConfig withBackendSettingSourceFieldName(BackendVariantSetting backendVariantSetting, String sourceFieldName)
+   {
+      if(this.backendSettingSourceFieldNameMap == null)
+      {
+         this.backendSettingSourceFieldNameMap = new HashMap<>();
+      }
+      this.backendSettingSourceFieldNameMap.put(backendVariantSetting, sourceFieldName);
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backendSettingSourceFieldNameMap
+    *******************************************************************************/
+   public BackendVariantsConfig withBackendSettingSourceFieldNameMap(Map<BackendVariantSetting, String> backendSettingSourceFieldNameMap)
+   {
+      this.backendSettingSourceFieldNameMap = backendSettingSourceFieldNameMap;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for variantTypeKey
+    *******************************************************************************/
+   public String getVariantTypeKey()
+   {
+      return (this.variantTypeKey);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantTypeKey
+    *******************************************************************************/
+   public void setVariantTypeKey(String variantTypeKey)
+   {
+      this.variantTypeKey = variantTypeKey;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantTypeKey
+    *******************************************************************************/
+   public BackendVariantsConfig withVariantTypeKey(String variantTypeKey)
+   {
+      this.variantTypeKey = variantTypeKey;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for optionsTableName
+    *******************************************************************************/
+   public BackendVariantsConfig withOptionsTableName(String optionsTableName)
+   {
+      this.optionsTableName = optionsTableName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for optionsFilter
+    *******************************************************************************/
+   public BackendVariantsConfig withOptionsFilter(QQueryFilter optionsFilter)
+   {
+      this.optionsFilter = optionsFilter;
+      return (this);
+   }
+
+
+   /*******************************************************************************
+    ** Getter for variantRecordLookupFunction
+    *******************************************************************************/
+   public QCodeReference getVariantRecordLookupFunction()
+   {
+      return (this.variantRecordLookupFunction);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for variantRecordLookupFunction
+    *******************************************************************************/
+   public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction)
+   {
+      this.variantRecordLookupFunction = variantRecordLookupFunction;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for variantRecordLookupFunction
+    *******************************************************************************/
+   public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction)
+   {
+      this.variantRecordLookupFunction = variantRecordLookupFunction;
+      return (this);
+   }
+
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java

@@ -0,0 +1,106 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.variants;
+
+
+import java.io.Serializable;
+import java.util.function.Function;
+import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
+
+
+/*******************************************************************************
+ ** Utility methods for backends working with Variants.
+ *******************************************************************************/
+public class BackendVariantsUtil
+{
+
+   /*******************************************************************************
+    ** Get the variant id from the session for the backend.
+    *******************************************************************************/
+   public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException
+   {
+      QSession session        = QContext.getQSession();
+      String   variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
+      if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey))
+      {
+         throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'"));
+      }
+      Serializable variantId = session.getBackendVariants().get(variantTypeKey);
+      return variantId;
+   }
+
+
+
+   /*******************************************************************************
+    ** For backends that use variants, look up the variant record (in theory, based
+    ** on an id in the session's backend variants map, then fetched from the backend's
+    ** variant options table.
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException
+   {
+      Serializable variantId = getVariantId(backendMetaData);
+
+      QRecord record;
+      if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null)
+      {
+         Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction());
+         if(o instanceof UnsafeFunction<?,?,?> unsafeFunction)
+         {
+            record = ((UnsafeFunction<Serializable, QRecord, QException>) unsafeFunction).apply(variantId);
+         }
+         else if(o instanceof Function<?,?> function)
+         {
+            record = ((Function<Serializable, QRecord>) function).apply(variantId);
+         }
+         else
+         {
+            throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)"));
+         }
+      }
+      else
+      {
+         GetInput getInput = new GetInput();
+         getInput.setShouldMaskPasswords(false);
+         getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
+         getInput.setPrimaryKey(variantId);
+         GetOutput getOutput = new GetAction().execute(getInput);
+
+         record = getOutput.getRecord();
+      }
+
+      if(record == null)
+      {
+         throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'"));
+      }
+      return record;
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/LegacyBackendVariantSetting.java

@@ -0,0 +1,39 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.variants;
+
+
+/*******************************************************************************
+ ** temporary class, while we migrate from original way that variants were set up
+ ** e.g., by calling 'variantOptionsTableUsernameField', to the new way, using
+ ** the BackendVariantsConfig which uses a map of enum constants.
+ **
+ ** so when those deprecated setters are removed, this enum can be too.
+ *****************************************************************************/
+public enum LegacyBackendVariantSetting implements BackendVariantSetting
+{
+   USERNAME,
+   PASSWORD,
+   API_KEY,
+   CLIENT_ID,
+   CLIENT_SECRET
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java

@@ -56,7 +56,16 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 /*******************************************************************************
  ** Note - exists under 2 names, for the RenderSavedReport process, and for the
- ** ScheduledReport table
+ ** ScheduledReport table (and can be used in your custom code too:
+ *
+ ** by default, in qqq backend core, we'll assume this widget is being used on the
+ ** view screen for a ScheduledReport, with field names that we know from that table.
+ ** But, allow it to be used on a different table (optionally with different field names),
+ ** coming from the input map.
+ **
+ ** e.g., that one may set in widget metaData as:
+ **    .withDefaultValue("tableName", "myTable")
+ **    .withDefaultValue("fieldNameId", "identifier"), etc.
  *******************************************************************************/
 public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRenderer
 {
@@ -88,11 +97,16 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere
          }
          else if(input.getQueryParams().containsKey("id"))
          {
-            QRecord scheduledReportRecord = new GetAction().executeForRecord(new GetInput(ScheduledReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get("id"))));
-            QRecord record                = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(scheduledReportRecord.getValueInteger("savedReportId"))));
+            String tableName              = input.getQueryParams().getOrDefault("tableName", ScheduledReport.TABLE_NAME);
+            String fieldNameId            = input.getQueryParams().getOrDefault("fieldNameId", "id");
+            String fieldNameSavedReportId = input.getQueryParams().getOrDefault("fieldNameSavedReportId", "savedReportId");
+            String fieldNameInputValues   = input.getQueryParams().getOrDefault("fieldNameInputValues", "inputValues");
+
+            QRecord hostRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get(fieldNameId))));
+            QRecord record     = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(hostRecord.getValueInteger(fieldNameSavedReportId))));
             savedReport = new SavedReport(record);
 
-            String inputValues = scheduledReportRecord.getValueString("inputValues");
+            String inputValues = hostRecord.getValueString(fieldNameInputValues);
             if(StringUtils.hasContent(inputValues))
             {
                JSONObject jsonObject = JsonUtils.toJSONObject(inputValues);
@@ -197,8 +211,8 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere
       }
       catch(Exception e)
       {
-         LOG.warn("Error rendering scheduled report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams())));
-         throw (new QException("Error rendering scheduled report values dynamic form widget", e));
+         LOG.warn("Error rendering report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams())));
+         throw (new QException("Error rendering report values dynamic form widget", e));
       }
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java

@@ -50,7 +50,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer;
 import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType;
 import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
@@ -351,8 +350,7 @@ public class SavedReportsMetaDataProvider
          .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "renderedReportStatusId")))
          .withSection(new QFieldSection("input", new QIcon().withName("input"), Tier.T2, List.of("userId", "reportFormat")))
          .withSection(new QFieldSection("output", new QIcon().withName("output"), Tier.T2, List.of("jobUuid", "resultPath", "rowCount", "errorMessage", "startTime", "endTime")))
-         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")))
-         .withoutCapabilities(Capability.allWriteCapabilities());
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
 
       table.getField("renderedReportStatusId").setAdornments(List.of(new FieldAdornment(AdornmentType.CHIP)
          .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.RUNNING.getId(), "pending", AdornmentType.ChipValues.COLOR_SECONDARY))

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java

@@ -55,6 +55,8 @@ public class BulkInsertExtractStep extends AbstractExtractStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
+      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
+
       int rowsAdded     = 0;
       int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java

@@ -22,16 +22,39 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
 import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
 import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class BulkInsertLoadStep extends LoadViaInsertStep
+public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface
 {
+   private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class);
+
+   private Serializable firstInsertedPrimaryKey = null;
+   private Serializable lastInsertedPrimaryKey  = null;
+
+
 
    /*******************************************************************************
     **
@@ -42,4 +65,66 @@ public class BulkInsertLoadStep extends LoadViaInsertStep
       return (QInputSource.USER);
    }
 
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   {
+      super.runOnePage(runBackendStepInput, runBackendStepOutput);
+
+      QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
+
+      List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
+      for(QRecord insertedRecord : insertedRecords)
+      {
+         if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
+         {
+            if(firstInsertedPrimaryKey == null)
+            {
+               firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+            }
+
+            lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
+   {
+      ArrayList<ProcessSummaryLineInterface> processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen);
+
+      try
+      {
+         if(firstInsertedPrimaryKey != null)
+         {
+            QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName"));
+            QFieldMetaData field = table.getField(table.getPrimaryKeyField());
+            if(field.getType().isNumeric())
+            {
+               ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey);
+               if(Objects.equals(firstInsertedPrimaryKey, lastInsertedPrimaryKey))
+               {
+                  idsLine.setMessage("Inserted " + field.getLabel() + " " + firstInsertedPrimaryKey);
+               }
+               idsLine.setCount(null);
+               processSummary.add(idsLine);
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error adding inserted-keys process summary line", e);
+      }
+
+      return (processSummary);
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java

@@ -32,16 +32,20 @@ import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
 import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang3.BooleanUtils;
 
@@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
    {
       try
       {
-         BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
+         QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput);
 
+         BulkLoadProfile bulkLoadProfile;
+         if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
+         {
+            //////////////////////////////////////////////////////////////////////////////
+            // if running headless, build bulkLoadProfile from the saved profile record //
+            //////////////////////////////////////////////////////////////////////////////
+            if(savedBulkLoadProfileRecord == null)
+            {
+               throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
+            }
+
+            SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
+
+            try
+            {
+               bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class);
+            }
+            catch(Exception e)
+            {
+               throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
+            }
+         }
+         else
+         {
             ///////////////////////////////////////////////////////////////////
             // read process values - construct a bulkLoadProfile out of them //
             ///////////////////////////////////////////////////////////////////
-         BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
+            bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput);
+         }
 
          /////////////////////////////////////////////////////////////////////////
          // put the list of bulk load profile into the process state - it's the //
@@ -183,6 +212,16 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
          /////////////////////////////////////////////////////////////////////////////////////////////////////
          runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
 
+         if(BulkInsertStepUtils.isHeadless(runBackendStepInput))
+         {
+            ////////////////////////////////////////////////////////////////////////
+            // if running headless, always go straight to the preview screen next //
+            // todo actually, we could make this execute, right?                  //
+            ////////////////////////////////////////////////////////////////////////
+            BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
+         }
+         else
+         {
             if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping))
             {
                //////////////////////////////////////////////////////////////////////////////////
@@ -200,6 +239,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
                BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput);
             }
          }
+      }
       catch(Exception e)
       {
          LOG.warn("Error in bulk insert receive mapping", e);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java

@@ -29,11 +29,13 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
+import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.json.JSONArray;
@@ -69,6 +71,18 @@ public class BulkInsertStepUtils
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput)
+   {
+      ArrayList<StorageInput> storageInputs = new ArrayList<>();
+      storageInputs.add(storageInput);
+      runProcessInput.addValue("theFile", storageInputs);
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -144,13 +158,62 @@ public class BulkInsertStepUtils
    /***************************************************************************
     **
     ***************************************************************************/
-   public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
+   public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
       Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId");
       if(savedBulkLoadProfileId != null)
       {
          QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId);
          runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord);
-      }
+         return (savedBulkLoadProfileRecord);
+      }
+
+      return (null);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static boolean isHeadless(RunBackendStepInput runBackendStepInput)
+   {
+      return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void setHeadless(RunProcessInput runProcessInput)
+   {
+      runProcessInput.addValue("isHeadless", true);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage)
+   {
+      runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput)
+   {
+      Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage");
+      if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage)
+      {
+         return (processTracerKeyRecordMessage);
+      }
+      
+      return (null);
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java

@@ -258,6 +258,11 @@ public class BulkLoadValueMapper
          valuesNotFound.add(value);
       }
 
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // todo - we should probably be doing a lot of what QJavalinImplementation.finishPossibleValuesRequest does here //
+      //  to apply possible-value filters.  difficult to pass values in, but needed...                                 //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
       searchPossibleValueSourceInput.setIdList(idList);
       searchPossibleValueSourceInput.setLimit(values.size());
       LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0)));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java

@@ -239,6 +239,10 @@ public class ColumnStatsStep implements BackendStep
 
          QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator();
          qPossibleValueTranslator.translatePossibleValuesInRecords(table, valueCounts, queryJoin == null ? null : List.of(queryJoin), null);
+
+         /////////////////////////////////////////////////////////////////////////////////////////////////
+         // todo - be aware of possible name collisions here!! (e.g., a table w/ a field named `count`) //
+         /////////////////////////////////////////////////////////////////////////////////////////////////
          QValueFormatter.setDisplayValuesInRecords(table, Map.of(fieldName, field, "count", countField), valueCounts);
 
          runBackendStepOutput.addValue("valueCounts", valueCounts);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java

@@ -100,8 +100,13 @@ public class RenderSavedReportExecuteStep implements BackendStep
          String       sendToEmailAddress  = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS);
          String       emailSubject        = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT);
          SavedReport  savedReport         = new SavedReport(runBackendStepInput.getRecords().get(0));
+
          String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport);
-         String       storageReference     = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension();
+         String storageReference     = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_REFERENCE);
+         if(!StringUtils.hasContent(storageReference))
+         {
+            storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension();
+         }
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // if sending an email (or emails), validate the addresses before doing anything so user gets error and can fix //
@@ -241,7 +246,7 @@ public class RenderSavedReportExecuteStep implements BackendStep
    /*******************************************************************************
     **
     *******************************************************************************/
-   private String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report)
+   public static String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report)
    {
       DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmm").withZone(ZoneId.systemDefault());
       String            datePart  = formatter.format(Instant.now());

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java

@@ -56,6 +56,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf
    public static final String FROM_EMAIL_ADDRESS            = "fromEmailAddress";
    public static final String REPLY_TO_EMAIL_ADDRESS        = "replyToEmailAddress";
    public static final String FIELD_NAME_STORAGE_TABLE_NAME = "storageTableName";
+   public static final String FIELD_NAME_STORAGE_REFERENCE  = "storageReference";
    public static final String FIELD_NAME_REPORT_FORMAT      = "reportFormat";
    public static final String FIELD_NAME_EMAIL_ADDRESS      = "reportDestinationEmailAddress";
    public static final String FIELD_NAME_EMAIL_SUBJECT      = "emailSubject";
@@ -81,6 +82,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf
                .withField(new QFieldMetaData(FROM_EMAIL_ADDRESS, QFieldType.STRING))
                .withField(new QFieldMetaData(REPLY_TO_EMAIL_ADDRESS, QFieldType.STRING))
                .withField(new QFieldMetaData(FIELD_NAME_STORAGE_TABLE_NAME, QFieldType.STRING))
+               .withField(new QFieldMetaData(FIELD_NAME_STORAGE_REFERENCE, QFieldType.STRING))
                .withRecordListMetaData(new QRecordListMetaData().withTableName(SavedReport.TABLE_NAME)))
             .withCode(new QCodeReference(RenderSavedReportPreStep.class)))
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -173,8 +173,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
     *******************************************************************************/
    protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List<Serializable> sourceKeyList)
    {
-      String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey;
-      return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList));
+      String         destinationTableForeignKeyFieldName = getSyncProcessConfig().destinationTableForeignKey;
+      String         destinationTableName                = getSyncProcessConfig().destinationTable;
+      QFieldMetaData destinationForeignKeyField          = QContext.getQInstance().getTable(destinationTableName).getField(destinationTableForeignKeyFieldName);
+
+      List<Serializable> sourceKeysInDestinationKeyTypeList = null;
+      if(sourceKeyList != null)
+      {
+         sourceKeysInDestinationKeyTypeList = new ArrayList<>();
+         for(Serializable sourceKey : sourceKeyList)
+         {
+            sourceKeysInDestinationKeyTypeList.add(ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKey));
+         }
+      }
+
+      return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyFieldName, QCriteriaOperator.IN, sourceKeysInDestinationKeyTypeList));
    }
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.tracing;
+
+
+/*******************************************************************************
+ ** Specialization of process tracer message, to indicate a 'key record' that was
+ ** used as an input or trigger to a process.
+ *******************************************************************************/
+public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage
+{
+   private final String  tableName;
+   private final Integer recordId;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public ProcessTracerKeyRecordMessage(String tableName, Integer recordId)
+   {
+      super("Process Key Record is " + tableName + " " + recordId);
+      this.tableName = tableName;
+      this.recordId = recordId;
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableName
+    *******************************************************************************/
+   public String getTableName()
+   {
+      return (this.tableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for recordId
+    *******************************************************************************/
+   public Integer getRecordId()
+   {
+      return (this.recordId);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java

@@ -22,13 +22,16 @@
 package com.kingsrook.qqq.backend.core.processes.tracing;
 
 
+import java.io.Serializable;
+
+
 /*******************************************************************************
  ** Basic class that can be passed in to ProcessTracerInterface.handleMessage.
  ** This class just provides for a string message.  We anticipate subclasses
  ** that may have more specific data, that specific tracer implementations may
  ** be aware of.  
  *******************************************************************************/
-public class ProcessTracerMessage
+public class ProcessTracerMessage implements Serializable
 {
    private String message;
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java

@@ -0,0 +1,343 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.scheduler;
+
+
+import java.text.ParseException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** class to give a human-friendly descriptive string from a cron expression.
+ ** (written in half by my friend Mr. Chatty G)
+ *******************************************************************************/
+public class CronDescriber
+{
+   private static final Map<String, String> DAY_OF_WEEK_MAP = new HashMap<>();
+   private static final Map<String, String> MONTH_MAP       = new HashMap<>();
+
+   static
+   {
+      DAY_OF_WEEK_MAP.put("1", "Sunday");
+      DAY_OF_WEEK_MAP.put("2", "Monday");
+      DAY_OF_WEEK_MAP.put("3", "Tuesday");
+      DAY_OF_WEEK_MAP.put("4", "Wednesday");
+      DAY_OF_WEEK_MAP.put("5", "Thursday");
+      DAY_OF_WEEK_MAP.put("6", "Friday");
+      DAY_OF_WEEK_MAP.put("7", "Saturday");
+
+      ////////////////////////////////
+      // Quartz also allows SUN-SAT //
+      ////////////////////////////////
+      DAY_OF_WEEK_MAP.put("SUN", "Sunday");
+      DAY_OF_WEEK_MAP.put("MON", "Monday");
+      DAY_OF_WEEK_MAP.put("TUE", "Tuesday");
+      DAY_OF_WEEK_MAP.put("WED", "Wednesday");
+      DAY_OF_WEEK_MAP.put("THU", "Thursday");
+      DAY_OF_WEEK_MAP.put("FRI", "Friday");
+      DAY_OF_WEEK_MAP.put("SAT", "Saturday");
+
+      MONTH_MAP.put("1", "January");
+      MONTH_MAP.put("2", "February");
+      MONTH_MAP.put("3", "March");
+      MONTH_MAP.put("4", "April");
+      MONTH_MAP.put("5", "May");
+      MONTH_MAP.put("6", "June");
+      MONTH_MAP.put("7", "July");
+      MONTH_MAP.put("8", "August");
+      MONTH_MAP.put("9", "September");
+      MONTH_MAP.put("10", "October");
+      MONTH_MAP.put("11", "November");
+      MONTH_MAP.put("12", "December");
+
+      ////////////////////////////////
+      // Quartz also allows JAN-DEC //
+      ////////////////////////////////
+      MONTH_MAP.put("JAN", "January");
+      MONTH_MAP.put("FEB", "February");
+      MONTH_MAP.put("MAR", "March");
+      MONTH_MAP.put("APR", "April");
+      MONTH_MAP.put("MAY", "May");
+      MONTH_MAP.put("JUN", "June");
+      MONTH_MAP.put("JUL", "July");
+      MONTH_MAP.put("AUG", "August");
+      MONTH_MAP.put("SEP", "September");
+      MONTH_MAP.put("OCT", "October");
+      MONTH_MAP.put("NOV", "November");
+      MONTH_MAP.put("DEC", "December");
+   }
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static String getDescription(String cronExpression) throws ParseException
+   {
+      String[] parts = cronExpression.trim().toUpperCase().split("\\s+");
+      if(parts.length < 6 || parts.length > 7)
+      {
+         throw new ParseException("Invalid cron expression: " + cronExpression, 0);
+      }
+
+      String seconds    = parts[0];
+      String minutes    = parts[1];
+      String hours      = parts[2];
+      String dayOfMonth = parts[3];
+      String month      = parts[4];
+      String dayOfWeek  = parts[5];
+      String year       = parts.length == 7 ? parts[6] : "*";
+
+      StringBuilder description = new StringBuilder();
+
+      description.append("At ");
+      description.append(describeTime(seconds, minutes, hours));
+      description.append(", on ");
+      description.append(describeDayOfMonth(dayOfMonth));
+      description.append(" of ");
+      description.append(describeMonth(month));
+      description.append(", ");
+      description.append(describeDayOfWeek(dayOfWeek));
+      if(!year.equals("*"))
+      {
+         description.append(", in ").append(year);
+      }
+      description.append(".");
+
+      return description.toString();
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String describeTime(String seconds, String minutes, String hours)
+   {
+      return String.format("%s, %s, %s", describePart(seconds, "second"), describePart(minutes, "minute"), describePart(hours, "hour"));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String describeDayOfMonth(String dayOfMonth)
+   {
+      if(dayOfMonth.equals("?"))
+      {
+         return "every day";
+      }
+      else if(dayOfMonth.equals("L"))
+      {
+         return "the last day";
+      }
+      else if(dayOfMonth.contains("W"))
+      {
+         return "the nearest weekday to day " + dayOfMonth.replace("W", "");
+      }
+      else
+      {
+         return (describePart(dayOfMonth, "day"));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String describeMonth(String month)
+   {
+      if(month.equals("*"))
+      {
+         return "every month";
+      }
+      else if(month.contains("-"))
+      {
+         String[] parts = month.split("-");
+         return String.format("%s to %s", MONTH_MAP.getOrDefault(parts[0], parts[0]), MONTH_MAP.getOrDefault(parts[1], parts[1]));
+      }
+      else
+      {
+         String[]     months     = month.split(",");
+         List<String> monthNames = Arrays.stream(months).map(m -> MONTH_MAP.getOrDefault(m, m)).toList();
+         return StringUtils.joinWithCommasAndAnd(monthNames);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String describeDayOfWeek(String dayOfWeek)
+   {
+      if(dayOfWeek.equals("?") || dayOfWeek.equals("*"))
+      {
+         return "every day of the week";
+      }
+      else if(dayOfWeek.equals("L"))
+      {
+         return "the last day of the week";
+      }
+      else if(dayOfWeek.contains("#"))
+      {
+         String[] parts = dayOfWeek.split("#");
+         return String.format("the %s %s of the month", ordinal(parts[1]), DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0]));
+      }
+      else if(dayOfWeek.contains("-"))
+      {
+         String[] parts = dayOfWeek.split("-");
+         return String.format("from %s to %s", DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0]), DAY_OF_WEEK_MAP.getOrDefault(parts[1], parts[1]));
+      }
+      else
+      {
+         String[]     days     = dayOfWeek.split(",");
+         List<String> dayNames = Arrays.stream(days).map(d -> DAY_OF_WEEK_MAP.getOrDefault(d, d)).toList();
+         return StringUtils.joinWithCommasAndAnd(dayNames);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String describePart(String part, String label)
+   {
+      if(part.equals("*"))
+      {
+         return "every " + label;
+      }
+      else if(part.contains("/"))
+      {
+         String[] parts = part.split("/");
+         if(parts[0].equals("*"))
+         {
+            parts[0] = "0";
+         }
+         return String.format("every %s " + label + "s starting at %s", parts[1], parts[0]);
+      }
+      else if(part.contains(","))
+      {
+         List<String> partsList = Arrays.stream(part.split(",")).toList();
+
+         if(label.equals("hour"))
+         {
+            List<String> hourNames = partsList.stream().map(p -> hourToAmPm(p)).toList();
+            return StringUtils.joinWithCommasAndAnd(hourNames);
+         }
+         else
+         {
+            if(label.equals("day"))
+            {
+               return "days " + StringUtils.joinWithCommasAndAnd(partsList);
+            }
+            else
+            {
+               return StringUtils.joinWithCommasAndAnd(partsList) + " " + label + "s";
+            }
+         }
+      }
+      else if(part.contains("-"))
+      {
+         String[] parts = part.split("-");
+         if(label.equals("day"))
+         {
+            return String.format("%ss from %s to %s", label, parts[0], parts[1]);
+         }
+         else if(label.equals("hour"))
+         {
+            return String.format("from %s to %s", hourToAmPm(parts[0]), hourToAmPm(parts[1]));
+         }
+         else
+         {
+            return String.format("from %s to %s %s", parts[0], parts[1], label + "s");
+         }
+      }
+      else
+      {
+         if(label.equals("day"))
+         {
+            return label + " " + part;
+         }
+         if(label.equals("hour"))
+         {
+            return hourToAmPm(part);
+         }
+         else
+         {
+            return part + " " + label + "s";
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String hourToAmPm(String part)
+   {
+      try
+      {
+         int hour = Integer.parseInt(part);
+         return switch(hour)
+         {
+            case 0 -> "midnight";
+            case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 -> hour + " AM";
+            case 12 -> "noon";
+            case 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 -> (hour - 12) + " PM";
+            default -> hour + " hours";
+         };
+      }
+      catch(Exception e)
+      {
+         return part + " hours";
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static String ordinal(String number)
+   {
+      int n = Integer.parseInt(number);
+      if(n >= 11 && n <= 13)
+      {
+         return n + "th";
+      }
+
+      return switch(n % 10)
+      {
+         case 1 -> n + "st";
+         case 2 -> n + "nd";
+         case 3 -> n + "rd";
+         default -> n + "th";
+      };
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java

@@ -0,0 +1,90 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.scheduler;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Field display behavior, to add a human-redable tooltip to cron-expressions.
+ *******************************************************************************/
+public class CronExpressionTooltipFieldBehavior implements FieldDisplayBehavior<CronExpressionTooltipFieldBehavior>
+{
+
+   /***************************************************************************
+    ** Add both this behavior, and the tooltip adornment to a field
+    ** Note, if either was already there, then that part is left alone.
+    ***************************************************************************/
+   public static void addToField(QFieldMetaData fieldMetaData)
+   {
+      CronExpressionTooltipFieldBehavior existingBehavior = fieldMetaData.getBehaviorOnlyIfSet(CronExpressionTooltipFieldBehavior.class);
+      if(existingBehavior == null)
+      {
+         fieldMetaData.withBehavior(new CronExpressionTooltipFieldBehavior());
+      }
+
+      if(fieldMetaData.getAdornment(AdornmentType.TOOLTIP).isEmpty())
+      {
+         fieldMetaData.withFieldAdornment((new FieldAdornment(AdornmentType.TOOLTIP)
+            .withValue(AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, true)));
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
+   {
+      for(QRecord record : recordList)
+      {
+         try
+         {
+            String cronExpression = record.getValueString(field.getName());
+            if(StringUtils.hasContent(cronExpression))
+            {
+               String description = CronDescriber.getDescription(cronExpression);
+               record.setDisplayValue(field.getName() + ":" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, description);
+            }
+         }
+         catch(Exception e)
+         {
+            /////////////////////
+            // just leave null //
+            /////////////////////
+         }
+      }
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/QScheduleManager.java

@@ -441,11 +441,16 @@ public class QScheduleManager
             try
             {
                HashMap<String, Serializable> parameters = new HashMap<>(paramMap);
-               HashMap<String, Serializable> variantMap = new HashMap<>(Map.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField())));
+
+               String variantTypeKey                 = backendMetaData.getBackendVariantsConfig().getVariantTypeKey();
+               String variantOptionsTableName        = backendMetaData.getBackendVariantsConfig().getOptionsTableName();
+               String variantOptionsTableIdFieldName = QContext.getQInstance().getTable(variantOptionsTableName).getPrimaryKeyField();
+
+               HashMap<String, Serializable> variantMap = new HashMap<>(Map.of(variantTypeKey, qRecord.getValue(variantOptionsTableIdFieldName)));
                parameters.put("backendVariantData", variantMap);
 
-               String identity    = schedulableIdentity.getIdentity() + ";" + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField());
-               String description = schedulableIdentity.getDescription() + " for variant: " + backendMetaData.getVariantOptionsTableTypeValue() + "=" + qRecord.getValue(backendMetaData.getVariantOptionsTableIdField());
+               String identity    = schedulableIdentity.getIdentity() + ";" + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName);
+               String description = schedulableIdentity.getDescription() + " for variant: " + variantTypeKey + "=" + qRecord.getValue(variantOptionsTableIdFieldName);
 
                BasicSchedulableIdentity variantIdentity = new BasicSchedulableIdentity(identity, description);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/SchedulerUtils.java

@@ -34,9 +34,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -102,7 +99,8 @@ public class SchedulerUtils
                try
                {
                   QBackendMetaData  backendMetaData  = qInstance.getBackend(process.getVariantBackend());
-                  Map<String, Serializable> thisVariantData = MapBuilder.of(backendMetaData.getVariantOptionsTableTypeValue(), qRecord.getValue(backendMetaData.getVariantOptionsTableIdField()));
+                  QTableMetaData variantTable = QContext.getQInstance().getTable(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
+                  Map<String, Serializable> thisVariantData = MapBuilder.of(backendMetaData.getBackendVariantsConfig().getVariantTypeKey(), qRecord.getValue(variantTable.getPrimaryKeyField()));
                   executeSingleProcess(process, thisVariantData, processInputValues);
                }
                catch(Exception e)
@@ -181,8 +179,8 @@ public class SchedulerUtils
          QBackendMetaData  backendMetaData  = QContext.getQInstance().getBackend(processMetaData.getVariantBackend());
 
          QueryInput queryInput = new QueryInput();
-         queryInput.setTableName(backendMetaData.getVariantOptionsTableName());
-         queryInput.setFilter(new QQueryFilter(new QFilterCriteria(backendMetaData.getVariantOptionsTableTypeField(), QCriteriaOperator.EQUALS, backendMetaData.getVariantOptionsTableTypeValue())));
+         queryInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName());
+         queryInput.setFilter(backendMetaData.getBackendVariantsConfig().getOptionsFilter());
 
          QueryOutput queryOutput = new QueryAction().execute(queryInput);
          records = queryOutput.getRecords();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java

@@ -247,6 +247,16 @@ public class Memoization<K, V>
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void clearKey(K key)
+   {
+      this.map.remove(key);
+   }
+
+
+
    /*******************************************************************************
     ** Setter for timeoutSeconds
     **

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java

@@ -0,0 +1,188 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.dashboard.widgets;
+
+
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for RecordListWidgetRenderer 
+ *******************************************************************************/
+class RecordListWidgetRendererTest extends BaseTest
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private QWidgetMetaData defineWidget()
+   {
+      return RecordListWidgetRenderer.widgetMetaDataBuilder("testRecordListWidget")
+         .withTableName(TestUtils.TABLE_NAME_SHAPE)
+         .withMaxRows(20)
+         .withLabel("Some Shapes")
+         .withFilter(new QQueryFilter()
+            .withCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, "${input.maxShapeId}")
+            .withCriteria("name", QCriteriaOperator.NOT_EQUALS, "Square")
+            .withOrderBy(new QFilterOrderBy("id", false))
+         ).getWidgetMetaData();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidation() throws QInstanceValidationException
+   {
+      {
+         QInstance       qInstance      = TestUtils.defineInstance();
+         QWidgetMetaData widgetMetaData = defineWidget();
+         widgetMetaData.getDefaultValues().remove("tableName");
+         qInstance.addWidget(widgetMetaData);
+
+         assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance))
+            .isInstanceOf(QInstanceValidationException.class)
+            .hasMessageContaining("defaultValue for tableName must be given");
+      }
+
+      {
+         QInstance       qInstance      = TestUtils.defineInstance();
+         QWidgetMetaData widgetMetaData = defineWidget();
+         widgetMetaData.getDefaultValues().remove("filter");
+         qInstance.addWidget(widgetMetaData);
+
+         assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance))
+            .isInstanceOf(QInstanceValidationException.class)
+            .hasMessageContaining("defaultValue for filter must be given");
+      }
+
+      {
+         QInstance       qInstance      = TestUtils.defineInstance();
+         QWidgetMetaData widgetMetaData = defineWidget();
+         widgetMetaData.getDefaultValues().remove("tableName");
+         widgetMetaData.getDefaultValues().remove("filter");
+         qInstance.addWidget(widgetMetaData);
+
+         assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance))
+            .isInstanceOf(QInstanceValidationException.class)
+            .hasMessageContaining("defaultValue for filter must be given")
+            .hasMessageContaining("defaultValue for tableName must be given");
+      }
+
+      {
+         QInstance       qInstance      = TestUtils.defineInstance();
+         QWidgetMetaData widgetMetaData = defineWidget();
+         QQueryFilter    filter         = (QQueryFilter) widgetMetaData.getDefaultValues().get("filter");
+         filter.addCriteria(new QFilterCriteria("noField", QCriteriaOperator.EQUALS, "noValue"));
+         qInstance.addWidget(widgetMetaData);
+
+         assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance))
+            .isInstanceOf(QInstanceValidationException.class)
+            .hasMessageContaining("Criteria fieldName noField is not a field in this table");
+      }
+
+      {
+         QInstance       qInstance      = TestUtils.defineInstance();
+         QWidgetMetaData widgetMetaData = defineWidget();
+         qInstance.addWidget(widgetMetaData);
+
+         //////////////////////////////////
+         // make sure valid setup passes //
+         //////////////////////////////////
+         new QInstanceValidator().validate(qInstance);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRender() throws QException
+   {
+      QInstance       qInstance      = TestUtils.defineInstance();
+      QWidgetMetaData widgetMetaData = defineWidget();
+      qInstance.addWidget(widgetMetaData);
+
+      TestUtils.insertDefaultShapes(qInstance);
+      TestUtils.insertExtraShapes(qInstance);
+
+      {
+         RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer();
+         RenderWidgetInput        input                    = new RenderWidgetInput();
+         input.setWidgetMetaData(widgetMetaData);
+         input.setQueryParams(Map.of("maxShapeId", "1"));
+         RenderWidgetOutput output = recordListWidgetRenderer.render(input);
+
+         ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData();
+         assertEquals(1, widgetData.getTotalRows());
+         assertEquals(1, widgetData.getQueryOutput().getRecords().get(0).getValue("id"));
+         assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name"));
+      }
+
+      {
+         RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer();
+         RenderWidgetInput        input                    = new RenderWidgetInput();
+         input.setWidgetMetaData(widgetMetaData);
+         input.setQueryParams(Map.of("maxShapeId", "4"));
+         RenderWidgetOutput output = recordListWidgetRenderer.render(input);
+
+         ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData();
+         assertEquals(3, widgetData.getTotalRows());
+
+         /////////////////////////////////////////////////////////////////////////
+         // id=2,name=Square was skipped due to NOT_EQUALS Square in the filter //
+         // max-shape-id applied we don't get id=5 or 6                         //
+         // and they're ordered as specified in the filter (id desc)            //
+         /////////////////////////////////////////////////////////////////////////
+         assertEquals(4, widgetData.getQueryOutput().getRecords().get(0).getValue("id"));
+         assertEquals("Rectangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name"));
+
+         assertEquals(3, widgetData.getQueryOutput().getRecords().get(1).getValue("id"));
+         assertEquals("Circle", widgetData.getQueryOutput().getRecords().get(1).getValue("name"));
+
+         assertEquals(1, widgetData.getQueryOutput().getRecords().get(2).getValue("id"));
+         assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(2).getValue("name"));
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java

@@ -23,10 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables;
 
 
 import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 
 
@@ -50,4 +54,18 @@ class CountActionTest extends BaseTest
       CountOutput result = new CountAction().execute(request);
       assertNotNull(result);
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testStaticWrapper() throws QException
+   {
+      TestUtils.insertDefaultShapes(QContext.getQInstance());
+      assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, null));
+      assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter()));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java

@@ -30,19 +30,23 @@ import java.time.LocalTime;
 import java.time.Month;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
@@ -237,4 +241,102 @@ class QValueFormatterTest extends BaseTest
       assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBlobValuesToDownloadUrls()
+   {
+      byte[] blobBytes = "hello".getBytes();
+      {
+         QTableMetaData table = new QTableMetaData()
+            .withName("testTable")
+            .withPrimaryKeyField("id")
+            .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+            .withField(new QFieldMetaData("blobField", QFieldType.BLOB)
+               .withFieldAdornment(new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD)
+                  .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "blob-%s.txt")
+                  .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS, new ArrayList<>(List.of("id")))));
+
+         //////////////////////////////////////////////////////////////////
+         // verify display value gets set to formated file-name + fields //
+         // and raw value becomes URL for downloading the byte           //
+         //////////////////////////////////////////////////////////////////
+         QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes);
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
+         assertEquals("/data/testTable/47/blobField/blob-47.txt", record.getValueString("blobField"));
+         assertEquals("blob-47.txt", record.getDisplayValue("blobField"));
+
+         ////////////////////////////////////////////////////////
+         // verify that w/ no blob value, we don't do anything //
+         ////////////////////////////////////////////////////////
+         QRecord recordWithoutBlobValue = new QRecord().withValue("id", 47);
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(recordWithoutBlobValue));
+         assertNull(recordWithoutBlobValue.getValue("blobField"));
+         assertNull(recordWithoutBlobValue.getDisplayValue("blobField"));
+      }
+
+      {
+         FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD)
+            .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName");
+
+         QTableMetaData table = new QTableMetaData()
+            .withName("testTable")
+            .withPrimaryKeyField("id")
+            .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+            .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+            .withField(new QFieldMetaData("blobField", QFieldType.BLOB)
+               .withFieldAdornment(adornment));
+
+         ////////////////////////////////////////////////////
+         // here get the file name directly from one field //
+         ////////////////////////////////////////////////////
+         QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt");
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
+         assertEquals("/data/testTable/47/blobField/myBlob.txt", record.getValueString("blobField"));
+         assertEquals("myBlob.txt", record.getDisplayValue("blobField"));
+
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // switch to use dynamic url, rerun, and assert we get the values as they were on the record before the call //
+         ///////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         adornment.withValue(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, true);
+         record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt")
+            .withDisplayValue("blobField:" + AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, "/something-custom/")
+            .withDisplayValue("blobField", "myDisplayValue");
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
+         assertArrayEquals(blobBytes, record.getValueByteArray("blobField"));
+         assertEquals("myDisplayValue", record.getDisplayValue("blobField"));
+      }
+
+      {
+         FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD);
+
+         QTableMetaData table = new QTableMetaData()
+            .withName("testTable")
+            .withLabel("Test Table")
+            .withPrimaryKeyField("id")
+            .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+            .withField(new QFieldMetaData("blobField", QFieldType.BLOB).withLabel("Blob").withFieldAdornment(adornment));
+
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // w/o file name format or whatever, generate a file name from table & id & field labels //
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes);
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
+         assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob", record.getValueString("blobField"));
+         assertEquals("Test Table 47 Blob", record.getDisplayValue("blobField"));
+
+         ////////////////////////////////////////
+         // add a default extension and re-run //
+         ////////////////////////////////////////
+         adornment.withValue(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION, "html");
+         record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes);
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
+         assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob.html", record.getValueString("blobField"));
+         assertEquals("Test Table 47 Blob.html", record.getDisplayValue("blobField"));
+      }
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java

@@ -27,7 +27,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.BaseTest;
-import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.enrichment.testplugins.TestEnricherPlugin;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
@@ -595,27 +596,31 @@ class QInstanceEnricherTest extends BaseTest
    {
       QInstance qInstance = TestUtils.defineInstance();
 
-      QInstanceEnricher.addEnricherPlugin(new QInstanceEnricherPluginInterface<QFieldMetaData>()
-      {
-         /***************************************************************************
-          *
-          ***************************************************************************/
-         @Override
-         public void enrich(QFieldMetaData field, QInstance qInstance)
-         {
-            if(field != null)
-            {
-               field.setLabel(field.getLabel() + " Plugged");
-            }
-         }
-      });
+      QInstanceEnricher.addEnricherPlugin(new TestEnricherPlugin());
 
       new QInstanceEnricher(qInstance).enrich();
 
       qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
       qInstance.getProcesses().values().forEach(process -> process.getInputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
       qInstance.getProcesses().values().forEach(process -> process.getOutputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
+   }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDiscoverAndAddPlugins() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      new QInstanceEnricher(qInstance).enrich();
+      qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).doesNotEndWith("Plugged")));
+
+      qInstance = TestUtils.defineInstance();
+      QInstanceEnricher.discoverAndAddPluginsInPackage(getClass().getPackageName() + ".enrichment.testplugins");
+      new QInstanceEnricher(qInstance).enrich();
+      qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged")));
    }
 
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
@@ -55,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
@@ -93,12 +95,15 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.TableAutomationAction;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig;
 import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -182,6 +187,143 @@ public class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBackendVariants()
+   {
+      BackendVariantSetting setting = new BackendVariantSetting() {};
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(true)),
+         "Missing backendVariantsConfig in backend [variant] which is marked as usesVariants");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(false)
+            .withBackendVariantsConfig(new BackendVariantsConfig())),
+         "Should not have a backendVariantsConfig");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(null)
+            .withBackendVariantsConfig(new BackendVariantsConfig())),
+         "Should not have a backendVariantsConfig");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(true)
+            .withBackendVariantsConfig(new BackendVariantsConfig())),
+         "Missing variantTypeKey in backendVariantsConfig",
+         "Missing optionsTableName in backendVariantsConfig",
+         "Missing or empty backendSettingSourceFieldNameMap");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(true)
+            .withBackendVariantsConfig(new BackendVariantsConfig()
+               .withVariantTypeKey("myVariant")
+               .withOptionsTableName("notATable")
+               .withBackendSettingSourceFieldNameMap(Map.of(setting, "field")))),
+         "Unrecognized optionsTableName [notATable] in backendVariantsConfig");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(true)
+            .withBackendVariantsConfig(new BackendVariantsConfig()
+               .withVariantTypeKey("myVariant")
+               .withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
+               .withOptionsFilter(new QQueryFilter(new QFilterCriteria("notAField", QCriteriaOperator.EQUALS, 1)))
+               .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")))),
+         "optionsFilter in backendVariantsConfig in backend [variant]: Criteria fieldName notAField is not a field");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(true)
+            .withBackendVariantsConfig(new BackendVariantsConfig()
+               .withVariantTypeKey("myVariant")
+               .withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
+               .withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))),
+         "Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap");
+
+      assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+            .withName("variant")
+            .withUsesVariants(true)
+            .withBackendVariantsConfig(new BackendVariantsConfig()
+               .withVariantTypeKey("myVariant")
+               .withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
+               .withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class))
+               .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
+            )),
+         "VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function");
+
+      assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+         .withName("variant")
+         .withUsesVariants(true)
+         .withBackendVariantsConfig(new BackendVariantsConfig()
+            .withVariantTypeKey("myVariant")
+            .withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
+            .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName"))
+         )));
+
+      assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+         .withName("variant")
+         .withUsesVariants(true)
+         .withBackendVariantsConfig(new BackendVariantsConfig()
+            .withVariantTypeKey("myVariant")
+            .withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
+            .withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class))
+            .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
+         )));
+
+      assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData()
+         .withName("variant")
+         .withUsesVariants(true)
+         .withBackendVariantsConfig(new BackendVariantsConfig()
+            .withVariantTypeKey("myVariant")
+            .withOptionsTableName(TestUtils.TABLE_NAME_PERSON)
+            .withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class))
+            .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier"))
+         )));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class VariantRecordFunction implements Function<Serializable, QRecord>
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public QRecord apply(Serializable serializable)
+      {
+         return null;
+      }
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   public static class VariantRecordUnsafeFunction implements UnsafeFunction<Serializable, QRecord, QException>
+   {
+      /***************************************************************************
+       **
+       ***************************************************************************/
+      @Override
+      public QRecord apply(Serializable serializable) throws QException
+      {
+         return null;
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Test an instance with null tables - should throw.
     **

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java

@@ -0,0 +1,48 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.instances.enrichment.testplugins;
+
+
+import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestEnricherPlugin implements QInstanceEnricherPluginInterface<QFieldMetaData>
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void enrich(QFieldMetaData field, QInstance qInstance)
+   {
+      if(field != null)
+      {
+         field.setLabel(field.getLabel() + " Plugged");
+      }
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java

@@ -66,7 +66,7 @@ class NowWithOffsetTest extends BaseTest
          assertThat(twoWeeksFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), allowedDiff);
 
          long oneMonthAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli();
-         assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusOneDay);
+         assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); // two days, to work on 3/1...
 
          long twoMonthsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli();
          assertThat(twoMonthsFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), allowedDiffPlusTwoDays);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/EmptyMetaDataProducerOutputTest.java

@@ -0,0 +1,44 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata;
+
+
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for EmptyMetaDataProducerOutput 
+ *******************************************************************************/
+class EmptyMetaDataProducerOutputTest
+{
+
+   /*******************************************************************************
+    ** sorry, just here to avoid a dip in coverage.
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      QInstance qInstance = new QInstance();
+      new EmptyMetaDataProducerOutput().addSelfToInstance(qInstance);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java

@@ -0,0 +1,151 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits;
+
+
+import java.util.LinkedHashMap;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.OtherTableMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.SomeTableMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for QBitProducer 
+ *******************************************************************************/
+class QBitProducerTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      TestQBitConfig config = new TestQBitConfig()
+         .withOtherTableConfig(ProvidedOrSuppliedTableConfig.provideTableUsingBackendNamed(TestUtils.MEMORY_BACKEND_NAME))
+         .withIsSomeTableEnabled(true)
+         .withSomeSetting("yes")
+         .withTableMetaDataCustomizer((i, table) ->
+         {
+            if(table.getBackendName() == null)
+            {
+               table.setBackendName(TestUtils.DEFAULT_BACKEND_NAME);
+            }
+
+            table.addField(new QFieldMetaData("custom", QFieldType.STRING));
+
+            return (table);
+         });
+
+      QInstance qInstance = QContext.getQInstance();
+      new TestQBitProducer().withTestQBitConfig(config).produce(qInstance);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      // OtherTable should have been provided by the qbit, with the backend name we told it above (MEMORY) //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME);
+      assertNotNull(otherTable);
+      assertEquals(TestUtils.MEMORY_BACKEND_NAME, otherTable.getBackendName());
+      assertNotNull(otherTable.getField("custom"));
+
+      QBitMetaData sourceQBit = otherTable.getSourceQBit();
+      assertEquals("testQBit", sourceQBit.getArtifactId());
+
+      ////////////////////////////////////////////////////////////////////////////////
+      // SomeTable should have been provided, w/ backend name set by the customizer //
+      ////////////////////////////////////////////////////////////////////////////////
+      QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME);
+      assertNotNull(someTable);
+      assertEquals(TestUtils.DEFAULT_BACKEND_NAME, someTable.getBackendName());
+      assertNotNull(otherTable.getField("custom"));
+
+      TestQBitConfig qBitConfig = (TestQBitConfig) someTable.getSourceQBitConfig();
+      assertEquals("yes", qBitConfig.getSomeSetting());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDisableThings() throws QException
+   {
+      TestQBitConfig config = new TestQBitConfig()
+         .withOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY))
+         .withIsSomeTableEnabled(false);
+
+      QInstance qInstance = QContext.getQInstance();
+      new TestQBitProducer().withTestQBitConfig(config).produce(qInstance);
+
+      //////////////////////////////////////
+      // neither table should be produced //
+      //////////////////////////////////////
+      QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME);
+      assertNull(otherTable);
+
+      QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME);
+      assertNull(someTable);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testValidationErrors() throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      TestQBitConfig config = new TestQBitConfig();
+
+      assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance))
+         .isInstanceOf(QBitConfigValidationException.class)
+         .hasMessageContaining("otherTableConfig must be set")
+         .hasMessageContaining("isSomeTableEnabled must be set");
+      qInstance.setQBits(new LinkedHashMap<>());
+
+      config.setIsSomeTableEnabled(true);
+      assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance))
+         .isInstanceOf(QBitConfigValidationException.class)
+         .hasMessageContaining("otherTableConfig must be set");
+      qInstance.setQBits(new LinkedHashMap<>());
+
+      config.setOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY));
+      new TestQBitProducer().withTestQBitConfig(config).produce(qInstance);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java

@@ -0,0 +1,181 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.ProvidedOrSuppliedTableConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestQBitConfig implements QBitConfig
+{
+   private MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer;
+
+   private Boolean                       isSomeTableEnabled;
+   private ProvidedOrSuppliedTableConfig otherTableConfig;
+
+   private String someSetting;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void validate(QInstance qInstance, List<String> errors)
+   {
+      assertCondition(otherTableConfig != null, "otherTableConfig must be set", errors);
+      assertCondition(isSomeTableEnabled != null, "isSomeTableEnabled must be set", errors);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for otherTableConfig
+    *******************************************************************************/
+   public ProvidedOrSuppliedTableConfig getOtherTableConfig()
+   {
+      return (this.otherTableConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for otherTableConfig
+    *******************************************************************************/
+   public void setOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig)
+   {
+      this.otherTableConfig = otherTableConfig;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for otherTableConfig
+    *******************************************************************************/
+   public TestQBitConfig withOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig)
+   {
+      this.otherTableConfig = otherTableConfig;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for isSomeTableEnabled
+    *******************************************************************************/
+   public Boolean getIsSomeTableEnabled()
+   {
+      return (this.isSomeTableEnabled);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isSomeTableEnabled
+    *******************************************************************************/
+   public void setIsSomeTableEnabled(Boolean isSomeTableEnabled)
+   {
+      this.isSomeTableEnabled = isSomeTableEnabled;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isSomeTableEnabled
+    *******************************************************************************/
+   public TestQBitConfig withIsSomeTableEnabled(Boolean isSomeTableEnabled)
+   {
+      this.isSomeTableEnabled = isSomeTableEnabled;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for tableMetaDataCustomizer
+    *******************************************************************************/
+   public MetaDataCustomizerInterface<QTableMetaData> getTableMetaDataCustomizer()
+   {
+      return (this.tableMetaDataCustomizer);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for tableMetaDataCustomizer
+    *******************************************************************************/
+   public void setTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
+   {
+      this.tableMetaDataCustomizer = tableMetaDataCustomizer;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for tableMetaDataCustomizer
+    *******************************************************************************/
+   public TestQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface<QTableMetaData> tableMetaDataCustomizer)
+   {
+      this.tableMetaDataCustomizer = tableMetaDataCustomizer;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for someSetting
+    *******************************************************************************/
+   public String getSomeSetting()
+   {
+      return (this.someSetting);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for someSetting
+    *******************************************************************************/
+   public void setSomeSetting(String someSetting)
+   {
+      this.someSetting = someSetting;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for someSetting
+    *******************************************************************************/
+   public TestQBitConfig withSomeSetting(String someSetting)
+   {
+      this.someSetting = someSetting;
+      return (this);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java

@@ -0,0 +1,92 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class TestQBitProducer implements QBitProducer
+{
+   private TestQBitConfig testQBitConfig;
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void produce(QInstance qInstance, String namespace) throws QException
+   {
+      QBitMetaData qBitMetaData = new QBitMetaData()
+         .withGroupId("test.com.kingsrook.qbits")
+         .withArtifactId("testQBit")
+         .withVersion("0.1.0")
+         .withNamespace(namespace)
+         .withConfig(testQBitConfig);
+      qInstance.addQBit(qBitMetaData);
+
+      List<MetaDataProducerInterface<?>> producers = MetaDataProducerHelper.findProducers(getClass().getPackageName() + ".metadata");
+      finishProducing(qInstance, qBitMetaData, testQBitConfig, producers);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for testQBitConfig
+    *******************************************************************************/
+   public TestQBitConfig getTestQBitConfig()
+   {
+      return (this.testQBitConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for testQBitConfig
+    *******************************************************************************/
+   public void setTestQBitConfig(TestQBitConfig testQBitConfig)
+   {
+      this.testQBitConfig = testQBitConfig;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for testQBitConfig
+    *******************************************************************************/
+   public TestQBitProducer withTestQBitConfig(TestQBitConfig testQBitConfig)
+   {
+      this.testQBitConfig = testQBitConfig;
+      return (this);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Meta Data Producer for OtherTable
+ *******************************************************************************/
+public class OtherTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, TestQBitConfig>
+{
+   public static final String NAME = "otherTable";
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig().getOtherTableConfig().getDoProvideTable());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName(NAME)
+         .withPrimaryKeyField("id")
+         .withBackendName(getQBitConfig().getOtherTableConfig().getBackendName())
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER));
+
+      return (qTableMetaData);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+
+
+/*******************************************************************************
+ ** Meta Data Producer for SomeTable
+ *******************************************************************************/
+public class SomeTableMetaDataProducer extends QBitComponentMetaDataProducer<QTableMetaData, TestQBitConfig>
+{
+   public static final String NAME = "someTable";
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public boolean isEnabled()
+   {
+      return (getQBitConfig().getIsSomeTableEnabled());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QTableMetaData produce(QInstance qInstance) throws QException
+   {
+      QTableMetaData qTableMetaData = new QTableMetaData()
+         .withName(NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER));
+
+      return (qTableMetaData);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/SectionFactoryTest.java

@@ -0,0 +1,62 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.tables;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for SectionFactory 
+ *******************************************************************************/
+class SectionFactoryTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test()
+   {
+      QFieldSection t1section = SectionFactory.defaultT1("id", "name");
+      assertEquals(SectionFactory.getDefaultT1name(), t1section.getName());
+      assertEquals(SectionFactory.getDefaultT1iconName(), t1section.getIcon().getName());
+      assertEquals(Tier.T1, t1section.getTier());
+      assertEquals(List.of("id", "name"), t1section.getFieldNames());
+
+      QFieldSection t2section = SectionFactory.defaultT2("size", "age");
+      assertEquals(SectionFactory.getDefaultT2name(), t2section.getName());
+      assertEquals(SectionFactory.getDefaultT2iconName(), t2section.getIcon().getName());
+      assertEquals(Tier.T2, t2section.getTier());
+      assertEquals(List.of("size", "age"), t2section.getFieldNames());
+
+      QFieldSection t3section = SectionFactory.defaultT3("createDate", "modifyDate");
+      assertEquals(SectionFactory.getDefaultT3name(), t3section.getName());
+      assertEquals(SectionFactory.getDefaultT3iconName(), t3section.getIcon().getName());
+      assertEquals(Tier.T3, t3section.getTier());
+      assertEquals(List.of("createDate", "modifyDate"), t3section.getFieldNames());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java

@@ -0,0 +1,157 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.tables;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for TablesCustomPossibleValueProvider 
+ *******************************************************************************/
+class TablesCustomPossibleValueProviderTest extends BaseTest
+{
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   void beforeEach()
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      qInstance.addTable(new QTableMetaData()
+         .withName("hidden")
+         .withIsHidden(true)
+         .withBackendName(TestUtils.MEMORY_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER)));
+
+      qInstance.addTable(new QTableMetaData()
+         .withName("restricted")
+         .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION))
+         .withBackendName(TestUtils.MEMORY_BACKEND_NAME)
+         .withPrimaryKeyField("id")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER)));
+
+      qInstance.addPossibleValueSource(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(qInstance));
+
+      QContext.init(qInstance, newSession());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetPossibleValue()
+   {
+      TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider();
+
+      QPossibleValue<String> possibleValue = provider.getPossibleValue(TestUtils.TABLE_NAME_PERSON);
+      assertEquals(TestUtils.TABLE_NAME_PERSON, possibleValue.getId());
+      assertEquals("Person", possibleValue.getLabel());
+
+      assertNull(provider.getPossibleValue("no-such-table"));
+      assertNull(provider.getPossibleValue("hidden"));
+      assertNull(provider.getPossibleValue("restricted"));
+
+      QContext.getQSession().withPermission("restricted.hasAccess");
+      assertNotNull(provider.getPossibleValue("restricted"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testSearchPossibleValue() throws QException
+   {
+      TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider();
+
+      List<QPossibleValue<String>> list = provider.search(new SearchPossibleValueSourceInput()
+         .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME));
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON));
+      assertThat(list).noneMatch(p -> p.getId().equals("no-such-table"));
+      assertThat(list).noneMatch(p -> p.getId().equals("hidden"));
+      assertThat(list).noneMatch(p -> p.getId().equals("restricted"));
+      assertNull(provider.getPossibleValue("restricted"));
+
+      list = provider.search(new SearchPossibleValueSourceInput()
+         .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)
+         .withIdList(List.of(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_SHAPE, "hidden")));
+      assertEquals(2, list.size());
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON));
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE));
+      assertThat(list).noneMatch(p -> p.getId().equals("hidden"));
+
+      list = provider.search(new SearchPossibleValueSourceInput()
+         .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)
+         .withLabelList(List.of("Person", "Shape", "Restricted")));
+      assertEquals(2, list.size());
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON));
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE));
+      assertThat(list).noneMatch(p -> p.getId().equals("restricted"));
+
+      list = provider.search(new SearchPossibleValueSourceInput()
+         .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)
+         .withSearchTerm("restricted"));
+      assertEquals(0, list.size());
+
+      /////////////////////////////////////////
+      // add permission for restricted table //
+      /////////////////////////////////////////
+      QContext.getQSession().withPermission("restricted.hasAccess");
+      list = provider.search(new SearchPossibleValueSourceInput()
+         .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)
+         .withSearchTerm("restricted"));
+      assertEquals(1, list.size());
+
+      list = provider.search(new SearchPossibleValueSourceInput()
+         .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)
+         .withLabelList(List.of("Person", "Shape", "Restricted")));
+      assertEquals(3, list.size());
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON));
+      assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE));
+      assertThat(list).anyMatch(p -> p.getId().equals("restricted"));
+
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java

@@ -0,0 +1,100 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata.variants;
+
+
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+
+/*******************************************************************************
+ ** Unit test for BackendVariantsUtil 
+ *******************************************************************************/
+class BackendVariantsUtilTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetVariantId() throws QException
+   {
+      QBackendMetaData myBackend = getBackendMetaData();
+
+      assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend))
+         .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'");
+
+      QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701));
+      assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static QBackendMetaData getBackendMetaData()
+   {
+      QBackendMetaData myBackend = new QBackendMetaData()
+         .withName("TestBackend")
+         .withUsesVariants(true)
+         .withBackendVariantsConfig(new BackendVariantsConfig()
+            .withOptionsTableName(TestUtils.TABLE_NAME_SHAPE)
+            .withVariantTypeKey("yourSelectedShape"));
+      return myBackend;
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetVariantRecord() throws QException
+   {
+      QBackendMetaData myBackend = getBackendMetaData();
+
+      TestUtils.insertDefaultShapes(QContext.getQInstance());
+
+      assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend))
+         .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'");
+
+      QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701));
+      assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend))
+         .hasMessageContaining("Could not find Backend Variant in table shape with id '1701'");
+
+      QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1));
+      QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend);
+      assertEquals(1, variantRecord.getValueInteger("id"));
+      assertNotNull(variantRecord.getValue("name"));
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java

@@ -22,24 +22,21 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
 
 
-import java.io.IOException;
 import java.io.OutputStream;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
 import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
 import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
@@ -61,6 +58,9 @@ import static org.junit.jupiter.api.Assertions.assertNull;
  *******************************************************************************/
 class BulkInsertFullProcessTest extends BaseTest
 {
+   private static final String defaultEmail = "noone@kingsrook.com";
+
+
 
    /*******************************************************************************
     **
@@ -116,48 +116,20 @@ class BulkInsertFullProcessTest extends BaseTest
    @Test
    void test() throws Exception
    {
-      String defaultEmail = "noone@kingsrook.com";
-
-      ///////////////////////////////////////
-      // make sure table is empty to start //
-      ///////////////////////////////////////
       assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
 
-      QInstance qInstance   = QContext.getQInstance();
-      String    processName = "PersonBulkInsertV2";
-      new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
-
       /////////////////////////////////////////////////////////
       // start the process - expect to go to the upload step //
       /////////////////////////////////////////////////////////
       RunProcessInput  runProcessInput  = new RunProcessInput();
-      runProcessInput.setProcessName(processName);
-      runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
-      RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      RunProcessOutput runProcessOutput = startProcess(runProcessInput);
       String           processUUID      = runProcessOutput.getProcessUUID();
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload");
 
-      //////////////////////////////
-      // simulate the file upload //
-      //////////////////////////////
-      String       storageReference = UUID.randomUUID() + ".csv";
-      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
-      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
-      {
-         outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
-      }
-      catch(IOException e)
-      {
-         throw (e);
-      }
-
       //////////////////////////
       // continue post-upload //
       //////////////////////////
-      runProcessInput.setProcessUUID(processUUID);
-      runProcessInput.setStartAfterStep("upload");
-      runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput)));
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      runProcessOutput = continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(2));
       assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues"));
       assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters"));
 
@@ -176,29 +148,10 @@ class BulkInsertFullProcessTest extends BaseTest
 
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping");
 
-      ////////////////////////////////////////////////////////////////////////////////
-      // all subsequent steps will want these data - so set up a lambda to set them //
-      ////////////////////////////////////////////////////////////////////////////////
-      Consumer<RunProcessInput> addProfileToRunProcessInput = (RunProcessInput input) ->
-      {
-         input.addValue("version", "v1");
-         input.addValue("layout", "FLAT");
-         input.addValue("hasHeaderRow", "true");
-         input.addValue("fieldListJSON", JsonUtils.toJson(List.of(
-            new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3),
-            new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4),
-            new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail),
-            new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)),
-            new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8)
-         )));
-      };
-
       ////////////////////////////////
       // continue post file-mapping //
       ////////////////////////////////
-      runProcessInput.setStartAfterStep("fileMapping");
-      addProfileToRunProcessInput.accept(runProcessInput);
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      runProcessOutput = continueProcessPostFileMapping(runProcessInput);
       Serializable valueMappingField = runProcessOutput.getValue("valueMappingField");
       assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class);
       assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName());
@@ -211,23 +164,20 @@ class BulkInsertFullProcessTest extends BaseTest
       /////////////////////////////////
       // continue post value-mapping //
       /////////////////////////////////
-      runProcessInput.setStartAfterStep("valueMapping");
-      runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2)));
-      addProfileToRunProcessInput.accept(runProcessInput);
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      runProcessOutput = continueProcessPostValueMapping(runProcessInput);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
 
       /////////////////////////////////
       // continue post review screen //
       /////////////////////////////////
-      runProcessInput.setStartAfterStep("review");
-      addProfileToRunProcessInput.accept(runProcessInput);
-      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
       assertThat(runProcessOutput.getRecords()).hasSize(2);
       assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
       assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
       assertThat(runProcessOutput.getException()).isEmpty();
 
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2");
+
       ////////////////////////////////////
       // query for the inserted records //
       ////////////////////////////////////
@@ -249,4 +199,136 @@ class BulkInsertFullProcessTest extends BaseTest
       assertNull(records.get(1).getValue("noOfShoes"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testOneRow() throws Exception
+   {
+      ///////////////////////////////////////
+      // make sure table is empty to start //
+      ///////////////////////////////////////
+      assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
+
+      RunProcessInput  runProcessInput  = new RunProcessInput();
+      RunProcessOutput runProcessOutput = startProcess(runProcessInput);
+      String           processUUID      = runProcessOutput.getProcessUUID();
+
+      continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(1));
+      continueProcessPostFileMapping(runProcessInput);
+      continueProcessPostValueMapping(runProcessInput);
+      runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
+
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // all that just so we can make sure this message is right (because it was wrong when we first wrote it, lol) //
+      ////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id 1");
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static RunProcessOutput continueProcessPostReviewScreen(RunProcessInput runProcessInput) throws QException
+   {
+      RunProcessOutput runProcessOutput;
+      runProcessInput.setStartAfterStep("review");
+      addProfileToRunProcessInput(runProcessInput);
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      return runProcessOutput;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static RunProcessOutput continueProcessPostValueMapping(RunProcessInput runProcessInput) throws QException
+   {
+      runProcessInput.setStartAfterStep("valueMapping");
+      runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2)));
+      addProfileToRunProcessInput(runProcessInput);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      return (runProcessOutput);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException
+   {
+      RunProcessOutput runProcessOutput;
+      runProcessInput.setStartAfterStep("fileMapping");
+      addProfileToRunProcessInput(runProcessInput);
+      runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      return runProcessOutput;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static RunProcessOutput continueProcessPostUpload(RunProcessInput runProcessInput, String processUUID, StorageInput storageInput) throws QException
+   {
+      runProcessInput.setProcessUUID(processUUID);
+      runProcessInput.setStartAfterStep("upload");
+      runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput)));
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      return (runProcessOutput);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static StorageInput simulateFileUpload(int noOfRows) throws Exception
+   {
+      String       storageReference = UUID.randomUUID() + ".csv";
+      StorageInput storageInput     = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
+      try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
+      {
+         outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + (noOfRows == 2 ? getPersonCsvRow2() : "")).getBytes());
+      }
+      return storageInput;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static RunProcessOutput startProcess(RunProcessInput runProcessInput) throws QException
+   {
+      runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert");
+      runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
+      RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
+      return runProcessOutput;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static void addProfileToRunProcessInput(RunProcessInput input)
+   {
+      input.addValue("version", "v1");
+      input.addValue("layout", "FLAT");
+      input.addValue("hasHeaderRow", "true");
+      input.addValue("fieldListJSON", JsonUtils.toJson(List.of(
+         new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3),
+         new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4),
+         new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail),
+         new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)),
+         new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8)
+      )));
+   }
+
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java

@@ -0,0 +1,75 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.scheduler;
+
+
+import java.text.ParseException;
+import com.kingsrook.qqq.backend.core.BaseTest;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for CronDescriber 
+ *******************************************************************************/
+class CronDescriberTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws ParseException
+   {
+      assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * * * ?"));
+      assertEquals("At 0 seconds, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 * * * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 * * * ?"));
+      assertEquals("At 0 seconds, 0 and 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 0 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, 1 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 1 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, 11 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 11 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, noon, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 12 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, 1 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 13 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, 11 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 23 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on day 10 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10 * ?"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on days 10 and 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on days from 10 to 15 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10-15 * ?"));
+      assertEquals("At from 10 to 15 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("10-15 0 0 * * ?"));
+      assertEquals("At 30 seconds, 30 minutes, from 8 AM to 4 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("30 30 8-16 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on every 3 days starting at 0 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 */3 * ?"));
+      assertEquals("At every 5 seconds starting at 0, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0/5 0 0 * * ?"));
+      assertEquals("At 0 seconds, every 30 minutes starting at 3, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 3/30 0 * * ?"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, and Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI"));
+      assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday and Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7"));
+      assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, and 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?"));
+      assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, and 52 minutes, every hour, on every day of January, March, and September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010"));
+
+      assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * ? * *"));
+      assertEquals("At every second, every minute, every hour, on every day of January to June, every day of the week.", CronDescriber.getDescription("* * * ? 1-6 *"));
+      assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * * 1,3,5 * *"));
+      // todo fix has 2-4 hours and 3 PM, s/b 2 AM to 4 AM and 3 PM assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * 2-4,15 1,3,5 * *"));
+      // hour failing on 3,2-7 (at least in TS side?)
+      // 3,2-7 makes 3,2 to July
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.scheduler;
+
+
+import com.kingsrook.qqq.backend.core.BaseTest;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Unit test for CronExpressionTooltipFieldBehavior 
+ *******************************************************************************/
+class CronExpressionTooltipFieldBehaviorTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      QFieldMetaData field = new QFieldMetaData("cronExpression", QFieldType.STRING);
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE)
+         .addField(field);
+
+      CronExpressionTooltipFieldBehavior.addToField(field);
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SHAPE).withRecord(
+         new QRecord().withValue("name", "Square").withValue("cronExpression", "* * * * * ?")));
+
+      QRecord record = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_SHAPE).withPrimaryKey(1).withShouldGenerateDisplayValues(true));
+      assertThat(record.getDisplayValue("cronExpression:" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC))
+         .contains("every second");
+   }
+
+}

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/AbstractAPIAction.java

@@ -61,7 +61,6 @@ public abstract class AbstractAPIAction
 
       apiActionUtil.setBackendMetaData(this.backendMetaData);
       apiActionUtil.setActionInput(actionInput);
-      apiActionUtil.setSession(session);
    }
 
 }

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -36,7 +36,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
-import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -61,6 +60,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
 import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -77,6 +79,7 @@ import com.kingsrook.qqq.backend.module.api.exceptions.RetryableServerErrorExcep
 import com.kingsrook.qqq.backend.module.api.model.AuthorizationType;
 import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
 import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData;
+import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendVariantSetting;
 import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.BooleanUtils;
@@ -114,7 +117,6 @@ public class BaseAPIActionUtil
 {
    private final QLogger LOG = QLogger.getLogger(BaseAPIActionUtil.class);
 
-   protected QSession                 session; // todo not commit - delete!!
    protected APIBackendMetaData       backendMetaData;
    protected AbstractTableActionInput actionInput;
 
@@ -777,8 +779,8 @@ public class BaseAPIActionUtil
    {
       if(backendMetaData.getUsesVariants())
       {
-         QRecord record = getVariantRecord();
-         return (record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField()));
+         QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData);
+         return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY)));
       }
 
       return (backendMetaData.getApiKey());
@@ -786,6 +788,18 @@ public class BaseAPIActionUtil
 
 
 
+   /***************************************************************************
+    ** todo - once deprecated variant methods are removed from QBackendMetaData,
+    ** then we can remove the LegacyBackendVariantSetting enum, and this param.
+    ***************************************************************************/
+   private String getVariantSettingSourceFieldName(APIBackendMetaData backendMetaData, LegacyBackendVariantSetting legacyBackendVariantSetting, APIBackendVariantSetting apiBackendVariantSetting)
+   {
+      Map<BackendVariantSetting, String> map = CollectionUtils.nonNullMap(backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap());
+      return map.getOrDefault(legacyBackendVariantSetting, map.get(apiBackendVariantSetting));
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -793,8 +807,11 @@ public class BaseAPIActionUtil
    {
       if(backendMetaData.getUsesVariants())
       {
-         QRecord record = getVariantRecord();
-         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField())));
+         QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData);
+         return (Pair.of(
+            record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)),
+            record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD))
+         ));
       }
 
       return (Pair.of(backendMetaData.getUsername(), backendMetaData.getPassword()));
@@ -802,46 +819,6 @@ public class BaseAPIActionUtil
 
 
 
-   /*******************************************************************************
-    ** For backends that use variants, look up the variant record (in theory, based
-    ** on an id in the session's backend variants map, then fetched from the backend's
-    ** variant options table.
-    *******************************************************************************/
-   protected QRecord getVariantRecord() throws QException
-   {
-      Serializable variantId = getVariantId();
-      GetInput     getInput  = new GetInput();
-      getInput.setShouldMaskPasswords(false);
-      getInput.setTableName(backendMetaData.getVariantOptionsTableName());
-      getInput.setPrimaryKey(variantId);
-      GetOutput getOutput = new GetAction().execute(getInput);
-
-      QRecord record = getOutput.getRecord();
-      if(record == null)
-      {
-         throw (new QException("Could not find Backend Variant in table " + backendMetaData.getVariantOptionsTableName() + " with id '" + variantId + "'"));
-      }
-      return record;
-   }
-
-
-
-   /*******************************************************************************
-    ** Get the variant id from the session for the backend.
-    *******************************************************************************/
-   protected Serializable getVariantId() throws QException
-   {
-      QSession session = QContext.getQSession();
-      if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getVariantOptionsTableTypeValue()))
-      {
-         throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'"));
-      }
-      Serializable variantId = session.getBackendVariants().get(backendMetaData.getVariantOptionsTableTypeValue());
-      return variantId;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -854,7 +831,7 @@ public class BaseAPIActionUtil
       String accessTokenKey = "accessToken";
       if(backendMetaData.getUsesVariants())
       {
-         Serializable variantId = getVariantId();
+         Serializable variantId = BackendVariantsUtil.getVariantId(backendMetaData);
          accessTokenKey = accessTokenKey + ":" + variantId;
       }
 
@@ -944,8 +921,11 @@ public class BaseAPIActionUtil
    {
       if(backendMetaData.getUsesVariants())
       {
-         QRecord record = getVariantRecord();
-         return (Pair.of(record.getValueString(backendMetaData.getVariantOptionsTableClientIdField()), record.getValueString(backendMetaData.getVariantOptionsTableClientSecretField())));
+         QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData);
+         return (Pair.of(
+            record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)),
+            record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET))
+         ));
       }
 
       return (Pair.of(backendMetaData.getClientId(), backendMetaData.getClientSecret()));
@@ -1480,9 +1460,9 @@ public class BaseAPIActionUtil
     ** Setter for session
     **
     *******************************************************************************/
+   @Deprecated(since = "wasn't used.")
    public void setSession(QSession session)
    {
-      this.session = session;
    }
 
 

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/metadata/APIBackendVariantSetting.java

@@ -0,0 +1,38 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.api.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+
+
+/*******************************************************************************
+ ** settings that the API backend module can get from a backend variant.
+ *******************************************************************************/
+public enum APIBackendVariantSetting implements BackendVariantSetting
+{
+   USERNAME,
+   PASSWORD,
+   API_KEY,
+   CLIENT_ID,
+   CLIENT_SECRET
+}

qqq-backend-module-filesystem/pom.xml

@@ -50,6 +50,17 @@
          <artifactId>aws-java-sdk-s3</artifactId>
          <version>1.12.261</version>
       </dependency>
+      <dependency>
+         <groupId>org.apache.sshd</groupId>
+         <artifactId>sshd-sftp</artifactId>
+         <version>2.14.0</version>
+      </dependency>
+      <dependency>
+         <groupId>org.apache.sshd</groupId>
+         <artifactId>sshd-sftp</artifactId>
+         <version>2.14.0</version>
+      </dependency>
+
       <dependency>
          <groupId>cloud.localstack</groupId>
          <artifactId>localstack-utils</artifactId>
@@ -57,6 +68,20 @@
          <scope>test</scope>
       </dependency>
 
+      <dependency>
+         <groupId>org.testcontainers</groupId>
+         <artifactId>testcontainers</artifactId>
+         <version>1.15.3</version>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <!-- this was added to help make testcontainers work -->
+         <groupId>net.java.dev.jna</groupId>
+         <artifactId>jna</artifactId>
+         <version>5.7.0</version>
+         <scope>test</scope>
+      </dependency>
+
       <!-- Common deps for all qqq modules -->
       <dependency>
          <groupId>org.apache.maven.plugins</groupId>

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -25,9 +25,13 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Serializable;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
 import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
@@ -36,8 +40,12 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
@@ -47,12 +55,20 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
+import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
+import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
+import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.NotImplementedException;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -68,6 +84,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 {
    private static final QLogger LOG = QLogger.getLogger(AbstractBaseFilesystemAction.class);
 
+   protected QRecord backendVariantRecord = null;
+
 
 
    /*******************************************************************************
@@ -80,10 +98,26 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
 
 
+   /***************************************************************************
+    ** get the size of the specified file, null if not supported/available
+    ***************************************************************************/
+   public abstract Long getFileSize(FILE file);
+
+   /***************************************************************************
+    ** get the createDate of the specified file, null if not supported/available
+    ***************************************************************************/
+   public abstract Instant getFileCreateDate(FILE file);
+
+   /***************************************************************************
+    ** get the createDate of the specified file, null if not supported/available
+    ***************************************************************************/
+   public abstract Instant getFileModifyDate(FILE file);
+
    /*******************************************************************************
-    ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses.
+    ** List the files for a table - or optionally, just a single file name -
+    ** to be implemented in module-specific subclasses.
     *******************************************************************************/
-   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException;
+   public abstract List<FILE> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedSingleFileName) throws QException;
 
    /*******************************************************************************
     ** Read the contents of a file - to be implemented in module-specific subclasses.
@@ -107,7 +141,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     **
     ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
     *******************************************************************************/
-   public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException;
+   public abstract void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException;
 
    /*******************************************************************************
     ** Move a file from a source path, to a destination path.
@@ -116,13 +150,21 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    public abstract void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException;
 
+
+
    /*******************************************************************************
     ** e.g., with a base path of /foo/
     ** and a table path of /bar/
     ** and a file at /foo/bar/baz.txt
     ** give us just the baz.txt part.
     *******************************************************************************/
-   public abstract String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData sourceBackend, QTableMetaData sourceTable);
+   public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table)
+   {
+      String tablePath           = getFullBasePath(table, backend);
+      String strippedPath        = filePath.replaceFirst(".*" + tablePath, "");
+      String withoutLeadingSlash = stripLeadingSlash(strippedPath); // todo - dangerous, do all backends really want this??
+      return (withoutLeadingSlash);
+   }
 
 
 
@@ -133,7 +175,17 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    public String getFullBasePath(QTableMetaData table, QBackendMetaData backendBase)
    {
       AbstractFilesystemBackendMetaData metaData = getBackendMetaData(AbstractFilesystemBackendMetaData.class, backendBase);
-      String                            fullPath = StringUtils.hasContent(metaData.getBasePath()) ? metaData.getBasePath() : "";
+
+      String basePath = metaData.getBasePath();
+      if(backendBase.getUsesVariants())
+      {
+         Map<BackendVariantSetting, String> fieldNameMap = backendBase.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap();
+         if(fieldNameMap.containsKey(SFTPBackendVariantSetting.BASE_PATH))
+         {
+            basePath = backendVariantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.BASE_PATH));
+         }
+      }
+      String fullPath = StringUtils.hasContent(basePath) ? basePath : "";
 
       AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
       if(StringUtils.hasContent(tableDetails.getBasePath()))
@@ -164,6 +216,34 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String stripLeadingSlash(String path)
+   {
+      if(path == null)
+      {
+         return (null);
+      }
+      return (path.replaceFirst("^/+", ""));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static String stripTrailingSlash(String path)
+   {
+      if(path == null)
+      {
+         return (null);
+      }
+      return (path.replaceFirst("/+$", ""));
+   }
+
+
+
    /*******************************************************************************
     ** Get the backend metaData, type-checked as the requested type.
     *******************************************************************************/
@@ -202,26 +282,174 @@ public abstract class AbstractBaseFilesystemAction<FILE>
 
       try
       {
-         QueryOutput queryOutput = new QueryOutput(queryInput);
-
          QTableMetaData                        table        = queryInput.getTable();
          AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
-         List<FILE>                            files        = listFiles(table, queryInput.getBackend(), queryInput.getFilter());
 
-         int recordCount = 0;
+         QueryOutput queryOutput = new QueryOutput(queryInput);
+
+         String     requestedPath = null;
+         List<FILE> files         = listFiles(table, queryInput.getBackend(), requestedPath);
 
-         FILE_LOOP:
-         for(FILE file : files)
-         {
-            InputStream inputStream = readFile(file);
          switch(tableDetails.getCardinality())
          {
-               case MANY:
+            case MANY -> completeExecuteQueryForManyTable(queryInput, queryOutput, files, table, tableDetails);
+            case ONE -> completeExecuteQueryForOneTable(queryInput, queryOutput, files, table, tableDetails);
+            default -> throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
+         }
+
+         return (queryOutput);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Error executing query", e);
+         throw new QException("Error executing query", e);
+      }
+      finally
+      {
+         postAction();
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void setRecordValueIfFieldNameHasContent(QRecord record, String fieldName, UnsafeSupplier<Serializable, ?> valueSupplier)
+   {
+      if(StringUtils.hasContent(fieldName))
+      {
+         try
+         {
+            record.setValue(fieldName, valueSupplier.get());
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Error setting record value for field", e, logPair("fieldName", fieldName));
+         }
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void completeExecuteQueryForOneTable(QueryInput queryInput, QueryOutput queryOutput, List<FILE> files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException
+   {
+      int           recordCount = 0;
+      List<QRecord> records     = new ArrayList<>();
+
+      for(FILE file : files)
+      {
+         ////////////////////////////////////////////////////////////////////////////////
+         // for one-record tables, put the entire file's contents into a single record //
+         ////////////////////////////////////////////////////////////////////////////////
+         String  filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
+         QRecord record              = new QRecord();
+
+         setRecordValueIfFieldNameHasContent(record, tableDetails.getFileNameFieldName(), () -> filePathWithoutBase);
+         setRecordValueIfFieldNameHasContent(record, tableDetails.getBaseNameFieldName(), () -> stripAllPaths(filePathWithoutBase));
+         setRecordValueIfFieldNameHasContent(record, tableDetails.getSizeFieldName(), () -> getFileSize(file));
+         setRecordValueIfFieldNameHasContent(record, tableDetails.getCreateDateFieldName(), () -> getFileCreateDate(file));
+         setRecordValueIfFieldNameHasContent(record, tableDetails.getModifyDateFieldName(), () -> getFileModifyDate(file));
+
+         if(shouldHeavyFileContentsBeRead(queryInput, table, tableDetails))
+         {
+            try(InputStream inputStream = readFile(file))
+            {
+               byte[] bytes = inputStream.readAllBytes();
+               record.withValue(tableDetails.getContentsFieldName(), bytes);
+            }
+            catch(Exception e)
+            {
+               record.addError(new SystemErrorStatusMessage("Error reading file contents: " + e.getMessage()));
+            }
+         }
+         else
+         {
+            Long size = record.getValueLong(tableDetails.getSizeFieldName());
+            if(size != null)
+            {
+               if(record.getBackendDetails() == null)
+               {
+                  record.setBackendDetails(new HashMap<>());
+               }
+
+               if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null)
+               {
+                  record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>());
+               }
+
+               ((Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size);
+            }
+         }
+
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // the listFiles method may have used a "path" criteria.                                                //
+         // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////
+         QQueryFilter filterForRecords = queryInput.getFilter();
+         // if(filterForRecords != null)
+         // {
+         //    filterForRecords = filterForRecords.clone();
+         //    CollectionUtils.nonNullList(filterForRecords.getCriteria())
+         //       .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria);
+         // }
+
+         if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record))
+         {
+            records.add(record);
+         }
+      }
+
+      BackendQueryFilterUtils.sortRecordList(queryInput.getFilter(), records);
+      records = BackendQueryFilterUtils.applySkipAndLimit(queryInput.getFilter(), records);
+      queryOutput.addRecords(records);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private Serializable stripAllPaths(String filePath)
+   {
+      if(filePath == null)
+      {
+         return null;
+      }
+
+      return (filePath.replaceFirst(".*/", ""));
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected static boolean isPathEqualsCriteria(QFilterCriteria criteria)
+   {
+      return "path".equals(criteria.getFieldName()) && QCriteriaOperator.EQUALS.equals(criteria.getOperator());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private void completeExecuteQueryForManyTable(QueryInput queryInput, QueryOutput queryOutput, List<FILE> files, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) throws QException, IOException
+   {
+      int recordCount = 0;
+
+      for(FILE file : files)
+      {
+         try(InputStream inputStream = readFile(file))
          {
             LOG.info("Extracting records from file", logPair("table", table.getName()), logPair("path", getFullPathForFile(file)));
             switch(tableDetails.getRecordFormat())
             {
-                     case CSV:
+               case CSV ->
                {
                   String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
                   fileContents = customizeFileContentsAfterReading(table, fileContents);
@@ -242,9 +470,8 @@ public abstract class AbstractBaseFilesystemAction<FILE>
                      addBackendDetailsToRecords(recordsInFile, file);
                      queryOutput.addRecords(recordsInFile);
                   }
-                        break;
                }
-                     case JSON:
+               case JSON ->
                {
                   String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
                   fileContents = customizeFileContentsAfterReading(table, fileContents);
@@ -254,61 +481,30 @@ public abstract class AbstractBaseFilesystemAction<FILE>
                   addBackendDetailsToRecords(recordsInFile, file);
 
                   queryOutput.addRecords(recordsInFile);
-                        break;
                }
-                     default:
-                     {
-                        throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
+               default -> throw new IllegalStateException("Unexpected table record format: " + tableDetails.getRecordFormat());
             }
          }
-                  break;
-               }
-               case ONE:
-               {
-                  ////////////////////////////////////////////////////////////////////////////////
-                  // for one-record tables, put the entire file's contents into a single record //
-                  ////////////////////////////////////////////////////////////////////////////////
-                  String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table);
-                  byte[] bytes               = inputStream.readAllBytes();
-
-                  QRecord record = new QRecord()
-                     .withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase)
-                     .withValue(tableDetails.getContentsFieldName(), bytes);
-                  queryOutput.addRecord(record);
-
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  // keep our own count - in case the query output is using a pipe (e.g., so we can't just call a .size()) //
-                  ///////////////////////////////////////////////////////////////////////////////////////////////////////////
-                  recordCount++;
-
-                  ////////////////////////////////////////////////////////////////////////////
-                  // break out of the file loop if we have hit the limit (if one was given) //
-                  ////////////////////////////////////////////////////////////////////////////
-                  if(queryInput.getFilter() != null && queryInput.getFilter().getLimit() != null)
-                  {
-                     if(recordCount >= queryInput.getFilter().getLimit())
-                     {
-                        break FILE_LOOP;
-                     }
-                  }
-
-                  break;
-               }
-               default:
-               {
-                  throw new IllegalStateException("Unexpected table cardinality: " + tableDetails.getCardinality());
-               }
       }
    }
 
-         return queryOutput;
-      }
-      catch(Exception e)
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static boolean shouldHeavyFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails)
    {
-         LOG.warn("Error executing query", e);
-         throw new QException("Error executing query", e);
+      boolean doReadContents = true;
+      if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy())
+      {
+         if(!queryInput.getShouldFetchHeavyFields())
+         {
+            doReadContents = false;
          }
       }
+      return doReadContents;
+   }
 
 
 
@@ -319,7 +515,16 @@ public abstract class AbstractBaseFilesystemAction<FILE>
    {
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(countInput.getTableName());
-      queryInput.setFilter(countInput.getFilter());
+
+      QQueryFilter filter = countInput.getFilter();
+      if(filter != null)
+      {
+         filter = filter.clone();
+         filter.setSkip(null);
+         filter.setLimit(null);
+      }
+
+      queryInput.setFilter(filter);
       QueryOutput queryOutput = executeQuery(queryInput);
 
       CountOutput countOutput = new CountOutput();
@@ -353,11 +558,24 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     ** Method that subclasses can override to add pre-action things (e.g., setting up
     ** s3 client).
     *******************************************************************************/
-   public void preAction(QBackendMetaData backendMetaData)
+   public void preAction(QBackendMetaData backendMetaData) throws QException
    {
-      /////////////////////////////////////////////////////////////////////
-      // noop in base class - subclasses can add functionality if needed //
-      /////////////////////////////////////////////////////////////////////
+      if(backendMetaData.getUsesVariants())
+      {
+         this.backendVariantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData);
+      }
+   }
+
+
+
+   /***************************************************************************
+    ** Method that subclasses can override to add post-action things (e.g., closing resources)
+    ***************************************************************************/
+   public void postAction()
+   {
+      //////////////////
+      // noop in base //
+      //////////////////
    }
 
 
@@ -410,12 +628,20 @@ public abstract class AbstractBaseFilesystemAction<FILE>
          if(tableDetails.getCardinality().equals(Cardinality.ONE))
          {
             for(QRecord record : insertInput.getRecords())
+            {
+               try
                {
                   String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString(tableDetails.getFileNameFieldName()));
                   writeFile(backend, fullPath, record.getValueByteArray(tableDetails.getContentsFieldName()));
                   record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
                   output.addRecord(record);
                }
+               catch(Exception e)
+               {
+                  record.addError(new SystemErrorStatusMessage("Error writing file: " + e.getMessage()));
+                  output.addRecord(record);
+               }
+            }
          }
          else
          {
@@ -428,5 +654,63 @@ public abstract class AbstractBaseFilesystemAction<FILE>
       {
          throw new QException("Error executing insert: " + e.getMessage(), e);
       }
+      finally
+      {
+         postAction();
       }
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected DeleteOutput executeDelete(DeleteInput deleteInput) throws QException
+   {
+      try
+      {
+         preAction(deleteInput.getBackend());
+
+         DeleteOutput output = new DeleteOutput();
+         output.setRecordsWithErrors(new ArrayList<>());
+
+         QTableMetaData   table   = deleteInput.getTable();
+         QBackendMetaData backend = deleteInput.getBackend();
+
+         AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+         if(tableDetails.getCardinality().equals(Cardinality.ONE))
+         {
+            int deletedCount = 0;
+            for(Serializable primaryKey : deleteInput.getPrimaryKeys())
+            {
+               try
+               {
+                  deleteFile(table, stripDuplicatedSlashes(getFullBasePath(table, backend) + "/" + primaryKey));
+                  deletedCount++;
+               }
+               catch(Exception e)
+               {
+                  String message = ObjectUtils.tryElse(() -> ExceptionUtils.getRootException(e).getMessage(), "Message not available");
+                  output.addRecordWithError(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withError(new SystemErrorStatusMessage("Error deleting file: " + message)));
+               }
+            }
+            output.setDeletedRecordCount(deletedCount);
+         }
+         else
+         {
+            throw (new NotImplementedException("Delete is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality()));
+         }
+
+         return (output);
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing delete: " + e.getMessage(), e);
+      }
+      finally
+      {
+         postAction();
+      }
+   }
+
+}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java

@@ -41,6 +41,10 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
 
    private String contentsFieldName;
    private String fileNameFieldName;
+   private String baseNameFieldName;
+   private String sizeFieldName;
+   private String createDateFieldName;
+   private String modifyDateFieldName;
 
 
 
@@ -281,4 +285,128 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
       }
 
    }
+
+   /*******************************************************************************
+    ** Getter for sizeFieldName
+    *******************************************************************************/
+   public String getSizeFieldName()
+   {
+      return (this.sizeFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for sizeFieldName
+    *******************************************************************************/
+   public void setSizeFieldName(String sizeFieldName)
+   {
+      this.sizeFieldName = sizeFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for sizeFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withSizeFieldName(String sizeFieldName)
+   {
+      this.sizeFieldName = sizeFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for createDateFieldName
+    *******************************************************************************/
+   public String getCreateDateFieldName()
+   {
+      return (this.createDateFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for createDateFieldName
+    *******************************************************************************/
+   public void setCreateDateFieldName(String createDateFieldName)
+   {
+      this.createDateFieldName = createDateFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for createDateFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withCreateDateFieldName(String createDateFieldName)
+   {
+      this.createDateFieldName = createDateFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for modifyDateFieldName
+    *******************************************************************************/
+   public String getModifyDateFieldName()
+   {
+      return (this.modifyDateFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for modifyDateFieldName
+    *******************************************************************************/
+   public void setModifyDateFieldName(String modifyDateFieldName)
+   {
+      this.modifyDateFieldName = modifyDateFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for modifyDateFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withModifyDateFieldName(String modifyDateFieldName)
+   {
+      this.modifyDateFieldName = modifyDateFieldName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for baseNameFieldName
+    *******************************************************************************/
+   public String getBaseNameFieldName()
+   {
+      return (this.baseNameFieldName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for baseNameFieldName
+    *******************************************************************************/
+   public void setBaseNameFieldName(String baseNameFieldName)
+   {
+      this.baseNameFieldName = baseNameFieldName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for baseNameFieldName
+    *******************************************************************************/
+   public AbstractFilesystemTableBackendDetails withBaseNameFieldName(String baseNameFieldName)
+   {
+      this.baseNameFieldName = baseNameFieldName;
+      return (this);
+   }
+
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java

@@ -22,14 +22,23 @@
 package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
 
 
+import java.util.ArrayList;
+import java.util.List;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory;
 import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule;
 import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
 import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
 import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule;
+import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
 
 
 /*******************************************************************************
@@ -53,6 +62,8 @@ public class FilesystemTableMetaDataBuilder
    private String           basePath;
    private String           glob;
 
+   private String contentsAdornmentFileNameField = "baseName";
+
 
 
    /*******************************************************************************
@@ -60,26 +71,64 @@ public class FilesystemTableMetaDataBuilder
     *******************************************************************************/
    public QTableMetaData buildStandardCardinalityOneTable()
    {
+      boolean includeCreateDate = true;
       AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType())
       {
-         case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails();
+         case S3BackendModule.BACKEND_TYPE ->
+         {
+            includeCreateDate = false;
+            yield new S3TableBackendDetails();
+         }
          case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails();
+         case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails();
          default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType());
       };
 
+      List<QFieldMetaData> fields = new ArrayList<>();
+
+      fields.add((new QFieldMetaData("fileName", QFieldType.STRING)));
+      fields.add((new QFieldMetaData("baseName", QFieldType.STRING)));
+      fields.add((new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)));
+      fields.add((new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)));
+      fields.add((new QFieldMetaData("contents", QFieldType.BLOB)
+         .withIsHeavy(true)
+         .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD)
+            .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s")
+            .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, contentsAdornmentFileNameField
+         ))));
+
+      QFieldSection t3Section = SectionFactory.defaultT3("modifyDate");
+
+      AbstractFilesystemTableBackendDetails backendDetails = tableBackendDetails
+         .withCardinality(Cardinality.ONE)
+         .withFileNameFieldName("fileName")
+         .withBaseNameFieldName("baseName")
+         .withContentsFieldName("contents")
+         .withSizeFieldName("size")
+         .withModifyDateFieldName("modifyDate")
+         .withBasePath(basePath)
+         .withGlob(glob);
+
+      if(includeCreateDate)
+      {
+         fields.add((new QFieldMetaData("createDate", QFieldType.DATE_TIME)));
+         backendDetails.setCreateDateFieldName("createDate");
+
+         ArrayList<String> t3FieldNames = new ArrayList<>(t3Section.getFieldNames());
+         t3FieldNames.add(0, "createDate");
+         t3Section.setFieldNames(t3FieldNames);
+      }
+
       return new QTableMetaData()
          .withName(name)
          .withIsHidden(true)
          .withBackendName(backend.getName())
          .withPrimaryKeyField("fileName")
-         .withField(new QFieldMetaData("fileName", QFieldType.INTEGER))
-         .withField(new QFieldMetaData("contents", QFieldType.STRING))
-         .withBackendDetails(tableBackendDetails
-            .withCardinality(Cardinality.ONE)
-            .withFileNameFieldName("fileName")
-            .withContentsFieldName("contents")
-            .withBasePath(basePath)
-            .withGlob(glob));
+         .withFields(fields)
+         .withSection(SectionFactory.defaultT1("fileName"))
+         .withSection(SectionFactory.defaultT2("baseName", "contents", "size"))
+         .withSection(t3Section)
+         .withBackendDetails(backendDetails);
    }
 
 
@@ -206,4 +255,35 @@ public class FilesystemTableMetaDataBuilder
       return (this);
    }
 
+
+   /*******************************************************************************
+    ** Getter for contentsAdornmentFileNameField
+    *******************************************************************************/
+   public String getContentsAdornmentFileNameField()
+   {
+      return (this.contentsAdornmentFileNameField);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentsAdornmentFileNameField
+    *******************************************************************************/
+   public void setContentsAdornmentFileNameField(String contentsAdornmentFileNameField)
+   {
+      this.contentsAdornmentFileNameField = contentsAdornmentFileNameField;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentsAdornmentFileNameField
+    *******************************************************************************/
+   public FilesystemTableMetaDataBuilder withContentsAdornmentFileNameField(String contentsAdornmentFileNameField)
+   {
+      this.contentsAdornmentFileNameField = contentsAdornmentFileNameField;
+      return (this);
+   }
+
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java

@@ -130,7 +130,11 @@ public class SharedFilesystemBackendModuleUtils
       }
       else
       {
-         throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // this happens in base class now, like, for query action, so, we think okay to just ignore. //
+         ///////////////////////////////////////////////////////////////////////////////////////////////
+         // throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName()));
+         return (true);
       }
    }
 

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java

@@ -35,10 +35,14 @@ import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -61,11 +65,55 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
 
 
 
+   /***************************************************************************
+    *
+    ***************************************************************************/
+   @Override
+   public Long getFileSize(File file)
+   {
+      return (file.length());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileCreateDate(File file)
+   {
+      try
+      {
+         Path                path         = file.toPath();
+         BasicFileAttributes attrs        = Files.readAttributes(path, BasicFileAttributes.class);
+         FileTime            creationTime = attrs.creationTime();
+         return creationTime.toInstant();
+      }
+      catch(IOException e)
+      {
+         LOG.warn("Error getting file createDate", e, logPair("file", file));
+         return (null);
+      }
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileModifyDate(File file)
+   {
+      return Instant.ofEpochMilli(file.lastModified());
+   }
+
+
+
    /*******************************************************************************
     ** List the files for this table.
     *******************************************************************************/
    @Override
-   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<File> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
    {
       try
       {
@@ -84,7 +132,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
 
          for(String matchedFile : matchedFiles)
          {
-            if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails))
+            boolean isMatch = true;
+            if(StringUtils.hasContent(requestedPath))
+            {
+               QQueryFilter filter = new QQueryFilter(new QFilterCriteria(tableBackendDetails.getFileNameFieldName(), QCriteriaOperator.EQUALS, requestedPath));
+               isMatch = SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails);
+            }
+
+            if(isMatch)
             {
                rs.add(new File(fullPath + File.separatorChar + matchedFile));
             }
@@ -175,7 +230,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction<File>
     ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
     *******************************************************************************/
    @Override
-   public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
+   public void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException
    {
       File file = new File(fileReference);
       if(!file.exists())

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java

@@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
-import org.apache.commons.lang.NotImplementedException;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class FilesystemDeleteAction implements DeleteInterface
+public class FilesystemDeleteAction extends AbstractFilesystemAction implements DeleteInterface
 {
 
    /*******************************************************************************
@@ -40,21 +39,19 @@ public class FilesystemDeleteAction implements DeleteInterface
     *******************************************************************************/
    public DeleteOutput execute(DeleteInput deleteInput) throws QException
    {
-      throw new NotImplementedException("Filesystem delete not implemented");
-      /*
-      try
+      return (executeDelete(deleteInput));
+   }
+
+
+
+   /*******************************************************************************
+    ** Specify whether this particular module's update action can & should fetch
+    ** records before updating them, e.g., for audits or "not-found-checks"
+    *******************************************************************************/
+   @Override
+   public boolean supportsPreFetchQuery()
    {
-         DeleteResult rs = new DeleteResult();
-         QTableMetaData table = deleteRequest.getTable();
-
-
-         // return rs;
-      }
-      catch(Exception e)
-      {
-         throw new QException("Error executing delete: " + e.getMessage(), e);
-      }
-      */
+      return (false);
    }
 
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java

@@ -94,7 +94,7 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
          if(VALUE_DELETE.equals(moveOrDelete))
          {
             LOG.info("Deleting ETL source file: " + sourceFile);
-            actionBase.deleteFile(QContext.getQInstance(), table, sourceFile);
+            actionBase.deleteFile(table, sourceFile);
          }
          else if(VALUE_MOVE.equals(moveOrDelete))
          {

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java

@@ -63,7 +63,6 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
-import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -314,13 +313,13 @@ public class FilesystemImporterStep implements BackendStep
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws FilesystemException
+   private static <F> void removeSourceFileIfSoConfigured(Boolean removeFileAfterImport, AbstractBaseFilesystemAction<F> sourceActionBase, QTableMetaData sourceTable, QBackendMetaData sourceBackend, String sourceFileName) throws QException
    {
       if(removeFileAfterImport)
       {
          String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend);
          LOG.info("Removing source file", logPair("path", fullBasePath + "/" + sourceFileName), logPair("sourceTable", sourceTable.getName()));
-         sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName);
+         sourceActionBase.deleteFile(sourceTable, fullBasePath + "/" + sourceFileName);
       }
       else
       {

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModule.java

@@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3;
 
 
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
@@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
+import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
 import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
@@ -112,6 +114,17 @@ public class S3BackendModule implements QBackendModuleInterface, FilesystemBacke
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public CountInterface getCountInterface()
+   {
+      return new S3CountAction();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -24,15 +24,16 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.Instant;
 import java.util.List;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@@ -56,11 +57,44 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Long getFileSize(S3ObjectSummary s3ObjectSummary)
+   {
+      return (s3ObjectSummary.getSize());
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileCreateDate(S3ObjectSummary s3ObjectSummary)
+   {
+      return null;
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public Instant getFileModifyDate(S3ObjectSummary s3ObjectSummary)
+   {
+      return s3ObjectSummary.getLastModified().toInstant();
+   }
+
+
+
    /*******************************************************************************
     ** Setup the s3 utils object to be used for this action.
     *******************************************************************************/
    @Override
-   public void preAction(QBackendMetaData backendMetaData)
+   public void preAction(QBackendMetaData backendMetaData) throws QException
    {
       super.preAction(backendMetaData);
 
@@ -129,7 +163,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
     ** List the files for a table.
     *******************************************************************************/
    @Override
-   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException
+   public List<S3ObjectSummary> listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException
    {
       S3BackendMetaData                     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase);
       AbstractFilesystemTableBackendDetails tableDetails      = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
@@ -141,7 +175,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
       ////////////////////////////////////////////////////////////////////
       // todo - look at metadata to configure the s3 client here?       //
       ////////////////////////////////////////////////////////////////////
-      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, filter, tableDetails);
+      return getS3Utils().listObjectsInBucketMatchingGlob(bucketName, fullPath, glob, requestedPath, tableDetails);
    }
 
 
@@ -179,20 +213,6 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private String stripLeadingSlash(String path)
-   {
-      if(path == null)
-      {
-         return (null);
-      }
-      return (path.replaceFirst("^/+", ""));
-   }
-
-
-
    /*******************************************************************************
     ** Get a string that represents the full path to a file.
     *******************************************************************************/
@@ -228,9 +248,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
     ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit
     *******************************************************************************/
    @Override
-   public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException
+   public void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException
    {
-      QBackendMetaData backend     = instance.getBackend(table.getBackendName());
+      QBackendMetaData backend     = QContext.getQInstance().getBackend(table.getBackendName());
       String           bucketName  = ((S3BackendMetaData) backend).getBucketName();
       String           cleanedPath = stripLeadingSlash(stripDuplicatedSlashes(fileReference));