Support api queryCriteria and orderBy for removed fields; more/better use of api names for tables & fields in openApi spec; pass qInstance through supplemental validation chain;

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -272,7 +272,7 @@ public class QInstanceEnricher
 
          for(QSupplementalTableMetaData supplementalTableMetaData : CollectionUtils.nonNullMap(table.getSupplementalMetaData()).values())
          {
-            supplementalTableMetaData.enrich(table);
+            supplementalTableMetaData.enrich(qInstance, table);
          }
       }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QSupplementalTableMetaData.java

@@ -22,6 +22,9 @@
 package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
 /*******************************************************************************
  ** Base-class for table-level meta-data defined by some supplemental module, etc,
  ** outside of qqq core
@@ -60,7 +63,7 @@ public abstract class QSupplementalTableMetaData
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void enrich(QTableMetaData table)
+   public void enrich(QInstance qInstance, QTableMetaData table)
    {
       ////////////////////////
       // noop in base class //

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java

@@ -36,9 +36,12 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.APIVersion;
+import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
 import com.kingsrook.qqq.api.model.actions.HttpApiResponse;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiOperation;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessCustomizers;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInput;
 import com.kingsrook.qqq.api.model.metadata.processes.ApiProcessInputFieldsContainer;
@@ -104,6 +107,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
 import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.CouldNotFindQueryFilterForExtractStepException;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@@ -150,6 +154,7 @@ public class ApiImplementation
 
       QTableMetaData table     = validateTableAndVersion(apiInstanceMetaData, version, tableApiName, ApiOperation.QUERY_BY_QUERY_STRING);
       String         tableName = table.getName();
+      String         apiName   = apiInstanceMetaData.getName();
 
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
@@ -231,6 +236,8 @@ public class ApiImplementation
          badRequestMessages.add("includeCount must be either true or false");
       }
 
+      Map<String, QFieldMetaData> tableApiFields = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, version, tableName));
+
       if(StringUtils.hasContent(orderBy))
       {
          for(String orderByPart : orderBy.split(","))
@@ -238,6 +245,7 @@ public class ApiImplementation
             orderByPart = orderByPart.trim();
             String[] orderByNameDirection = orderByPart.split(" +");
             boolean  asc                  = true;
+            String   apiFieldName         = orderByNameDirection[0];
             if(orderByNameDirection.length == 2)
             {
                if("asc".equalsIgnoreCase(orderByNameDirection[1]))
@@ -250,7 +258,7 @@ public class ApiImplementation
                }
                else
                {
-                  badRequestMessages.add("orderBy direction for field " + orderByNameDirection[0] + " must be either ASC or DESC.");
+                  badRequestMessages.add("orderBy direction for field " + apiFieldName + " must be either ASC or DESC.");
                }
             }
             else if(orderByNameDirection.length > 2)
@@ -258,14 +266,27 @@ public class ApiImplementation
                badRequestMessages.add("Unrecognized format for orderBy clause: " + orderByPart + ".  Expected:  fieldName [ASC|DESC].");
             }
 
-            try
+            QFieldMetaData field = tableApiFields.get(apiFieldName);
+            if(field == null)
             {
-               QFieldMetaData field = table.getField(orderByNameDirection[0]);
+               badRequestMessages.add("Unrecognized orderBy field name: " + apiFieldName + ".");
-               filter.withOrderBy(new QFilterOrderBy(field.getName(), asc));
             }
-            catch(Exception e)
+            else
             {
-               badRequestMessages.add("Unrecognized orderBy field name: " + orderByNameDirection[0] + ".");
+               QFilterOrderBy filterOrderBy = new QFilterOrderBy(field.getName(), asc);
+
+               ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
+               if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
+               {
+                  filterOrderBy.setFieldName(apiFieldMetaData.getReplacedByFieldName());
+               }
+               else if(apiFieldMetaData.getCustomValueMapper() != null)
+               {
+                  ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
+                  customValueMapper.customizeFilterOrderBy(queryInput, filterOrderBy, apiFieldName, apiFieldMetaData);
+               }
+
+               filter.withOrderBy(filterOrderBy);
             }
          }
       }
@@ -289,20 +310,36 @@ public class ApiImplementation
             continue;
          }
 
-         try
+         QFieldMetaData field = tableApiFields.get(name);
+         if(field == null)
          {
-            ////////////////////////////////////////////////////////////////////////////////////////////////
+            badRequestMessages.add("Unrecognized filter criteria field: " + name);
-            // todo - deal with removed fields; fields w/ custom value mappers (need new method(s) there) //
+         }
-            ////////////////////////////////////////////////////////////////////////////////////////////////
+         else
-
+         {
-            QFieldMetaData field = table.getField(name);
+            ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
             for(String value : values)
             {
                if(StringUtils.hasContent(value))
                {
                   try
                   {
-                     filter.addCriteria(parseQueryParamToCriteria(field, name, value));
+                     QFilterCriteria criteria = parseQueryParamToCriteria(field, name, value);
+
+                     /////////////////////////////////////////////
+                     // deal with replaced or customized fields //
+                     /////////////////////////////////////////////
+                     if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
+                     {
+                        criteria.setFieldName(apiFieldMetaData.getReplacedByFieldName());
+                     }
+                     else if(apiFieldMetaData.getCustomValueMapper() != null)
+                     {
+                        ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
+                        customValueMapper.customizeFilterCriteria(queryInput, filter, criteria, name, apiFieldMetaData);
+                     }
+
+                     filter.addCriteria(criteria);
                   }
                   catch(Exception e)
                   {
@@ -311,10 +348,6 @@ public class ApiImplementation
                }
             }
          }
-         catch(Exception e)
-         {
-            badRequestMessages.add("Unrecognized filter criteria field: " + name);
-         }
       }
 
       //////////////////////////////////////////
@@ -350,7 +383,7 @@ public class ApiImplementation
       ArrayList<Map<String, Serializable>> records = new ArrayList<>();
       for(QRecord record : queryOutput.getRecords())
       {
-         records.add(QRecordApiAdapter.qRecordToApiMap(record, tableName, apiInstanceMetaData.getName(), version));
+         records.add(QRecordApiAdapter.qRecordToApiMap(record, tableName, apiName, version));
       }
 
       /////////////////////////////

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -485,7 +485,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
                   .withDescription("How the results of the query should be sorted. SQL-style, comma-separated list of field names, each optionally followed by ASC or DESC (defaults to ASC).")
                   .withIn("query")
                   .withSchema(new Schema().withType("string"))
-                  .withExamples(buildOrderByExamples(primaryKeyApiName, tableApiFields)),
+                  .withExamples(buildOrderByExamples(apiName, primaryKeyApiName, tableApiFields)),
                new Parameter()
                   .withName("booleanOperator")
                   .withDescription("Whether to combine query field as an AND or an OR.  Default is AND.")
@@ -500,10 +500,12 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
          for(QFieldMetaData tableApiField : tableApiFields)
          {
-            String label = tableApiField.getLabel();
+            String fieldName = ApiFieldMetaData.getEffectiveApiFieldName(apiName, tableApiField);
+            String label     = tableApiField.getLabel();
+
             if(!StringUtils.hasContent(label))
             {
-               label = QInstanceEnricher.nameToLabel(tableApiField.getName());
+               label = QInstanceEnricher.nameToLabel(fieldName);
             }
 
             StringBuilder description = new StringBuilder("Query on the " + label + " field.  ");
@@ -517,7 +519,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
             }
 
             queryGet.getParameters().add(new Parameter()
-               .withName(tableApiField.getName())
+               .withName(fieldName)
                .withDescription(description.toString())
                .withIn("query")
                .withExplode(true)
@@ -892,6 +894,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       ////////////////////////////////
       List<Parameter> parameters      = new ArrayList<>();
       ApiProcessInput apiProcessInput = apiProcessMetaData.getInput();
+      String          apiName         = apiInstanceMetaData.getName();
       if(apiProcessInput != null)
       {
          ApiProcessInputFieldsContainer queryStringParams = apiProcessInput.getQueryStringParams();
@@ -912,12 +915,13 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
          if(bodyField != null)
          {
             ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(bodyField);
-            ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiInstanceMetaData.getName());
+            ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiName);
 
             String fieldLabel = bodyField.getLabel();
+            String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiName, bodyField);
             if(!StringUtils.hasContent(fieldLabel))
             {
-               fieldLabel = QInstanceEnricher.nameToLabel(bodyField.getName());
+               fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
             }
 
             String bodyDescription = "Value for the " + fieldLabel;
@@ -979,7 +983,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       ApiProcessOutputInterface output = apiProcessMetaData.getOutput();
       if(!ApiProcessMetaData.AsyncMode.ALWAYS.equals(apiProcessMetaData.getAsyncMode()))
       {
-         responses.putAll(output.getSpecResponses(apiInstanceMetaData.getName()));
+         responses.putAll(output.getSpecResponses(apiName));
       }
       if(!ApiProcessMetaData.AsyncMode.NEVER.equals(apiProcessMetaData.getAsyncMode()))
       {
@@ -1074,13 +1078,16 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
     *******************************************************************************/
    private Parameter processFieldToParameter(ApiInstanceMetaData apiInstanceMetaData, QFieldMetaData field)
    {
-      ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(field);
+      String apiName = apiInstanceMetaData.getName();
-      ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiInstanceMetaData.getName());
 
+      ApiFieldMetaDataContainer apiFieldMetaDataContainer = ApiFieldMetaDataContainer.ofOrNew(field);
+      ApiFieldMetaData          apiFieldMetaData          = apiFieldMetaDataContainer.getApiFieldMetaData(apiName);
+
+      String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiName, field);
       String fieldLabel = field.getLabel();
       if(!StringUtils.hasContent(fieldLabel))
       {
-         fieldLabel = QInstanceEnricher.nameToLabel(field.getName());
+         fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
       }
 
       String description = "Value for the " + fieldLabel + " field.";
@@ -1097,7 +1104,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       Schema fieldSchema = getFieldSchema(field, description, apiInstanceMetaData);
 
       Parameter parameter = new Parameter()
-         .withName(field.getName())
+         .withName(fieldName)
          .withDescription(description)
          .withRequired(field.getIsRequired())
          .withSchema(fieldSchema);
@@ -1213,14 +1220,15 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       for(QFieldMetaData field : tableApiFields)
       {
          String fieldLabel = field.getLabel();
+         String fieldName  = ApiFieldMetaData.getEffectiveApiFieldName(apiInstanceMetaData.getName(), field);
          if(!StringUtils.hasContent(fieldLabel))
          {
-            fieldLabel = QInstanceEnricher.nameToLabel(field.getName());
+            fieldLabel = QInstanceEnricher.nameToLabel(fieldName);
          }
 
          String defaultDescription = fieldLabel + " for the " + table.getLabel() + ".";
          Schema fieldSchema        = getFieldSchema(field, defaultDescription, apiInstanceMetaData);
-         tableFields.put(ApiFieldMetaData.getEffectiveApiFieldName(apiInstanceMetaData.getName(), field), fieldSchema);
+         tableFields.put(fieldName, fieldSchema);
       }
 
       //////////////////////////////////
@@ -1561,7 +1569,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
    /*******************************************************************************
     **
     *******************************************************************************/
-   private Map<String, Example> buildOrderByExamples(String primaryKeyApiName, List<? extends QFieldMetaData> tableApiFields)
+   private Map<String, Example> buildOrderByExamples(String apiName, String primaryKeyApiName, List<? extends QFieldMetaData> tableApiFields)
    {
       Map<String, Example> rs = new LinkedHashMap<>();
 
@@ -1569,7 +1577,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       List<String> fieldsForExample5 = new ArrayList<>();
       for(QFieldMetaData tableApiField : tableApiFields)
       {
-         String name = tableApiField.getName();
+         String name = ApiFieldMetaData.getEffectiveApiFieldName(apiName, tableApiField);
          if(primaryKeyApiName.equals(name) || fieldsForExample4.contains(name) || fieldsForExample5.contains(name))
          {
             continue;

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GetTableApiFieldsAction.java

@@ -24,7 +24,10 @@ package com.kingsrook.qqq.api.actions;
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
@@ -50,6 +53,65 @@ import org.apache.commons.lang.BooleanUtils;
  *******************************************************************************/
 public class GetTableApiFieldsAction extends AbstractQActionFunction<GetTableApiFieldsInput, GetTableApiFieldsOutput>
 {
+   private static Map<ApiNameVersionAndTableName, List<QFieldMetaData>>        fieldListCache = new HashMap<>();
+   private static Map<ApiNameVersionAndTableName, Map<String, QFieldMetaData>> fieldMapCache  = new HashMap<>();
+
+
+
+   /*******************************************************************************
+    ** Allow tests (that manipulate meta-data) to clear field caches.
+    *******************************************************************************/
+   public static void clearCaches()
+   {
+      fieldListCache.clear();
+      fieldMapCache.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** convenience (and caching) wrapper
+    *******************************************************************************/
+   public static Map<String, QFieldMetaData> getTableApiFieldMap(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
+   {
+      if(!fieldMapCache.containsKey(apiNameVersionAndTableName))
+      {
+         Map<String, QFieldMetaData> map = getTableApiFieldList(apiNameVersionAndTableName).stream().collect(Collectors.toMap(f -> (ApiFieldMetaData.getEffectiveApiFieldName(apiNameVersionAndTableName.apiName(), f)), f -> f));
+         fieldMapCache.put(apiNameVersionAndTableName, map);
+      }
+
+      return (fieldMapCache.get(apiNameVersionAndTableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** convenience (and caching) wrapper
+    *******************************************************************************/
+   public static List<QFieldMetaData> getTableApiFieldList(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
+   {
+      if(!fieldListCache.containsKey(apiNameVersionAndTableName))
+      {
+         List<QFieldMetaData> value = new GetTableApiFieldsAction().execute(new GetTableApiFieldsInput()
+            .withTableName(apiNameVersionAndTableName.tableName())
+            .withVersion(apiNameVersionAndTableName.apiVersion())
+            .withApiName(apiNameVersionAndTableName.apiName())).getFields();
+         fieldListCache.put(apiNameVersionAndTableName, value);
+      }
+      return (fieldListCache.get(apiNameVersionAndTableName));
+   }
+
+
+
+   /*******************************************************************************
+    ** Input-record for convenience methods
+    *******************************************************************************/
+   public record ApiNameVersionAndTableName(String apiName, String apiVersion, String tableName)
+   {
+
+   }
+
+
 
    /*******************************************************************************
     **

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java

@@ -29,12 +29,10 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.APIVersionRange;
 import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
-import com.kingsrook.qqq.api.model.actions.GetTableApiFieldsInput;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
@@ -66,20 +64,6 @@ public class QRecordApiAdapter
 {
    private static final QLogger LOG = QLogger.getLogger(QRecordApiAdapter.class);
 
-   private static Map<ApiNameVersionAndTableName, List<QFieldMetaData>>        fieldListCache = new HashMap<>();
-   private static Map<ApiNameVersionAndTableName, Map<String, QFieldMetaData>> fieldMapCache  = new HashMap<>();
-
-
-
-   /*******************************************************************************
-    ** Allow tests (that manipulate meta-data) to clear field caches.
-    *******************************************************************************/
-   public static void clearCaches()
-   {
-      fieldListCache.clear();
-      fieldMapCache.clear();
-   }
-
 
 
    /*******************************************************************************
@@ -92,7 +76,7 @@ public class QRecordApiAdapter
          return (null);
       }
 
-      List<QFieldMetaData>                tableApiFields = getTableApiFieldList(new ApiNameVersionAndTableName(apiName, apiVersion, tableName));
+      List<QFieldMetaData>                tableApiFields = GetTableApiFieldsAction.getTableApiFieldList(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, apiVersion, tableName));
       LinkedHashMap<String, Serializable> outputRecord   = new LinkedHashMap<>();
 
       /////////////////////////////////////////
@@ -111,7 +95,7 @@ public class QRecordApiAdapter
          else if(apiFieldMetaData.getCustomValueMapper() != null)
          {
             ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
-            value = customValueMapper.produceApiValue(record);
+            value = customValueMapper.produceApiValue(record, apiFieldName);
          }
          else
          {
@@ -185,7 +169,7 @@ public class QRecordApiAdapter
       ////////////////////////////////////////////////////////////////////////////////
       // make map of apiFieldNames (e.g., names as api uses them) to QFieldMetaData //
       ////////////////////////////////////////////////////////////////////////////////
-      Map<String, QFieldMetaData> apiFieldsMap           = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, apiVersion, tableName));
+      Map<String, QFieldMetaData> apiFieldsMap           = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, apiVersion, tableName));
       List<String>                unrecognizedFieldNames = new ArrayList<>();
       QRecord                     qRecord                = new QRecord();
 
@@ -241,7 +225,7 @@ public class QRecordApiAdapter
             else if(apiFieldMetaData.getCustomValueMapper() != null)
             {
                ApiFieldCustomValueMapper customValueMapper = QCodeLoader.getAdHoc(ApiFieldCustomValueMapper.class, apiFieldMetaData.getCustomValueMapper());
-               customValueMapper.consumeApiValue(qRecord, value, jsonObject);
+               customValueMapper.consumeApiValue(qRecord, value, jsonObject, jsonKey);
             }
             else
             {
@@ -332,7 +316,7 @@ public class QRecordApiAdapter
       {
          if(!supportedVersion.toString().equals(apiVersion))
          {
-            Map<String, QFieldMetaData> versionFields = getTableApiFieldMap(new ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName));
+            Map<String, QFieldMetaData> versionFields = GetTableApiFieldsAction.getTableApiFieldMap(new GetTableApiFieldsAction.ApiNameVersionAndTableName(apiName, supportedVersion.toString(), tableName));
             if(versionFields.containsKey(unrecognizedFieldName))
             {
                versionsWithThisField.add(supportedVersion.toString());
@@ -348,47 +332,4 @@ public class QRecordApiAdapter
       return (null);
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static Map<String, QFieldMetaData> getTableApiFieldMap(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
-   {
-      if(!fieldMapCache.containsKey(apiNameVersionAndTableName))
-      {
-         Map<String, QFieldMetaData> map = getTableApiFieldList(apiNameVersionAndTableName).stream().collect(Collectors.toMap(f -> (ApiFieldMetaData.getEffectiveApiFieldName(apiNameVersionAndTableName.apiName(), f)), f -> f));
-         fieldMapCache.put(apiNameVersionAndTableName, map);
-      }
-
-      return (fieldMapCache.get(apiNameVersionAndTableName));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QFieldMetaData> getTableApiFieldList(ApiNameVersionAndTableName apiNameVersionAndTableName) throws QException
-   {
-      if(!fieldListCache.containsKey(apiNameVersionAndTableName))
-      {
-         List<QFieldMetaData> value = new GetTableApiFieldsAction().execute(new GetTableApiFieldsInput()
-            .withTableName(apiNameVersionAndTableName.tableName())
-            .withVersion(apiNameVersionAndTableName.apiVersion())
-            .withApiName(apiNameVersionAndTableName.apiName())).getFields();
-         fieldListCache.put(apiNameVersionAndTableName, value);
-      }
-      return (fieldListCache.get(apiNameVersionAndTableName));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private record ApiNameVersionAndTableName(String apiName, String apiVersion, String tableName)
-   {
-
-   }
 }

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/actions/ApiFieldCustomValueMapper.java

@@ -23,6 +23,11 @@ package com.kingsrook.qqq.api.model.actions;
 
 
 import java.io.Serializable;
+import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import org.json.JSONObject;
 
@@ -34,9 +39,11 @@ public abstract class ApiFieldCustomValueMapper
 {
 
    /*******************************************************************************
-    **
+    ** When producing a JSON Object to send over the API (e.g., for a GET), this method
+    ** can run to customize the value that is produced, for the input QRecord's specified
+    ** fieldName
     *******************************************************************************/
-   public Serializable produceApiValue(QRecord record)
+   public Serializable produceApiValue(QRecord record, String apiFieldName)
    {
       /////////////////////
       // null by default //
@@ -46,10 +53,36 @@ public abstract class ApiFieldCustomValueMapper
 
 
 
+   /*******************************************************************************
+    ** When producing a QRecord (the first parameter) from a JSON Object that was
+    ** received from the API (e.g., a POST or PATCH) - this method can run to
+    ** allow customization of the incoming value.
+    *******************************************************************************/
+   public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject, String apiFieldName)
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject)
+   public void customizeFilterCriteria(QueryInput queryInput, QQueryFilter filter, QFilterCriteria criteria, String apiFieldName, ApiFieldMetaData apiFieldMetaData)
+   {
+      /////////////////////
+      // noop by default //
+      /////////////////////
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void customizeFilterOrderBy(QueryInput queryInput, QFilterOrderBy orderBy, String apiFieldName, ApiFieldMetaData apiFieldMetaData)
    {
       /////////////////////
       // noop by default //

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/tables/ApiTableMetaData.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.api.model.metadata.ApiOperation;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaData;
 import com.kingsrook.qqq.api.model.metadata.fields.ApiFieldMetaDataContainer;
 import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -80,7 +81,7 @@ public class ApiTableMetaData implements ApiOperation.EnabledOperationsProvider
    /*******************************************************************************
     **
     *******************************************************************************/
-   public void enrich(String apiName, QTableMetaData table)
+   public void enrich(QInstance qInstance, String apiName, QTableMetaData table)
    {
       if(initialVersion != null)
       {
@@ -95,7 +96,7 @@ public class ApiTableMetaData implements ApiOperation.EnabledOperationsProvider
 
          for(QFieldMetaData field : CollectionUtils.nonNullList(removedApiFields))
          {
-            new QInstanceEnricher(null).enrichField(field);
+            new QInstanceEnricher(qInstance).enrichField(field);
             ApiFieldMetaData apiFieldMetaData = ensureFieldHasApiSupplementalMetaData(apiName, field);
             if(apiFieldMetaData.getInitialVersion() == null)
             {

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/model/metadata/tables/ApiTableMetaDataContainer.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.api.model.metadata.tables;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import com.kingsrook.qqq.api.ApiSupplementType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QSupplementalTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -80,13 +81,13 @@ public class ApiTableMetaDataContainer extends QSupplementalTableMetaData
     **
     *******************************************************************************/
    @Override
-   public void enrich(QTableMetaData table)
+   public void enrich(QInstance qInstance, QTableMetaData table)
    {
-      super.enrich(table);
+      super.enrich(qInstance, table);
 
       for(Map.Entry<String, ApiTableMetaData> entry : CollectionUtils.nonNullMap(apis).entrySet())
       {
-         entry.getValue().enrich(entry.getKey(), table);
+         entry.getValue().enrich(qInstance, entry.getKey(), table);
       }
    }
 

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/ApiImplementationTest.java

@@ -23,9 +23,14 @@ package com.kingsrook.qqq.api.actions;
 
 
 import java.io.Serializable;
+import java.math.BigDecimal;
+import java.time.LocalDate;
+import java.time.Month;
+import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.api.BaseTest;
 import com.kingsrook.qqq.api.TestUtils;
+import com.kingsrook.qqq.api.javalin.QBadRequestException;
 import com.kingsrook.qqq.api.model.actions.ApiFieldCustomValueMapper;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
@@ -35,19 +40,23 @@ import com.kingsrook.qqq.api.model.metadata.tables.ApiAssociationMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaData;
 import com.kingsrook.qqq.api.model.metadata.tables.ApiTableMetaDataContainer;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
+import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
 import org.json.JSONObject;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -66,7 +75,7 @@ class ApiImplementationTest extends BaseTest
    @AfterEach
    void beforeAndAfterEach()
    {
-      QRecordApiAdapter.clearCaches();
+      GetTableApiFieldsAction.clearCaches();
    }
 
 
@@ -188,6 +197,43 @@ class ApiImplementationTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testQueryWithRemovedFields() throws QException
+   {
+      QInstance           qInstance           = QContext.getQInstance();
+      ApiInstanceMetaData apiInstanceMetaData = ApiInstanceMetaDataContainer.of(qInstance).getApiInstanceMetaData(TestUtils.API_NAME);
+
+      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecord(new QRecord()
+         .withValue("firstName", "Tim")
+         .withValue("noOfShoes", 2)
+         .withValue("birthDate", LocalDate.of(1980, Month.MAY, 31))
+         .withValue("cost", new BigDecimal("3.50"))
+         .withValue("price", new BigDecimal("9.99"))
+         .withValue("photo", "ABCD".getBytes())));
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+      // query by a field that wasn't in an old api version, but is in the table now - should fail //
+      ///////////////////////////////////////////////////////////////////////////////////////////////
+
+      assertThatThrownBy(() ->
+         ApiImplementation.query(apiInstanceMetaData, TestUtils.V2022_Q4, TestUtils.TABLE_NAME_PERSON, MapBuilder.of("noOfShoes", List.of("2"))))
+         .isInstanceOf(QBadRequestException.class)
+         .hasMessageContaining("Unrecognized filter criteria field");
+
+      {
+         /////////////////////////////////////////////
+         // query by a removed field (was replaced) //
+         /////////////////////////////////////////////
+         Map<String, Serializable> queryResult = ApiImplementation.query(apiInstanceMetaData, TestUtils.V2022_Q4, TestUtils.TABLE_NAME_PERSON, MapBuilder.of("shoeCount", List.of("2")));
+         assertEquals(1, queryResult.get("count"));
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -198,7 +244,7 @@ class ApiImplementationTest extends BaseTest
        **
        *******************************************************************************/
       @Override
-      public Serializable produceApiValue(QRecord record)
+      public Serializable produceApiValue(QRecord record, String apiFieldName)
       {
          return ("customValue-" + record.getValueString("lastName"));
       }
@@ -209,7 +255,7 @@ class ApiImplementationTest extends BaseTest
        **
        *******************************************************************************/
       @Override
-      public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject)
+      public void consumeApiValue(QRecord record, Object value, JSONObject fullApiJsonObject, String apiFieldName)
       {
          String valueString = ValueUtils.getValueAsString(value);
          valueString = valueString.replaceFirst("^stripThisAway-", "");