Handling of BLOBs

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java

@@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
@@ -108,6 +109,7 @@ public class ApiImplementation
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
       queryInput.setIncludeAssociations(true);
+      queryInput.setShouldFetchHeavyFields(true);
 
       PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ);
 
@@ -251,7 +253,7 @@ public class ApiImplementation
                {
                   try
                   {
-                     filter.addCriteria(parseQueryParamToCriteria(name, value));
+                     filter.addCriteria(parseQueryParamToCriteria(field, name, value));
                   }
                   catch(Exception e)
                   {
@@ -506,6 +508,7 @@ public class ApiImplementation
 
       getInput.setPrimaryKey(primaryKey);
       getInput.setIncludeAssociations(true);
+      getInput.setShouldFetchHeavyFields(true);
 
       GetAction getAction = new GetAction();
       GetOutput getOutput = getAction.execute(getInput);
@@ -911,7 +914,7 @@ public class ApiImplementation
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QFilterCriteria parseQueryParamToCriteria(String name, String value) throws QException
+   private static QFilterCriteria parseQueryParamToCriteria(QFieldMetaData field, String name, String value) throws QException
    {
       ///////////////////////////////////
       // process & discard a leading ! //
@@ -989,6 +992,14 @@ public class ApiImplementation
          throw (new QException("Unexpected noOfValues [" + selectedOperator.noOfValues + "] in operator [" + selectedOperator + "]"));
       }
 
+      if(field.getType().equals(QFieldType.BLOB))
+      {
+         if(!selectedOperator.equals(Operator.EMPTY))
+         {
+            throw (new QBadRequestException("Operator " + selectedOperator.prefix + " may not be used for field " + name + " (blob fields only support operators EMPTY or !EMPTY)"));
+         }
+      }
+
       return (new QFilterCriteria(name, isNot ? selectedOperator.negativeOperator : selectedOperator.positiveOperator, criteriaValues));
    }
 

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java

@@ -480,9 +480,19 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
          for(QFieldMetaData tableApiField : tableApiFields)
          {
+            StringBuilder description = new StringBuilder("Query on the " + tableApiField.getLabel() + " field.  ");
+            if(tableApiField.getType().equals(QFieldType.BLOB))
+            {
+               description.append("Can only query for EMPTY or !EMPTY.");
+            }
+            else
+            {
+               description.append("Can prefix value with an operator, else defaults to = (equals).");
+            }
+
             queryGet.getParameters().add(new Parameter()
                .withName(tableApiField.getName())
-               .withDescription("Query on the " + tableApiField.getLabel() + " field.  Can prefix value with an operator, else defaults to = (equals).")
+               .withDescription(description.toString())
                .withIn("query")
                .withExplode(true)
                .withSchema(new Schema()
@@ -837,6 +847,9 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       rs.put("criteriaStringNotLike", new ExampleWithListValue().withSummary("not starting with f").withValue(ListBuilder.of("!LIKE f%")));
       rs.put("criteriaStringMultiple", new ExampleWithListValue().withSummary("multiple criteria: between bar and foo and not equal to baz").withValue(ListBuilder.of("BETWEEN bar,foo", "!baz")));
 
+      rs.put("criteriaBlobEmpty", new ExampleWithListValue().withSummary("null value").withValue(ListBuilder.of("EMPTY")));
+      rs.put("criteriaBlobNotEmpty", new ExampleWithListValue().withSummary("non-null value").withValue(ListBuilder.of("!EMPTY")));
+
       return (rs);
    }
 
@@ -870,6 +883,10 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       {
          componentExamples.keySet().stream().filter(s -> s.startsWith("criteriaBoolean")).forEach(exampleRefs::add);
       }
+      else if(tableApiField.getType().equals(QFieldType.BLOB))
+      {
+         componentExamples.keySet().stream().filter(s -> s.startsWith("criteriaBlob")).forEach(exampleRefs::add);
+      }
 
       Map<String, Example> rs = new LinkedHashMap<>();
 
@@ -910,10 +927,16 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
     *******************************************************************************/
    private Schema getFieldSchema(QTableMetaData table, QFieldMetaData field)
    {
+      String description = field.getLabel() + " for the " + table.getLabel() + ".";
+      if(field.getType().equals(QFieldType.BLOB))
+      {
+         description = "Base64 encoded " + description;
+      }
+
       Schema fieldSchema = new Schema()
          .withType(getFieldType(field))
          .withFormat(getFieldFormat(field))
-         .withDescription(field.getLabel() + " for the " + table.getLabel() + ".");
+         .withDescription(description);
 
       if(!field.getIsEditable())
       {

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.api.actions;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -79,14 +81,23 @@ public class QRecordApiAdapter
       {
          ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
          String           apiFieldName     = ApiFieldMetaData.getEffectiveApiFieldName(apiName, field);
+
+         Serializable value = null;
          if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
          {
-            outputRecord.put(apiFieldName, record.getValue(apiFieldMetaData.getReplacedByFieldName()));
+            value = record.getValue(apiFieldMetaData.getReplacedByFieldName());
          }
          else
          {
-            outputRecord.put(apiFieldName, record.getValue(field.getName()));
+            value = record.getValue(field.getName());
          }
+
+         if(field.getType().equals(QFieldType.BLOB) && value instanceof byte[] bytes)
+         {
+            value = Base64.getEncoder().encodeToString(bytes);
+         }
+
+         outputRecord.put(apiFieldName, value);
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -142,6 +153,11 @@ public class QRecordApiAdapter
             QFieldMetaData field = apiFieldsMap.get(jsonKey);
             Object         value = jsonObject.isNull(jsonKey) ? null : jsonObject.get(jsonKey);
 
+            if(field.getType().equals(QFieldType.BLOB) && value instanceof String s)
+            {
+               value = Base64.getDecoder().decode(s);
+            }
+
             ////////////////////////////////////////////////////////////////////////////////////////////////////////
             // generally, omit non-editable fields -                                                              //
             // however - if we're asked to include the primary key (and this is the primary key), then include it //

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java

@@ -22,8 +22,8 @@
 package com.kingsrook.qqq.api;
 
 
-import java.time.LocalDate;
 import java.util.List;
+import java.util.function.Consumer;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
@@ -191,7 +191,8 @@ public class TestUtils
          // .withField(new QFieldMetaData("customValue", QFieldType.INTEGER).withPossibleValueSourceName(POSSIBLE_VALUE_SOURCE_CUSTOM))
          .withField(new QFieldMetaData("noOfShoes", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS))
          .withField(new QFieldMetaData("cost", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
-         .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY));
+         .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
+         .withField(new QFieldMetaData("photo", QFieldType.BLOB));
 
       table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD.getRole(), new QCodeReference(PersonPreInsertCustomizer.class, QCodeUsage.CUSTOMIZER));
 
@@ -378,11 +379,16 @@ public class TestUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void insertPersonRecord(Integer id, String firstName, String lastName, LocalDate birthDate) throws QException
+   public static void insertPersonRecord(Integer id, String firstName, String lastName, Consumer<QRecord> recordCustomizer) throws QException
    {
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
-      insertInput.setRecords(List.of(new QRecord().withValue("id", id).withValue("firstName", firstName).withValue("lastName", lastName).withValue("birthDate", birthDate)));
+      QRecord record = new QRecord().withValue("id", id).withValue("firstName", firstName).withValue("lastName", lastName);
+      if(recordCustomizer != null)
+      {
+         recordCustomizer.accept(record);
+      }
+      insertInput.setRecords(List.of(record));
       new InsertAction().execute(insertInput);
    }
 

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecActionTest.java

@@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -92,7 +93,13 @@ class GenerateOpenApiSpecActionTest extends BaseTest
                   .withTableName(table.getName())
                   .withVersion(supportedVersion.toString())
                   .withApiName(apiInstanceMetaData.getName()));
-               // System.out.println(output.getYaml());
+
+               if(table.getName().equals(TestUtils.TABLE_NAME_PERSON))
+               {
+                  assertThat(output.getYaml())
+                     .contains("Query on the First Name field.  Can prefix value with an operator")
+                     .contains("Query on the Photo field.  Can only query for EMPTY or !EMPTY");
+               }
             }
          }
       }

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java

@@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
@@ -61,12 +62,14 @@ class QRecordApiAdapterTest extends BaseTest
          .withValue("noOfShoes", 2)
          .withValue("birthDate", LocalDate.of(1980, Month.MAY, 31))
          .withValue("cost", new BigDecimal("3.50"))
-         .withValue("price", new BigDecimal("9.99"));
+         .withValue("price", new BigDecimal("9.99"))
+         .withValue("photo", "ABCD".getBytes());
 
       Map<String, Serializable> pastApiRecord = QRecordApiAdapter.qRecordToApiMap(person, TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2022_Q4);
       assertEquals(2, pastApiRecord.get("shoeCount")); // old field name - not currently in the QTable, but we can still get its value!
       assertFalse(pastApiRecord.containsKey("noOfShoes")); // current field name - doesn't appear in old api-version
       assertFalse(pastApiRecord.containsKey("cost")); // a current field name, but also not in this old api version
+      assertEquals("QUJDRA==", pastApiRecord.get("photo")); // base64 version of "ABCD".getBytes()
 
       Map<String, Serializable> currentApiRecord = QRecordApiAdapter.qRecordToApiMap(person, TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2023_Q1);
       assertFalse(currentApiRecord.containsKey("shoeCount")); // old field name - not in this current api version
@@ -109,9 +112,10 @@ class QRecordApiAdapterTest extends BaseTest
       // past version took shoeCount - so we still take that, but now put it in noOfShoes field of qRecord //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
       QRecord recordFromOldApi = QRecordApiAdapter.apiJsonObjectToQRecord(new JSONObject("""
-         {"firstName": "Tim", "shoeCount": 2}
+         {"firstName": "Tim", "shoeCount": 2, "photo": "QUJDRA=="}
          """), TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2022_Q4, true);
       assertEquals(2, recordFromOldApi.getValueInteger("noOfShoes"));
+      assertArrayEquals("ABCD".getBytes(), recordFromOldApi.getValueByteArray("photo"));
 
       ///////////////////////////////////////////
       // current version takes it as noOfShoes //

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java

@@ -64,6 +64,7 @@ import org.junit.jupiter.api.Test;
 import static com.kingsrook.qqq.api.TestUtils.insertPersonRecord;
 import static com.kingsrook.qqq.api.TestUtils.insertSimpsons;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -194,7 +195,7 @@ class QJavalinApiHandlerTest extends BaseTest
    @Test
    void testGet200() throws QException
    {
-      insertPersonRecord(1, "Homer", "Simpson");
+      insertPersonRecord(1, "Homer", "Simpson", qRecord -> qRecord.withValue("photo", "12345".getBytes()));
 
       HttpResponse<String> response = Unirest.get(BASE_URL + "/api/" + VERSION + "/person/1").asString();
       assertEquals(HttpStatus.OK_200, response.getStatus());
@@ -202,6 +203,7 @@ class QJavalinApiHandlerTest extends BaseTest
       assertEquals(1, jsonObject.getInt("id"));
       assertEquals("Homer", jsonObject.getString("firstName"));
       assertEquals("Simpson", jsonObject.getString("lastName"));
+      assertEquals("MTIzNDU=", jsonObject.getString("photo")); // base64 of "12345".getBytes()
       assertTrue(jsonObject.isNull("noOfShoes"));
       assertFalse(jsonObject.has("someNonField"));
    }
@@ -335,7 +337,7 @@ class QJavalinApiHandlerTest extends BaseTest
    @Test
    void testFieldDifferencesBetweenApis() throws QException
    {
-      insertPersonRecord(1, "Homer", "Simpson", LocalDate.of(1970, Month.JANUARY, 1));
+      insertPersonRecord(1, "Homer", "Simpson", qRecord -> qRecord.withValue("birthDate", LocalDate.of(1970, Month.JANUARY, 1)));
 
       /////////////////////////////////////////////////////////////
       // on the main api, birthDate has been renamed to birthDay //
@@ -364,7 +366,7 @@ class QJavalinApiHandlerTest extends BaseTest
    @Test
    void testQuery200SomethingFound() throws QException
    {
-      insertPersonRecord(1, "Homer", "Simpson");
+      insertPersonRecord(1, "Homer", "Simpson", qRecord -> qRecord.withValue("photo", "12345".getBytes()));
 
       HttpResponse<String> response = Unirest.get(BASE_URL + "/api/" + VERSION + "/person/query").asString();
       assertEquals(HttpStatus.OK_200, response.getStatus());
@@ -378,6 +380,7 @@ class QJavalinApiHandlerTest extends BaseTest
       assertEquals(1, jsonObject.getInt("id"));
       assertEquals("Homer", jsonObject.getString("firstName"));
       assertEquals("Simpson", jsonObject.getString("lastName"));
+      assertEquals("MTIzNDU=", jsonObject.getString("photo")); // base64 of "12345".getBytes()
       assertTrue(jsonObject.isNull("noOfShoes"));
       assertFalse(jsonObject.has("someNonField"));
    }
@@ -470,8 +473,11 @@ class QJavalinApiHandlerTest extends BaseTest
       assertPersonQueryFindsFirstNames(List.of(), "noOfShoes=!EMPTY");
       assertPersonQueryFindsFirstNames(List.of("Homer", "Marge", "Bart", "Lisa", "Maggie"), "id=!EMPTY&orderBy=id");
       assertPersonQueryFindsFirstNames(List.of(), "id=EMPTY");
+      assertPersonQueryFindsFirstNames(List.of("Homer", "Marge", "Bart", "Lisa", "Maggie"), "photo=EMPTY&orderBy=id");
+      assertPersonQueryFindsFirstNames(List.of(), "photo=!EMPTY");
 
       assertError("Unexpected value after operator EMPTY for field id", BASE_URL + "/api/" + VERSION + "/person/query?id=EMPTY 3");
+      assertError("Operator = may not be used for field photo (blob fields only support operators EMPTY or !EMPTY)", BASE_URL + "/api/" + VERSION + "/person/query?photo=ABCD");
    }
 
 
@@ -547,7 +553,7 @@ class QJavalinApiHandlerTest extends BaseTest
    {
       HttpResponse<String> response = Unirest.post(BASE_URL + "/api/" + VERSION + "/person/")
          .body("""
-            {"firstName": "Moe"}
+            {"firstName": "Moe", "photo": "MTIzNDU="}
             """)
          .asString();
       assertEquals(HttpStatus.CREATED_201, response.getStatus());
@@ -556,6 +562,7 @@ class QJavalinApiHandlerTest extends BaseTest
 
       QRecord record = getRecord(TestUtils.TABLE_NAME_PERSON, 1);
       assertEquals("Moe", record.getValueString("firstName"));
+      assertArrayEquals("12345".getBytes(), record.getValueByteArray("photo"));
    }
 
 
@@ -1293,6 +1300,7 @@ class QJavalinApiHandlerTest extends BaseTest
       getInput.setTableName(tableName);
       getInput.setPrimaryKey(id);
       getInput.setIncludeAssociations(true);
+      getInput.setShouldFetchHeavyFields(true);
       GetOutput getOutput = new GetAction().execute(getInput);
       QRecord   record    = getOutput.getRecord();
       return record;