From 614aead348dbc93ece5a919986c77cdaf8d937f8 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Mon, 15 May 2023 12:41:22 -0500
Subject: [PATCH 01/11] add more support for byte[]/BLOB field type; checkpoint

---
 .../core/model/data/QRecordEntity.java        |  3 ++-
 .../core/model/data/QRecordEntityField.java   |  5 +++++
 .../backend/core/model/data/QRecordEnum.java  |  3 ++-
 .../model/metadata/fields/QFieldType.java     |  4 ++++
 .../templates/ConvertHtmlToPdfActionTest.java | 19 +++++++++++++++----
 5 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java
index ee6827d1..138ed8f5 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java
@@ -253,7 +253,8 @@ public abstract class QRecordEntity
          || returnType.equals(BigDecimal.class)
          || returnType.equals(Instant.class)
          || returnType.equals(LocalDate.class)
-         || returnType.equals(LocalTime.class));
+         || returnType.equals(LocalTime.class)
+         || returnType.equals(byte[].class));
       /////////////////////////////////////////////
       // note - this list has implications upon: //
       // - QFieldType.fromClass                  //
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java
index 510308c0..a9999b25 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityField.java
@@ -165,6 +165,11 @@ public class QRecordEntityField
          {
             return (ValueUtils.getValueAsLocalTime(value));
          }
+
+         if(type.equals(byte[].class))
+         {
+            return (ValueUtils.getValueAsByteArray(value));
+         }
       }
       catch(Exception e)
       {
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java
index 83eea005..f5530b3b 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEnum.java
@@ -167,7 +167,8 @@ public interface QRecordEnum
          || returnType.equals(BigDecimal.class)
          || returnType.equals(Instant.class)
          || returnType.equals(LocalDate.class)
-         || returnType.equals(LocalTime.class));
+         || returnType.equals(LocalTime.class)
+         || returnType.equals(byte[].class));
       /////////////////////////////////////////////
       // note - this list has implications upon: //
       // - QFieldType.fromClass                  //
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java
index 367a4960..8971a846 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java
@@ -85,6 +85,10 @@ public enum QFieldType
       {
          return (BOOLEAN);
       }
+      if(c.equals(byte[].class))
+      {
+         return (BLOB);
+      }
 
       throw (new QException("Unrecognized class [" + c + "]"));
    }
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java
index a145a196..62393cf8 100644
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java
@@ -26,11 +26,15 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.file.Path;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.templates.ConvertHtmlToPdfInput;
+import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateInput;
+import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateOutput;
+import com.kingsrook.qqq.backend.core.model.templates.TemplateType;
 import org.junit.jupiter.api.Test;
 
 
@@ -46,10 +50,10 @@ class ConvertHtmlToPdfActionTest extends BaseTest
    @Test
    void test() throws QException, IOException
    {
-      QInstance             instance = QContext.getQInstance();
-      ConvertHtmlToPdfInput input    = new ConvertHtmlToPdfInput();
+      QInstance instance = QContext.getQInstance();
 
-      input.setHtml("""
+      RenderTemplateInput renderTemplateInput = new RenderTemplateInput();
+      renderTemplateInput.setCode("""
          <html>
              <head>
                  <style>
@@ -70,7 +74,7 @@ class ConvertHtmlToPdfActionTest extends BaseTest
                  <div class="center_div">
                      <h1>
                         <img src="images/qqq-logo-2.png" width=50>
-                        Hello QQQ!
+                        Hello, $name
                      </h1>
                      <div class="myclass">
                          <p>This is a test of converting HTML to PDF!!</p>
@@ -80,6 +84,12 @@ class ConvertHtmlToPdfActionTest extends BaseTest
              </body>
          </html>
          """);
+      renderTemplateInput.setTemplateType(TemplateType.VELOCITY);
+      renderTemplateInput.setContext(Map.of("name", "Darin"));
+      RenderTemplateOutput renderTemplateOutput = new RenderTemplateAction().execute(renderTemplateInput);
+
+      ConvertHtmlToPdfInput input = new ConvertHtmlToPdfInput();
+      input.setHtml(renderTemplateOutput.getResult());
 
       OutputStream outputStream = new FileOutputStream("/tmp/file.pdf");
       input.setOutputStream(outputStream);
@@ -97,6 +107,7 @@ class ConvertHtmlToPdfActionTest extends BaseTest
       /////////////////////////////////////////////////////////////////////////
       // for local dev on a mac, turn this on to auto-open the generated PDF //
       /////////////////////////////////////////////////////////////////////////
+      // todo not commit
       // Runtime.getRuntime().exec(new String[] { "/usr/bin/open", "/tmp/file.pdf" });
    }
 

From 36efc4c2d909663584be0a3293ba541f7e0287e5 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Wed, 24 May 2023 17:07:19 -0500
Subject: [PATCH 02/11] More BLOB; FILE_DOWNLOAD adornment; javalin field
 download endpoint

---
 .../core/instances/QInstanceValidator.java    |  57 +++++-
 .../model/metadata/fields/AdornmentType.java  |  22 +++
 .../frontend/QFrontendFieldMetaData.java      |  13 ++
 .../instances/QInstanceValidatorTest.java     |  31 ++++
 .../rdbms/actions/AbstractRDBMSAction.java    |   4 +
 .../rdbms/actions/RDBMSInsertAction.java      |   3 +-
 .../javalin/QJavalinImplementation.java       | 175 +++++++++++++++++-
 7 files changed, 301 insertions(+), 4 deletions(-)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
index 64f85dde..111d2dd2 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.instances;
 
 
+import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
@@ -52,6 +53,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QMiddlewareInstanceMetaData
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -82,6 +85,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheOf;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
 
 /*******************************************************************************
@@ -418,7 +422,7 @@ public class QInstanceValidator
             {
                table.getFields().forEach((fieldName, field) ->
                {
-                  validateTableField(qInstance, tableName, fieldName, field);
+                  validateTableField(qInstance, tableName, fieldName, table, field);
                });
             }
 
@@ -659,7 +663,7 @@ public class QInstanceValidator
    /*******************************************************************************
     **
     *******************************************************************************/
-   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QFieldMetaData field)
+   private void validateTableField(QInstance qInstance, String tableName, String fieldName, QTableMetaData table, QFieldMetaData field)
    {
       assertCondition(Objects.equals(fieldName, field.getName()),
          "Inconsistent naming in table " + tableName + " for field " + fieldName + "/" + field.getName() + ".");
@@ -701,6 +705,55 @@ public class QInstanceValidator
          assertCondition(fieldSecurityLock.getDefaultBehavior() != null, prefix + "has a fieldSecurityLock that is missing a defaultBehavior");
          assertCondition(CollectionUtils.nullSafeHasContents(fieldSecurityLock.getOverrideValues()), prefix + "has a fieldSecurityLock that is missing overrideValues");
       }
+
+      for(FieldAdornment adornment : CollectionUtils.nonNullList(field.getAdornments()))
+      {
+         Map<String, Serializable> adornmentValues = CollectionUtils.nonNullMap(adornment.getValues());
+         if(assertCondition(adornment.getType() != null, prefix + "has an adornment that is missing a type"))
+         {
+            String adornmentPrefix = prefix.trim() + ", " + adornment.getType() + " adornment ";
+            switch(adornment.getType())
+            {
+               case SIZE ->
+               {
+                  String width = ValueUtils.getValueAsString(adornmentValues.get("width"));
+                  if(assertCondition(StringUtils.hasContent(width), adornmentPrefix + "is missing a width value"))
+                  {
+                     assertNoException(() -> AdornmentType.Size.valueOf(width.toUpperCase()), adornmentPrefix + "has an unrecognized width value [" + width + "]");
+                  }
+               }
+               case FILE_DOWNLOAD ->
+               {
+                  String fileNameField = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+                  if(StringUtils.hasContent(fileNameField)) // file name isn't required - but if given, must be a field on the table.
+                  {
+                     assertNoException(() -> table.getField(fileNameField), adornmentPrefix + "specifies an unrecognized fileNameField [" + fileNameField + "]");
+                  }
+
+                  if(adornmentValues.containsKey(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS))
+                  {
+                     try
+                     {
+                        @SuppressWarnings("unchecked")
+                        List<String> formatFieldNames = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                        for(String formatFieldName : CollectionUtils.nonNullList(formatFieldNames))
+                        {
+                           assertNoException(() -> table.getField(formatFieldName), adornmentPrefix + "specifies an unrecognized field name in fileNameFormatFields [" + formatFieldName + "]");
+                        }
+                     }
+                     catch(Exception e)
+                     {
+                        errors.add(adornmentPrefix + "fileNameFormatFields could not be accessed (is it a List<String>?)");
+                     }
+                  }
+               }
+               default ->
+               {
+                  // no validations by default
+               }
+            }
+         }
+      }
    }
 
 
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java
index e46f4c2c..ba78ceef 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java
@@ -40,6 +40,7 @@ public enum AdornmentType
    CODE_EDITOR,
    RENDER_HTML,
    REVEAL,
+   FILE_DOWNLOAD,
    ERROR;
    //////////////////////////////////////////////////////////////////////////
    // keep these values in sync with AdornmentType.ts in qqq-frontend-core //
@@ -58,6 +59,26 @@ public enum AdornmentType
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public interface FileDownloadValues
+   {
+      String FILE_NAME_FIELD   = "fileNameField";
+      String DEFAULT_EXTENSION = "defaultExtension";
+      String DEFAULT_MIME_TYPE = "defaultMimeType";
+
+      ////////////////////////////////////////////////////
+      // use these two together, as in:                 //
+      // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" //
+      // FILE_NAME_FORMAT_FIELDS = "orderId"            //
+      ////////////////////////////////////////////////////
+      String FILE_NAME_FORMAT        = "fileNameFormat";
+      String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields";
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -112,6 +133,7 @@ public enum AdornmentType
       XSMALL,
       SMALL,
       MEDIUM,
+      MEDLARGE,
       LARGE,
       XLARGE;
 
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java
index d4414208..abb79f9c 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java
@@ -43,6 +43,7 @@ public class QFrontendFieldMetaData
    private QFieldType type;
    private boolean    isRequired;
    private boolean    isEditable;
+   private boolean    isHeavy;
    private String     possibleValueSourceName;
    private String     displayFormat;
 
@@ -64,6 +65,7 @@ public class QFrontendFieldMetaData
       this.type = fieldMetaData.getType();
       this.isRequired = fieldMetaData.getIsRequired();
       this.isEditable = fieldMetaData.getIsEditable();
+      this.isHeavy = fieldMetaData.getIsHeavy();
       this.possibleValueSourceName = fieldMetaData.getPossibleValueSourceName();
       this.displayFormat = fieldMetaData.getDisplayFormat();
       this.adornments = fieldMetaData.getAdornments();
@@ -126,6 +128,17 @@ public class QFrontendFieldMetaData
 
 
 
+   /*******************************************************************************
+    ** Getter for isHeavy
+    **
+    *******************************************************************************/
+   public boolean getIsHeavy()
+   {
+      return isHeavy;
+   }
+
+
+
    /*******************************************************************************
     ** Getter for displayFormat
     **
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
index e6e50a68..129aeddb 100644
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java
@@ -47,6 +47,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
@@ -1732,6 +1734,35 @@ class QInstanceValidatorTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testFieldAdornments()
+   {
+      Function<QInstance, QFieldMetaData> fieldExtractor = qInstance -> qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("firstName");
+
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment())), "adornment that is missing a type");
+      assertValidationSuccess((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment().withType(AdornmentType.REVEAL))));
+
+      ////////////////////////////////
+      // type-specific value checks //
+      ////////////////////////////////
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.SIZE))), "missing a width value");
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.SIZE).withValue("width", "foo"))), "unrecognized width value");
+      assertValidationSuccess((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.SIZE).withValue("width", AdornmentType.Size.MEDIUM))));
+      assertValidationSuccess((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(AdornmentType.Size.SMALL.toAdornment())));
+
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD).withValue("fileNameField", "foo"))), "unrecognized fileNameField [foo]");
+      assertValidationSuccess((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD).withValue("fileNameField", "lastName"))));
+
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD).withValue("fileNameFormatFields", "foo"))), "fileNameFormatFields could not be accessed");
+      assertValidationFailureReasons((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD).withValue("fileNameFormatFields", new ArrayList<>(List.of("foo"))))), "unrecognized field name in fileNameFormatFields [foo]");
+      assertValidationSuccess((qInstance -> fieldExtractor.apply(qInstance).withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD).withValue("fileNameFormatFields", new ArrayList<>(List.of("lastName"))))));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
index 6145e8ef..f6386885 100644
--- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
+++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
@@ -872,6 +872,10 @@ public abstract class AbstractRDBMSAction implements QActionInterface
          {
             return (QueryManager.getBoolean(resultSet, i));
          }
+         case BLOB:
+         {
+            return (QueryManager.getByteArray(resultSet, i));
+         }
          default:
          {
             throw new IllegalStateException("Unexpected field type: " + type);
diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
index 91465e49..dad6d6c0 100644
--- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
+++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
@@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
@@ -58,7 +59,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
 
       if(CollectionUtils.nullSafeIsEmpty(insertInput.getRecords()))
       {
-         LOG.debug("Insert request called with 0 records.  Returning with no-op");
+         LOG.debug("Insert request called with 0 records.  Returning with no-op", logPair("tableName", insertInput.getTableName()));
          rs.setRecords(new ArrayList<>());
          return (rs);
       }
diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
index be89714c..665047b3 100644
--- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
+++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
@@ -31,6 +31,7 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -53,6 +54,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction;
 import com.kingsrook.qqq.backend.core.adapters.QInstanceAdapter;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -95,7 +97,10 @@ import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -354,6 +359,9 @@ public class QJavalinImplementation
                put("", QJavalinImplementation::dataUpdate); // todo - want different semantics??
                delete("", QJavalinImplementation::dataDelete);
 
+               get("/{fieldName}/{filename}", QJavalinImplementation::dataDownloadRecordField);
+               post("/{fieldName}/{filename}", QJavalinImplementation::dataDownloadRecordField);
+
                QJavalinScriptsHandler.defineRecordRoutes();
             });
          });
@@ -733,6 +741,7 @@ public class QJavalinImplementation
          getInput.setTableName(tableName);
          getInput.setShouldGenerateDisplayValues(true);
          getInput.setShouldTranslatePossibleValues(true);
+         getInput.setShouldFetchHeavyFields(true);
 
          PermissionsHelper.checkTablePermissionThrowing(getInput, TablePermissionSubType.READ);
 
@@ -744,6 +753,149 @@ public class QJavalinImplementation
          GetAction getAction = new GetAction();
          GetOutput getOutput = getAction.execute(getInput);
 
+         ///////////////////////////////////////////////////////
+         // throw a not found error if the record isn't found //
+         ///////////////////////////////////////////////////////
+         QRecord record = getOutput.getRecord();
+         if(record == null)
+         {
+            throw (new QNotFoundException("Could not find " + table.getLabel() + " with "
+               + table.getFields().get(table.getPrimaryKeyField()).getLabel() + " of " + primaryKey));
+         }
+
+         setBlobValuesToDownloadUrls(table, List.of(record));
+
+         QJavalinAccessLogger.logEndSuccess();
+         context.result(JsonUtils.toJson(record));
+      }
+      catch(Exception e)
+      {
+         QJavalinAccessLogger.logEndFail(e);
+         handleException(context, e);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** For any BLOB type fields in the list of records, change their value to
+    ** the URL where they can be downloaded, and set their display value to a file name.
+    *******************************************************************************/
+   private static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
+   {
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getType().equals(QFieldType.BLOB))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // file name comes from:                                                                                            //
+            // if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
+            // - unless it was empty - then we do the "default thing":                                                          //
+            // else - the "default thing" is:                                                                                   //
+            // - tableLabel primaryKey fieldLabel                                                                               //
+            // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornments().stream().filter(a -> a.getType().equals(AdornmentType.FILE_DOWNLOAD)).findFirst();
+            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
+
+            if(fileDownloadAdornment.isPresent())
+            {
+               adornmentValues = fileDownloadAdornment.get().getValues();
+            }
+
+            String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+            String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
+            String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
+
+            for(QRecord record : records)
+            {
+               Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+               String       fileName   = null;
+
+               //////////////////////////////////////////////////
+               // try to make file name from the fileNameField //
+               //////////////////////////////////////////////////
+               if(StringUtils.hasContent(fileNameField))
+               {
+                  fileName = record.getValueString(fileNameField);
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  if(StringUtils.hasContent(fileNameFormat))
+                  {
+                     @SuppressWarnings("unchecked") // instance validation should make this safe!
+                     List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, fileNameFormatFields);
+                  }
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  //////////////////////////////////
+                  // make default name if missing //
+                  //////////////////////////////////
+                  fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
+
+                  if(StringUtils.hasContent(defaultExtension))
+                  {
+                     //////////////////////////////////////////
+                     // add default extension if we have one //
+                     //////////////////////////////////////////
+                     fileName += "." + defaultExtension;
+                  }
+               }
+
+               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               record.setDisplayValue(field.getName(), fileName);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void dataDownloadRecordField(Context context)
+   {
+      String tableName  = context.pathParam("table");
+      String primaryKey = context.pathParam("primaryKey");
+      String fieldName  = context.pathParam("fieldName");
+      String filename   = context.pathParam("filename");
+
+      try
+      {
+         QTableMetaData table    = qInstance.getTable(tableName);
+         GetInput       getInput = new GetInput();
+
+         setupSession(context, getInput);
+         QJavalinAccessLogger.logStart("downloadRecordField", logPair("table", tableName), logPair("primaryKey", primaryKey), logPair("fieldName", fieldName));
+
+         ////////////////////////////////////////////
+         // validate field name - 404 if not found //
+         ////////////////////////////////////////////
+         QFieldMetaData fieldMetaData;
+         try
+         {
+            fieldMetaData = table.getField(fieldName);
+         }
+         catch(Exception e)
+         {
+            throw (new QNotFoundException("Could not find field named " + fieldName + " on table " + tableName));
+         }
+
+         getInput.setTableName(tableName);
+         getInput.setShouldFetchHeavyFields(true);
+
+         PermissionsHelper.checkTablePermissionThrowing(getInput, TablePermissionSubType.READ);
+
+         getInput.setPrimaryKey(primaryKey);
+
+         GetAction getAction = new GetAction();
+         GetOutput getOutput = getAction.execute(getInput);
+
          ///////////////////////////////////////////////////////
          // throw a not found error if the record isn't found //
          ///////////////////////////////////////////////////////
@@ -753,8 +905,27 @@ public class QJavalinImplementation
                + table.getFields().get(table.getPrimaryKeyField()).getLabel() + " of " + primaryKey));
          }
 
+         String                   mimeType              = null;
+         Optional<FieldAdornment> fileDownloadAdornment = fieldMetaData.getAdornments().stream().filter(a -> a.getType().equals(AdornmentType.FILE_DOWNLOAD)).findFirst();
+         if(fileDownloadAdornment.isPresent())
+         {
+            Map<String, Serializable> values = fileDownloadAdornment.get().getValues();
+            mimeType = ValueUtils.getValueAsString(values.get(AdornmentType.FileDownloadValues.DEFAULT_MIME_TYPE));
+         }
+
+         if(mimeType != null)
+         {
+            context.contentType(mimeType);
+         }
+
+         if(context.queryParamMap().containsKey("download") || context.formParamMap().containsKey("download"))
+         {
+            context.header("Content-Disposition", "attachment; filename=" + filename);
+         }
+
+         context.result(getOutput.getRecord().getValueByteArray(fieldName));
+
          QJavalinAccessLogger.logEndSuccess();
-         context.result(JsonUtils.toJson(getOutput.getRecord()));
       }
       catch(Exception e)
       {
@@ -888,6 +1059,8 @@ public class QJavalinImplementation
          QueryAction queryAction = new QueryAction();
          QueryOutput queryOutput = queryAction.execute(queryInput);
 
+         setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
+
          QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS), logPairIfSlow("joins", queryJoins, SLOW_LOG_THRESHOLD_MS));
          context.result(JsonUtils.toJson(queryOutput));
       }

From a75530b466c0b78b59042be309e2f1b800746d31 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Tue, 30 May 2023 10:13:04 -0500
Subject: [PATCH 03/11] Updates for more heavy-field handling

---
 .../core/actions/audits/DMLAuditAction.java   |  59 ++++--
 .../core/actions/reporting/ExportAction.java  |  15 +-
 .../core/actions/tables/QueryAction.java      |  10 +
 .../core/instances/QInstanceEnricher.java     |   2 +-
 .../qqq/backend/core/model/data/QRecord.java  |   6 +-
 .../model/metadata/fields/FieldAdornment.java |  18 ++
 .../model/metadata/fields/QFieldMetaData.java |  24 +++
 .../columnstats/ColumnStatsStep.java          |   5 +
 .../rdbms/actions/RDBMSQueryAction.java       |  78 ++++++--
 .../qqq/backend/module/rdbms/TestUtils.java   |   1 +
 .../rdbms/actions/RDBMSQueryActionTest.java   |  33 ++++
 .../test/resources/prime-test-database.sql    |  13 +-
 .../javalin/QJavalinImplementation.java       | 178 ++++++++++++++----
 13 files changed, 362 insertions(+), 80 deletions(-)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
index 8c2f747e..b532fecd 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java
@@ -187,32 +187,57 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
                         continue;
                      }
 
-                     String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                     detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
-                     detailRecord.withValue("newValue", formattedValue);
+                     if(field.getType().equals(QFieldType.BLOB))
+                     {
+                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+                     }
+                     else
+                     {
+                        String formattedValue = getFormattedValueForAuditDetail(record, fieldName, field, value);
+                        detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formattedValue);
+                        detailRecord.withValue("newValue", formattedValue);
+                     }
                   }
                   else
                   {
                      if(!Objects.equals(oldValue, value))
                      {
-                        String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
-                        String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
-
-                        if(oldValue == null)
+                        if(field.getType().equals(QFieldType.BLOB))
                         {
-                           detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("newValue", formattedValue);
-                        }
-                        else if(value == null)
-                        {
-                           detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
-                           detailRecord.withValue("oldValue", formattedOldValue);
+                           if(oldValue == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel());
+                           }
+                           else if(value == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Removed " + field.getLabel());
+                           }
+                           else
+                           {
+                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel());
+                           }
                         }
                         else
                         {
-                           detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
-                           detailRecord.withValue("oldValue", formattedOldValue);
-                           detailRecord.withValue("newValue", formattedValue);
+                           String formattedValue    = getFormattedValueForAuditDetail(record, fieldName, field, value);
+                           String formattedOldValue = getFormattedValueForAuditDetail(oldRecord, fieldName, field, oldValue);
+
+                           if(oldValue == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Set " + field.getLabel() + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                              detailRecord.withValue("newValue", formattedValue);
+                           }
+                           else if(value == null)
+                           {
+                              detailRecord = new QRecord().withValue("message", "Removed " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " from " + field.getLabel());
+                              detailRecord.withValue("oldValue", formattedOldValue);
+                           }
+                           else
+                           {
+                              detailRecord = new QRecord().withValue("message", "Changed " + field.getLabel() + " from " + formatFormattedValueForDetailMessage(field, formattedOldValue) + " to " + formatFormattedValueForDetailMessage(field, formattedValue));
+                              detailRecord.withValue("oldValue", formattedOldValue);
+                              detailRecord.withValue("newValue", formattedValue);
+                           }
                         }
                      }
                   }
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java
index 2ca95347..406c1a41 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java
@@ -386,13 +386,22 @@ public class ExportAction
          fieldList = new ArrayList<>(table.getFields().values());
       }
 
-      //////////////////////////////////////////
-      // add fields for possible value labels //
-      //////////////////////////////////////////
       List<QFieldMetaData> returnList = new ArrayList<>();
       for(QFieldMetaData field : fieldList)
       {
+         /////////////////////////////////////////////////////////////////////////////////////////
+         // skip heavy fields.  they aren't fetched, and we generally think we don't want them. //
+         /////////////////////////////////////////////////////////////////////////////////////////
+         if(field.getIsHeavy())
+         {
+            continue;
+         }
+
          returnList.add(field);
+
+         //////////////////////////////////////////
+         // add fields for possible value labels //
+         //////////////////////////////////////////
          if(StringUtils.hasContent(field.getPossibleValueSourceName()))
          {
             returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name"));
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
index 0bf8e33c..5ba0002b 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
@@ -81,6 +81,16 @@ public class QueryAction
    {
       ActionHelper.validateSession(queryInput);
 
+      if(queryInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in query input"));
+      }
+
+      if(queryInput.getTable() == null)
+      {
+         throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance"));
+      }
+
       postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, queryInput.getTable(), TableCustomizers.POST_QUERY_RECORD.getRole());
       this.queryInput = queryInput;
 
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
index 38213836..465434bc 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
@@ -735,7 +735,7 @@ public class QInstanceEnricher
       QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData()
          .withName("upload")
          .withLabel("Upload File")
-         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true))
+         .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true))
          .withComponent(new QFrontendComponentMetaData()
             .withType(QComponentType.HELP_TEXT)
             .withValue("previewText", "file upload instructions")
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
index 00fbeb47..f826e6ff 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java
@@ -69,7 +69,11 @@ public class QRecord implements Serializable
 
    private Map<String, List<QRecord>> associatedRecords = new HashMap<>();
 
-   public static final String BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT = "jsonSourceObject";
+   ////////////////////////////////////////////////
+   // well-known keys for the backendDetails map //
+   ////////////////////////////////////////////////
+   public static final String BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT  = "jsonSourceObject"; // String of JSON
+   public static final String BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS = "heavyFieldLengths"; // Map<fieldName, length>
 
 
 
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java
index 6bf68f48..2ca70af1 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java
@@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields;
 import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.utils.Pair;
 
 
@@ -116,6 +118,22 @@ public class FieldAdornment
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @JsonIgnore
+   public Optional<Serializable> getValue(String key)
+   {
+      if(key != null && values != null)
+      {
+         return (Optional.ofNullable(values.get(key)));
+      }
+
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     ** Setter for values
     **
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java
index 298ffaf4..23f5b992 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.github.hervian.reflection.Fun;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -534,6 +535,29 @@ public class QFieldMetaData implements Cloneable
 
 
 
+   /*******************************************************************************
+    ** Getter for adornments
+    **
+    *******************************************************************************/
+   @JsonIgnore
+   public Optional<FieldAdornment> getAdornment(AdornmentType adornmentType)
+   {
+      if(adornmentType != null && adornments != null)
+      {
+         for(FieldAdornment adornment : adornments)
+         {
+            if(adornmentType.equals(adornment.getType()))
+            {
+               return Optional.of((adornment));
+            }
+         }
+      }
+
+      return (Optional.empty());
+   }
+
+
+
    /*******************************************************************************
     ** Setter for adornments
     **
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
index dee5e4d1..9cfe210e 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
@@ -134,6 +134,11 @@ public class ColumnStatsStep implements BackendStep
             throw (new QException("Could not find field by name: " + fieldName));
          }
 
+         if(field.getType().equals(QFieldType.BLOB))
+         {
+            throw (new QException("Column stats are not supported for this field's data type."));
+         }
+
          ////////////////////////////////////////////
          // do a count query grouped by this field //
          ////////////////////////////////////////////
diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
index c82a18d4..968901b0 100644
--- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
+++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
@@ -29,8 +29,10 @@ import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -43,8 +45,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.Pair;
 import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
 import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
 
@@ -112,9 +116,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          ////////////////////////////////////////////////////////////////////////////
          // build the list of fields that will be processed in the result-set loop //
          ////////////////////////////////////////////////////////////////////////////
-         List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream()
-            .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
-            .toList());
+         List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values().stream().toList());
          for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins()))
          {
             if(queryJoin.getSelect())
@@ -123,10 +125,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
                String         tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
                for(QFieldMetaData joinField : joinTable.getFields().values())
                {
-                  if(filterOutHeavyFieldsIfNeeded(joinField, queryInput.getShouldFetchHeavyFields()))
-                  {
-                     fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
-                  }
+                  fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName()));
                }
             }
          }
@@ -153,9 +152,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
                   for(int i = 1; i <= metaData.getColumnCount(); i++)
                   {
-                     QFieldMetaData qFieldMetaData = fieldList.get(i - 1);
-                     Serializable   value          = getFieldValueFromResultSet(qFieldMetaData, resultSet, i);
-                     values.put(qFieldMetaData.getName(), value);
+                     QFieldMetaData field = fieldList.get(i - 1);
+
+                     if(!queryInput.getShouldFetchHeavyFields() && field.getIsHeavy())
+                     {
+                        ///////////////////////////////////////////////////////////////////////////////////
+                        // if this is a non-fetched heavy field (e.g., we just fetched its length), then //
+                        // get the value here as an INTEGER, not a BLOB or whatever the field would be   //
+                        ///////////////////////////////////////////////////////////////////////////////////
+                        Serializable fieldLength = getFieldValueFromResultSet(QFieldType.INTEGER, resultSet, i);
+                        setHeavyFieldLengthInRecordBackendDetails(record, field, fieldLength);
+                     }
+                     else
+                     {
+                        Serializable value = getFieldValueFromResultSet(field, resultSet, i);
+                        values.put(field.getName(), value);
+                     }
                   }
 
                   queryOutput.addRecord(record);
@@ -195,6 +207,27 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   private static void setHeavyFieldLengthInRecordBackendDetails(QRecord record, QFieldMetaData field, Serializable fieldLength)
+   {
+      if(record.getBackendDetails() == null)
+      {
+         record.setBackendDetails(new HashMap<>());
+      }
+
+      if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null)
+      {
+         record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>());
+      }
+
+      ((Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(field.getName(), fieldLength);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -210,8 +243,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
       List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
       String columns = fieldList.stream()
-         .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
-         .map(field -> escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field)))
+         .map(field -> Pair.of(field, escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field))))
+         .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
          .collect(Collectors.joining(", "));
       StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
 
@@ -229,7 +262,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
             List<QFieldMetaData> joinFieldList = new ArrayList<>(joinTable.getFields().values());
             String joinColumns = joinFieldList.stream()
                .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
-               .map(field -> escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field)))
+               .map(field -> Pair.of(field, escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field))))
+               .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields()))
                .collect(Collectors.joining(", "));
             rs.append(", ").append(joinColumns);
          }
@@ -254,6 +288,24 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
 
 
+   /*******************************************************************************
+    ** if we're not fetching heavy fields, instead just get their length.  this
+    ** method wraps the field 'sql name' (e.g., column_name or table_name.column_name)
+    ** with the LENGTH() function, if needed.
+    *******************************************************************************/
+   private String wrapHeavyFieldsWithLengthFunctionIfNeeded(Pair<QFieldMetaData, String> fieldAndSqlName, boolean shouldFetchHeavyFields)
+   {
+      QFieldMetaData field   = fieldAndSqlName.getA();
+      String         sqlName = fieldAndSqlName.getB();
+      if(!shouldFetchHeavyFields && field.getIsHeavy())
+      {
+         return ("LENGTH(" + sqlName + ")");
+      }
+      return (sqlName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
index a2479159..2a9b32f4 100644
--- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
+++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
@@ -167,6 +167,7 @@ public class TestUtils
          .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed"))
          .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary"))
          .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked"))
+         .withField(new QFieldMetaData("homeTown", QFieldType.STRING).withBackendName("home_town"))
          .withBackendDetails(new RDBMSTableBackendDetails()
             .withTableName("person"));
    }
diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java
index 3d49f858..4e49d82a 100644
--- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java
+++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java
@@ -22,10 +22,12 @@
 package com.kingsrook.qqq.backend.module.rdbms.actions;
 
 
+import java.io.Serializable;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -1663,4 +1665,35 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          .hasSize(1);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testHeavyFields() throws QException
+   {
+      //////////////////////////////////////////////////////////
+      // set homeTown field as heavy - so it won't be fetched //
+      //////////////////////////////////////////////////////////
+      QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON)
+         .getField("homeTown")
+         .withIsHeavy(true);
+
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_PERSON);
+      List<QRecord> records = new QueryAction().execute(queryInput).getRecords();
+      assertThat(records).describedAs("No records should have the heavy homeTown field set").noneMatch(r -> r.getValue("homeTown") != null);
+      assertThat(records).describedAs("Some records should have a homeTown length backend detail set").anyMatch(r -> ((Map<String, Serializable>) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") != null);
+      assertThat(records).describedAs("Some records should have a null homeTown length backend").anyMatch(r -> ((Map<String, Serializable>) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") == null);
+
+      //////////////////////////////////////////////
+      // re-do the query, requesting heavy fields //
+      //////////////////////////////////////////////
+      queryInput.setShouldFetchHeavyFields(true);
+      records = new QueryAction().execute(queryInput).getRecords();
+      assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null);
+
+   }
+
 }
diff --git a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql
index 92c69194..87620a4c 100644
--- a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql
+++ b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql
@@ -32,14 +32,15 @@ CREATE TABLE person
    email VARCHAR(250) NOT NULL,
    is_employed BOOLEAN,
    annual_salary DECIMAL(12,2),
-   days_worked INTEGER
+   days_worked INTEGER,
+   home_town VARCHAR(80)
 );
 
-INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27);
-INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124);
-INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0);
-INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99);
-INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232);
+INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27, 'Chester');
+INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124, 'Chester');
+INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0, 'Decatur');
+INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99, 'Texas');
+INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232, null);
 
 DROP TABLE IF EXISTS personal_id_card;
 CREATE TABLE personal_id_card
diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
index 665047b3..77aaf73a 100644
--- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
+++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.javalin;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
 import java.io.Serializable;
@@ -116,6 +117,7 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
 import io.javalin.Javalin;
 import io.javalin.apibuilder.EndpointGroup;
 import io.javalin.http.Context;
+import io.javalin.http.UploadedFile;
 import org.apache.commons.io.FileUtils;
 import org.eclipse.jetty.http.HttpStatus;
 import org.json.JSONArray;
@@ -619,38 +621,17 @@ public class QJavalinImplementation
          updateInput.setTableName(table);
 
          PermissionsHelper.checkTablePermissionThrowing(updateInput, TablePermissionSubType.EDIT);
+         QTableMetaData tableMetaData = qInstance.getTable(table);
+
+         QJavalinAccessLogger.logStart("update", logPair("table", table), logPair("primaryKey", primaryKey));
 
          List<QRecord> recordList = new ArrayList<>();
          QRecord       record     = new QRecord();
          record.setTableName(table);
          recordList.add(record);
 
-         Map<?, ?> map = context.bodyAsClass(Map.class);
-         for(Map.Entry<?, ?> entry : map.entrySet())
-         {
-            String fieldName = ValueUtils.getValueAsString(entry.getKey());
-            Object value     = entry.getValue();
-
-            if(StringUtils.hasContent(String.valueOf(value)))
-            {
-               record.setValue(fieldName, (Serializable) value);
-            }
-            else if("".equals(value))
-            {
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               // if frontend sent us an empty string - put a null in the record's value map.                   //
-               // this could potentially be changed to be type-specific (e.g., store an empty-string for STRING //
-               // fields, but null for INTEGER, etc) - but, who really wants empty-string in database anyway?   //
-               ///////////////////////////////////////////////////////////////////////////////////////////////////
-               record.setValue(fieldName, null);
-            }
-         }
-
-         QTableMetaData tableMetaData = qInstance.getTable(table);
          record.setValue(tableMetaData.getPrimaryKeyField(), primaryKey);
-
-         QJavalinAccessLogger.logStart("update", logPair("table", table), logPair("primaryKey", primaryKey));
-
+         setRecordValuesForInsertOrUpdate(context, tableMetaData, record);
          updateInput.setRecords(recordList);
 
          UpdateAction updateAction = new UpdateAction();
@@ -668,6 +649,87 @@ public class QJavalinImplementation
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setRecordValuesForInsertOrUpdate(Context context, QTableMetaData tableMetaData, QRecord record) throws IOException
+   {
+      /////////////////////////
+      // process form params //
+      /////////////////////////
+      for(Map.Entry<String, List<String>> formParam : context.formParamMap().entrySet())
+      {
+         String       fieldName = formParam.getKey();
+         List<String> values    = formParam.getValue();
+         if(CollectionUtils.nullSafeHasContents(values))
+         {
+            String value = values.get(0);
+            if(StringUtils.hasContent(value))
+            {
+               record.setValue(fieldName, value);
+            }
+            else
+            {
+               record.setValue(fieldName, null);
+            }
+         }
+         else
+         {
+            // is this ever hit?
+            record.setValue(fieldName, null);
+         }
+      }
+
+      ////////////////////////////
+      // process uploaded files //
+      ////////////////////////////
+      for(Map.Entry<String, List<UploadedFile>> entry : CollectionUtils.nonNullMap(context.uploadedFileMap()).entrySet())
+      {
+         String             fieldName     = entry.getKey();
+         List<UploadedFile> uploadedFiles = entry.getValue();
+         if(uploadedFiles.size() > 0)
+         {
+            UploadedFile uploadedFile = uploadedFiles.get(0);
+            try(InputStream content = uploadedFile.content())
+            {
+               record.setValue(fieldName, content.readAllBytes());
+            }
+
+            QFieldMetaData blobField = tableMetaData.getField(fieldName);
+            blobField.getAdornment(AdornmentType.FILE_DOWNLOAD).ifPresent(adornment ->
+            {
+               adornment.getValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD).ifPresent(fileNameFieldName ->
+               {
+                  record.setValue(ValueUtils.getValueAsString(fileNameFieldName), uploadedFile.filename());
+               });
+            });
+         }
+      }
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      // if the record has any blob fields, and we're clearing them out (present in the values list, and set to null), //
+      // and they have a  file-name field associated with them, then also clear out that file-name field               //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      for(QFieldMetaData field : tableMetaData.getFields().values())
+      {
+         if(field.getType().equals(QFieldType.BLOB))
+         {
+            field.getAdornment(AdornmentType.FILE_DOWNLOAD).ifPresent(adornment ->
+            {
+               adornment.getValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD).ifPresent(fileNameFieldName ->
+               {
+                  if(record.getValues().containsKey(field.getName()) && record.getValue(field.getName()) == null)
+                  {
+                     record.setValue(ValueUtils.getValueAsString(fileNameFieldName), null);
+                  }
+               });
+            });
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -682,20 +744,13 @@ public class QJavalinImplementation
          QJavalinAccessLogger.logStart("insert", logPair("table", tableName));
 
          PermissionsHelper.checkTablePermissionThrowing(insertInput, TablePermissionSubType.INSERT);
+         QTableMetaData tableMetaData = qInstance.getTable(tableName);
 
          List<QRecord> recordList = new ArrayList<>();
          QRecord       record     = new QRecord();
          record.setTableName(tableName);
          recordList.add(record);
-
-         Map<?, ?> map = context.bodyAsClass(Map.class);
-         for(Map.Entry<?, ?> entry : map.entrySet())
-         {
-            if(StringUtils.hasContent(String.valueOf(entry.getValue())))
-            {
-               record.setValue(String.valueOf(entry.getKey()), (Serializable) entry.getValue());
-            }
-         }
+         setRecordValuesForInsertOrUpdate(context, tableMetaData, record);
          insertInput.setRecords(recordList);
 
          InsertAction insertAction = new InsertAction();
@@ -703,14 +758,14 @@ public class QJavalinImplementation
 
          if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
          {
-            throw (new QUserFacingException("Error inserting " + qInstance.getTable(tableName).getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0)));
+            throw (new QUserFacingException("Error inserting " + tableMetaData.getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0)));
          }
          if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getWarnings()))
          {
-            throw (new QUserFacingException("Warning inserting " + qInstance.getTable(tableName).getLabel() + ": " + insertOutput.getRecords().get(0).getWarnings().get(0)));
+            throw (new QUserFacingException("Warning inserting " + tableMetaData.getLabel() + ": " + insertOutput.getRecords().get(0).getWarnings().get(0)));
          }
 
-         QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (insertOutput.getRecords().get(0).getValue(qInstance.getTable(tableName).getPrimaryKeyField()))));
+         QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (insertOutput.getRecords().get(0).getValue(tableMetaData.getPrimaryKeyField()))));
          context.result(JsonUtils.toJson(insertOutput));
       }
       catch(Exception e)
@@ -795,7 +850,7 @@ public class QJavalinImplementation
             // - tableLabel primaryKey fieldLabel                                                                               //
             // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornments().stream().filter(a -> a.getType().equals(AdornmentType.FILE_DOWNLOAD)).findFirst();
+            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
             Map<String, Serializable> adornmentValues       = Collections.emptyMap();
 
             if(fileDownloadAdornment.isPresent())
@@ -809,6 +864,11 @@ public class QJavalinImplementation
 
             for(QRecord record : records)
             {
+               if(!doesFieldHaveValue(field, record))
+               {
+                  continue;
+               }
+
                Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
                String       fileName   = null;
 
@@ -826,7 +886,8 @@ public class QJavalinImplementation
                   {
                      @SuppressWarnings("unchecked") // instance validation should make this safe!
                      List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
-                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, fileNameFormatFields);
+                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
+                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
                   }
                }
 
@@ -855,6 +916,45 @@ public class QJavalinImplementation
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
+   {
+      boolean fieldHasValue = false;
+
+      try
+      {
+         if(record.getValue(field.getName()) != null)
+         {
+            fieldHasValue = true;
+         }
+         else if(field.getIsHeavy())
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
+            if(heavyFieldLengths != null)
+            {
+               Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
+               if(fieldLength != null && fieldLength > 0)
+               {
+                  fieldHasValue = true;
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
+      }
+
+      return fieldHasValue;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

From 364e9f420b4d78eedca2755bda7056917c7d7712 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Tue, 30 May 2023 11:13:20 -0500
Subject: [PATCH 04/11] Update to support both JSON and multipart form bodies
 for create and update.

---
 .../javalin/QJavalinImplementation.java       | 40 +++++++++
 .../javalin/QJavalinImplementationTest.java   | 82 ++++++++++++++++++-
 2 files changed, 118 insertions(+), 4 deletions(-)

diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
index 77aaf73a..9c8174eb 100644
--- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
+++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
@@ -654,6 +654,46 @@ public class QJavalinImplementation
     *******************************************************************************/
    private static void setRecordValuesForInsertOrUpdate(Context context, QTableMetaData tableMetaData, QRecord record) throws IOException
    {
+      String  contentType       = Objects.requireNonNullElse(context.header("content-type"), "");
+      boolean isContentTypeJson = contentType.toLowerCase().contains("json");
+
+      try
+      {
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // if the caller said they've sent JSON, or if they didn't supply a content-type, then try to read the body //
+         // as JSON.  if it throws, we'll continue by trying to read form params, but if it succeeds, we'll return.  //
+         //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(isContentTypeJson || !StringUtils.hasContent(contentType))
+         {
+            Map<?, ?> map = context.bodyAsClass(Map.class);
+            for(Map.Entry<?, ?> entry : map.entrySet())
+            {
+               String fieldName = ValueUtils.getValueAsString(entry.getKey());
+               Object value     = entry.getValue();
+
+               if(StringUtils.hasContent(String.valueOf(value)))
+               {
+                  record.setValue(fieldName, (Serializable) value);
+               }
+               else if("".equals(value))
+               {
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////
+                  // if frontend sent us an empty string - put a null in the record's value map.                   //
+                  // this could potentially be changed to be type-specific (e.g., store an empty-string for STRING //
+                  // fields, but null for INTEGER, etc) - but, who really wants empty-string in database anyway?   //
+                  ///////////////////////////////////////////////////////////////////////////////////////////////////
+                  record.setValue(fieldName, null);
+               }
+            }
+
+            return;
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error trying to read body as map", e, logPair("contentType", contentType));
+      }
+
       /////////////////////////
       // process form params //
       /////////////////////////
diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
index 0fa2e777..722cba48 100644
--- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
+++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
@@ -391,11 +391,13 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
    /*******************************************************************************
-    ** test an insert
+    ** test an insert - posting the data as a JSON object.
+    **
+    ** This was the original supported version, but multipart form was added in May 2023
     **
     *******************************************************************************/
    @Test
-   public void test_dataInsert()
+   public void test_dataInsertJson()
    {
       Map<String, Serializable> body = new HashMap<>();
       body.put("firstName", "Bobby");
@@ -425,11 +427,45 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
    /*******************************************************************************
-    ** test an update
+    ** test an insert - posting a multipart form.
     **
     *******************************************************************************/
    @Test
-   public void test_dataUpdate()
+   public void test_dataInsertMultipartForm()
+   {
+      HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person")
+         .header("Content-Type", "application/json")
+         .multiPartContent()
+         .field("firstName", "Bobby")
+         .field("lastName", "Hull")
+         .field("email", "bobby@hull.com")
+         .asString();
+
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertTrue(jsonObject.has("records"));
+      JSONArray records = jsonObject.getJSONArray("records");
+      assertEquals(1, records.length());
+      JSONObject record0 = records.getJSONObject(0);
+      assertTrue(record0.has("values"));
+      assertEquals("person", record0.getString("tableName"));
+      JSONObject values0 = record0.getJSONObject("values");
+      assertTrue(values0.has("firstName"));
+      assertEquals("Bobby", values0.getString("firstName"));
+      assertTrue(values0.has("id"));
+      assertEquals(7, values0.getInt("id"));
+   }
+
+
+
+   /*******************************************************************************
+    ** test an update - posting the data as a JSON object.
+    **
+    ** This was the original supported version, but multipart form was added in May 2023
+    **
+    *******************************************************************************/
+   @Test
+   public void test_dataUpdateJson()
    {
       Map<String, Serializable> body = new HashMap<>();
       body.put("firstName", "Free");
@@ -465,6 +501,44 @@ class QJavalinImplementationTest extends QJavalinTestBase
 
 
 
+   /*******************************************************************************
+    ** test an update - posting the data as a multipart form
+    **
+    *******************************************************************************/
+   @Test
+   public void test_dataUpdateMultipartForm()
+   {
+      HttpResponse<String> response = Unirest.patch(BASE_URL + "/data/person/4")
+         .multiPartContent()
+         .field("firstName", "Free")
+         .field("birthDate", "")
+         .asString();
+
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertTrue(jsonObject.has("records"));
+      JSONArray records = jsonObject.getJSONArray("records");
+      assertEquals(1, records.length());
+      JSONObject record0 = records.getJSONObject(0);
+      assertTrue(record0.has("values"));
+      assertEquals("person", record0.getString("tableName"));
+      JSONObject values0 = record0.getJSONObject("values");
+      assertEquals(4, values0.getInt("id"));
+      assertEquals("Free", values0.getString("firstName"));
+
+      ///////////////////////////////////////////////////////////////////
+      // re-GET the record, and validate that birthDate was nulled out //
+      ///////////////////////////////////////////////////////////////////
+      response = Unirest.get(BASE_URL + "/data/person/4").asString();
+      assertEquals(200, response.getStatus());
+      jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertTrue(jsonObject.has("values"));
+      JSONObject values = jsonObject.getJSONObject("values");
+      assertFalse(values.has("birthDate"));
+   }
+
+
+
    /*******************************************************************************
     ** test a delete
     **

From 794fb5e87a1795a31390aecb2d53f0d7ad3a2924 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Tue, 30 May 2023 13:48:26 -0500
Subject: [PATCH 05/11] Expand javalin tests

---
 .../javalin/QJavalinImplementationTest.java   | 137 +++++++++++++++---
 .../qqq/backend/javalin/TestUtils.java        |  14 +-
 .../src/test/resources/photo.png              | Bin 0 -> 150 bytes
 .../test/resources/prime-test-database.sql    |   6 +-
 4 files changed, 133 insertions(+), 24 deletions(-)
 create mode 100644 qqq-middleware-javalin/src/test/resources/photo.png

diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
index 722cba48..77af26c3 100644
--- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
+++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementationTest.java
@@ -22,6 +22,8 @@
 package com.kingsrook.qqq.backend.javalin;
 
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Serializable;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
@@ -185,6 +187,63 @@ class QJavalinImplementationTest extends QJavalinTestBase
       JSONObject values = jsonObject.getJSONObject("values");
       assertTrue(values.has("firstName"));
       assertTrue(values.has("id"));
+      assertTrue(values.has("photo"));
+
+      JSONObject displayValues = jsonObject.getJSONObject("displayValues");
+      assertEquals("darin-photo.png", displayValues.getString("photo"));
+
+      ////////////////////////////////////////////////////
+      // make sure person 2 doesn't have the blob value //
+      ////////////////////////////////////////////////////
+      response = Unirest.get(BASE_URL + "/data/person/2").asString();
+      assertEquals(200, response.getStatus());
+      jsonObject = JsonUtils.toJSONObject(response.getBody());
+      values = jsonObject.getJSONObject("values");
+      assertFalse(values.has("photo"));
+   }
+
+
+
+   /*******************************************************************************
+    ** test downloading a blob file
+    **
+    *******************************************************************************/
+   @Test
+   public void test_dataDownloadRecordField()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/data/person/1/photo/darin-photo.png").asString();
+      assertEquals(200, response.getStatus());
+      assertThat(response.getHeaders().get("content-type").get(0)).contains("image");
+
+      response = Unirest.get(BASE_URL + "/data/person/1/photo/darin-photo.png?download=1").asString();
+      assertEquals(200, response.getStatus());
+      assertThat(response.getHeaders().get("content-disposition").get(0))
+         .contains("attachment")
+         .contains("darin-photo.png");
+
+      /////////////////////////
+      // bad record id = 404 //
+      /////////////////////////
+      response = Unirest.get(BASE_URL + "/data/person/-1/photo/darin-photo.png").asString();
+      assertEquals(404, response.getStatus());
+
+      //////////////////////////
+      // bad field name = 404 //
+      //////////////////////////
+      response = Unirest.get(BASE_URL + "/data/person/1/notPhoto/darin-photo.png").asString();
+      assertEquals(404, response.getStatus());
+
+      /////////////////////////////
+      // missing file name = 404 //
+      /////////////////////////////
+      response = Unirest.get(BASE_URL + "/data/person/1/photo").asString();
+      assertEquals(404, response.getStatus());
+
+      //////////////////////////
+      // bad table name = 404 //
+      //////////////////////////
+      response = Unirest.get(BASE_URL + "/data/notPerson/1/photo/darin-photo.png").asString();
+      assertEquals(404, response.getStatus());
    }
 
 
@@ -431,29 +490,34 @@ class QJavalinImplementationTest extends QJavalinTestBase
     **
     *******************************************************************************/
    @Test
-   public void test_dataInsertMultipartForm()
+   public void test_dataInsertMultipartForm() throws IOException
    {
-      HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person")
-         .header("Content-Type", "application/json")
-         .multiPartContent()
-         .field("firstName", "Bobby")
-         .field("lastName", "Hull")
-         .field("email", "bobby@hull.com")
-         .asString();
+      try(InputStream photoInputStream = getClass().getResourceAsStream("/photo.png"))
+      {
+         HttpResponse<String> response = Unirest.post(BASE_URL + "/data/person")
+            .header("Content-Type", "application/json")
+            .multiPartContent()
+            .field("firstName", "Bobby")
+            .field("lastName", "Hull")
+            .field("email", "bobby@hull.com")
+            .field("photo", photoInputStream.readAllBytes(), "image")
+            .asString();
 
-      assertEquals(200, response.getStatus());
-      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
-      assertTrue(jsonObject.has("records"));
-      JSONArray records = jsonObject.getJSONArray("records");
-      assertEquals(1, records.length());
-      JSONObject record0 = records.getJSONObject(0);
-      assertTrue(record0.has("values"));
-      assertEquals("person", record0.getString("tableName"));
-      JSONObject values0 = record0.getJSONObject("values");
-      assertTrue(values0.has("firstName"));
-      assertEquals("Bobby", values0.getString("firstName"));
-      assertTrue(values0.has("id"));
-      assertEquals(7, values0.getInt("id"));
+         assertEquals(200, response.getStatus());
+         JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+         assertTrue(jsonObject.has("records"));
+         JSONArray records = jsonObject.getJSONArray("records");
+         assertEquals(1, records.length());
+         JSONObject record0 = records.getJSONObject(0);
+         assertTrue(record0.has("values"));
+         assertEquals("person", record0.getString("tableName"));
+         JSONObject values0 = record0.getJSONObject("values");
+         assertTrue(values0.has("firstName"));
+         assertEquals("Bobby", values0.getString("firstName"));
+         assertTrue(values0.has("id"));
+         assertEquals(7, values0.getInt("id"));
+         assertTrue(values0.has("photo"));
+      }
    }
 
 
@@ -718,4 +782,35 @@ class QJavalinImplementationTest extends QJavalinTestBase
       assertEquals(5, jsonObject.getJSONArray("options").getJSONObject(1).getInt("id"));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testServerInfo()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/serverInfo").asString();
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      assertTrue(jsonObject.has("startTimeMillis"));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testAuthenticationMetaData()
+   {
+      HttpResponse<String> response = Unirest.get(BASE_URL + "/metaData/authentication").asString();
+      assertEquals(200, response.getStatus());
+      JSONObject jsonObject = JsonUtils.toJSONObject(response.getBody());
+      assertNotNull(jsonObject);
+      assertTrue(jsonObject.has("name"));
+      assertTrue(jsonObject.has("type"));
+   }
+
 }
diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java
index bbd5b48c..c828f602 100644
--- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java
+++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/backend/javalin/TestUtils.java
@@ -44,6 +44,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeUsage;
 import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
@@ -229,7 +231,7 @@ public class TestUtils
     *******************************************************************************/
    public static QTableMetaData defineTablePerson()
    {
-      return new QTableMetaData()
+      QTableMetaData qTableMetaData = new QTableMetaData()
          .withName(TABLE_NAME_PERSON)
          .withLabel("Person")
          .withRecordLabelFormat("%s %s")
@@ -245,10 +247,20 @@ public class TestUtils
          .withField(new QFieldMetaData("partnerPersonId", QFieldType.INTEGER).withBackendName("partner_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON))
          .withField(new QFieldMetaData("email", QFieldType.STRING))
          .withField(new QFieldMetaData("testScriptId", QFieldType.INTEGER).withBackendName("test_script_id"))
+         .withField(new QFieldMetaData("photo", QFieldType.BLOB).withBackendName("photo"))
+         .withField(new QFieldMetaData("photoFileName", QFieldType.STRING).withBackendName("photo_file_name"))
          .withAssociatedScript(new AssociatedScript()
             .withFieldName("testScriptId")
             .withScriptTypeId(1)
             .withScriptTester(new QCodeReference(TestScriptAction.class, QCodeUsage.SCRIPT_TESTER)));
+
+      qTableMetaData.getField("photo")
+         .withIsHeavy(true)
+         .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD)
+            .withValue(AdornmentType.FileDownloadValues.DEFAULT_MIME_TYPE, "image")
+            .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "photoFileName"));
+
+      return (qTableMetaData);
    }
 
 
diff --git a/qqq-middleware-javalin/src/test/resources/photo.png b/qqq-middleware-javalin/src/test/resources/photo.png
new file mode 100644
index 0000000000000000000000000000000000000000..9ffe3ad29cdc9afe3480e41d26cfc28c01c3c206
GIT binary patch
literal 150
zcmeAS@N?(olHy`uVBq!ia0vp^Mj*_{3?x-PN__%SYymzYuCJ35|NsBr67(`1D8gCb
z5n0T@z;^_M8K-LVNdpDVJY5_^B&H@ONIYryDah>1JRwb}>GB!=178la_d2kBPF8GG
rV^o;WX>QBu!)@nLV^Hu<L5P7t)Qaahiwe66&;$lgS3j3^P6<r_SlcMW

literal 0
HcmV?d00001

diff --git a/qqq-middleware-javalin/src/test/resources/prime-test-database.sql b/qqq-middleware-javalin/src/test/resources/prime-test-database.sql
index a94c185b..bd233046 100644
--- a/qqq-middleware-javalin/src/test/resources/prime-test-database.sql
+++ b/qqq-middleware-javalin/src/test/resources/prime-test-database.sql
@@ -31,10 +31,12 @@ CREATE TABLE person
    birth_date DATE,
    email VARCHAR(250) NOT NULL,
    partner_person_id INT,
-   test_script_id INT
+   test_script_id INT,
+   photo BLOB,
+   photo_file_name VARCHAR(50)
 );
 
-INSERT INTO person (id, first_name, last_name, birth_date, email, partner_person_id) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 6);
+INSERT INTO person (id, first_name, last_name, birth_date, email, partner_person_id, photo, photo_file_name) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 6, '12345', 'darin-photo.png');
 INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com');
 INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com');
 INSERT INTO person (id, first_name, last_name, birth_date, email) VALUES (4, 'Tyler', 'Samples', '1990-01-01', 'tsamples@mmltholdings.com');

From c78d598035b9ebaef64eeb53bdaa07dbf2b5634f Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Tue, 30 May 2023 14:04:15 -0500
Subject: [PATCH 06/11] Add an eventCartridge (handler) for methods that throw;
 add template identifier (a name) to input

---
 .../templates/RenderTemplateAction.java       | 37 +++++++++++++--
 .../model/templates/RenderTemplateInput.java  | 32 +++++++++++++
 .../templates/RenderTemplateActionTest.java   | 47 +++++++++++++++++--
 3 files changed, 108 insertions(+), 8 deletions(-)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java
index 5449c942..1bb6e3ce 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateAction.java
@@ -26,20 +26,30 @@ import java.io.StringWriter;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateInput;
 import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateOutput;
 import com.kingsrook.qqq.backend.core.model.templates.TemplateType;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import org.apache.velocity.VelocityContext;
 import org.apache.velocity.app.Velocity;
+import org.apache.velocity.app.event.EventCartridge;
+import org.apache.velocity.app.event.MethodExceptionEventHandler;
 import org.apache.velocity.context.Context;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
 /*******************************************************************************
  ** Basic action to render a template!
+ **
+ ** hard-coded built to only assume Velocity right now.  could expand (and refactor) in future.
  *******************************************************************************/
 public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplateInput, RenderTemplateOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(RenderTemplateAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -52,9 +62,12 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
       if(TemplateType.VELOCITY.equals(input.getTemplateType()))
       {
          Velocity.init();
-         Context      context      = new VelocityContext(input.getContext());
+         Context context = new VelocityContext(input.getContext());
+
+         setupEventHandlers(context);
+
          StringWriter stringWriter = new StringWriter();
-         Velocity.evaluate(context, stringWriter, "logTag", input.getCode());
+         Velocity.evaluate(context, stringWriter, StringUtils.hasContent(input.getTemplateIdentifier()) ? input.getTemplateIdentifier() : "anonymous", input.getCode());
          output.setResult(stringWriter.getBuffer().toString());
       }
       else
@@ -67,12 +80,28 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void setupEventHandlers(Context context)
+   {
+      EventCartridge eventCartridge = new EventCartridge();
+      eventCartridge.addEventHandler((MethodExceptionEventHandler) (ctx, aClass, method, exception, info) ->
+      {
+         LOG.info("Exception in velocity template", exception, logPair("at", info.toString()));
+         return (null);
+      });
+      eventCartridge.attachToContext(context);
+   }
+
+
+
    /*******************************************************************************
     ** Most convenient static wrapper to render a Velocity template.
     *******************************************************************************/
    public static String renderVelocity(AbstractActionInput parentActionInput, Map<String, Object> context, String code) throws QException
    {
-      return (render(parentActionInput, TemplateType.VELOCITY, context, code));
+      return (render(TemplateType.VELOCITY, context, code));
    }
 
 
@@ -80,7 +109,7 @@ public class RenderTemplateAction extends AbstractQActionFunction<RenderTemplate
    /*******************************************************************************
     ** Convenient static wrapper to render a template of an arbitrary type (language).
     *******************************************************************************/
-   public static String render(AbstractActionInput parentActionInput, TemplateType templateType, Map<String, Object> context, String code) throws QException
+   public static String render(TemplateType templateType, Map<String, Object> context, String code) throws QException
    {
       RenderTemplateInput renderTemplateInput = new RenderTemplateInput();
       renderTemplateInput.setCode(code);
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/templates/RenderTemplateInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/templates/RenderTemplateInput.java
index eeaf05aa..87302d40 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/templates/RenderTemplateInput.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/templates/RenderTemplateInput.java
@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
  *******************************************************************************/
 public class RenderTemplateInput extends AbstractActionInput
 {
+   private String       templateIdentifier;
    private String       code; // todo - TemplateReference, like CodeReference??
    private TemplateType templateType;
 
@@ -147,4 +148,35 @@ public class RenderTemplateInput extends AbstractActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for templateIdentifier
+    *******************************************************************************/
+   public String getTemplateIdentifier()
+   {
+      return (this.templateIdentifier);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for templateIdentifier
+    *******************************************************************************/
+   public void setTemplateIdentifier(String templateIdentifier)
+   {
+      this.templateIdentifier = templateIdentifier;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for templateIdentifier
+    *******************************************************************************/
+   public RenderTemplateInput withTemplateIdentifier(String templateIdentifier)
+   {
+      this.templateIdentifier = templateIdentifier;
+      return (this);
+   }
+
 }
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java
index 1b8a8676..926b80f5 100644
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/RenderTemplateActionTest.java
@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateInput;
 import com.kingsrook.qqq.backend.core.model.templates.RenderTemplateOutput;
 import com.kingsrook.qqq.backend.core.model.templates.TemplateType;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -36,8 +37,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 /*******************************************************************************
  ** Unit test for RenderTemplateAction
  *******************************************************************************/
-class RenderTemplateActionTest extends BaseTest
+public class RenderTemplateActionTest extends BaseTest
 {
+   private int doThrowCallCount = 0;
+
+
 
    /*******************************************************************************
     **
@@ -82,11 +86,46 @@ class RenderTemplateActionTest extends BaseTest
    @Test
    void testMissingType()
    {
-      RenderTemplateInput parentActionInput = new RenderTemplateInput();
-
-      assertThatThrownBy(() -> RenderTemplateAction.render(parentActionInput, null, Map.of("name", "Darin"), "Hello, $name"))
+      assertThatThrownBy(() -> RenderTemplateAction.render(null, Map.of("name", "Darin"), "Hello, $name"))
          .isInstanceOf(QException.class)
          .hasMessageContaining("Unsupported Template Type");
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testExceptionInVelocity() throws QException
+   {
+      RenderTemplateInput renderTemplateInput = new RenderTemplateInput();
+      renderTemplateInput.setCode("""
+         This should throw: $this.doThrow().
+         This should throw silently: $!this.doThrow().
+         """);
+      renderTemplateInput.setContext(Map.of("this", this));
+      renderTemplateInput.setTemplateType(TemplateType.VELOCITY);
+      RenderTemplateOutput output = new RenderTemplateAction().execute(renderTemplateInput);
+      assertThat(output.getResult())
+         .contains("throw: $this.doThrow().")
+         .contains("throw silently: .");
+
+      ///////////////////////////////////////////////////////
+      // make sure our method got called twice as expected //
+      ///////////////////////////////////////////////////////
+      assertEquals(2, doThrowCallCount);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public String doThrow() throws Exception
+   {
+      doThrowCallCount++;
+      throw (new Exception("You asked to throw..."));
+   }
+
 }
\ No newline at end of file

From 7634246a03ddac21e73af11e337d678944c59d79 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Thu, 1 Jun 2023 16:13:02 -0500
Subject: [PATCH 07/11] Handling of BLOBs

---
 .../qqq/api/actions/ApiImplementation.java    | 15 +++++++++--
 .../actions/GenerateOpenApiSpecAction.java    | 27 +++++++++++++++++--
 .../qqq/api/actions/QRecordApiAdapter.java    | 20 ++++++++++++--
 .../java/com/kingsrook/qqq/api/TestUtils.java | 14 +++++++---
 .../GenerateOpenApiSpecActionTest.java        |  9 ++++++-
 .../api/actions/QRecordApiAdapterTest.java    |  8 ++++--
 .../api/javalin/QJavalinApiHandlerTest.java   | 16 ++++++++---
 7 files changed, 92 insertions(+), 17 deletions(-)

diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java
index 675d37b8..96a46201 100644
--- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java
+++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/ApiImplementation.java
@@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.Pair;
@@ -108,6 +109,7 @@ public class ApiImplementation
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
       queryInput.setIncludeAssociations(true);
+      queryInput.setShouldFetchHeavyFields(true);
 
       PermissionsHelper.checkTablePermissionThrowing(queryInput, TablePermissionSubType.READ);
 
@@ -251,7 +253,7 @@ public class ApiImplementation
                {
                   try
                   {
-                     filter.addCriteria(parseQueryParamToCriteria(name, value));
+                     filter.addCriteria(parseQueryParamToCriteria(field, name, value));
                   }
                   catch(Exception e)
                   {
@@ -506,6 +508,7 @@ public class ApiImplementation
 
       getInput.setPrimaryKey(primaryKey);
       getInput.setIncludeAssociations(true);
+      getInput.setShouldFetchHeavyFields(true);
 
       GetAction getAction = new GetAction();
       GetOutput getOutput = getAction.execute(getInput);
@@ -911,7 +914,7 @@ public class ApiImplementation
    /*******************************************************************************
     **
     *******************************************************************************/
-   private static QFilterCriteria parseQueryParamToCriteria(String name, String value) throws QException
+   private static QFilterCriteria parseQueryParamToCriteria(QFieldMetaData field, String name, String value) throws QException
    {
       ///////////////////////////////////
       // process & discard a leading ! //
@@ -989,6 +992,14 @@ public class ApiImplementation
          throw (new QException("Unexpected noOfValues [" + selectedOperator.noOfValues + "] in operator [" + selectedOperator + "]"));
       }
 
+      if(field.getType().equals(QFieldType.BLOB))
+      {
+         if(!selectedOperator.equals(Operator.EMPTY))
+         {
+            throw (new QBadRequestException("Operator " + selectedOperator.prefix + " may not be used for field " + name + " (blob fields only support operators EMPTY or !EMPTY)"));
+         }
+      }
+
       return (new QFilterCriteria(name, isNot ? selectedOperator.negativeOperator : selectedOperator.positiveOperator, criteriaValues));
    }
 
diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java
index 09a2db10..b9aa7981 100644
--- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java
+++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecAction.java
@@ -480,9 +480,19 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
 
          for(QFieldMetaData tableApiField : tableApiFields)
          {
+            StringBuilder description = new StringBuilder("Query on the " + tableApiField.getLabel() + " field.  ");
+            if(tableApiField.getType().equals(QFieldType.BLOB))
+            {
+               description.append("Can only query for EMPTY or !EMPTY.");
+            }
+            else
+            {
+               description.append("Can prefix value with an operator, else defaults to = (equals).");
+            }
+
             queryGet.getParameters().add(new Parameter()
                .withName(tableApiField.getName())
-               .withDescription("Query on the " + tableApiField.getLabel() + " field.  Can prefix value with an operator, else defaults to = (equals).")
+               .withDescription(description.toString())
                .withIn("query")
                .withExplode(true)
                .withSchema(new Schema()
@@ -837,6 +847,9 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       rs.put("criteriaStringNotLike", new ExampleWithListValue().withSummary("not starting with f").withValue(ListBuilder.of("!LIKE f%")));
       rs.put("criteriaStringMultiple", new ExampleWithListValue().withSummary("multiple criteria: between bar and foo and not equal to baz").withValue(ListBuilder.of("BETWEEN bar,foo", "!baz")));
 
+      rs.put("criteriaBlobEmpty", new ExampleWithListValue().withSummary("null value").withValue(ListBuilder.of("EMPTY")));
+      rs.put("criteriaBlobNotEmpty", new ExampleWithListValue().withSummary("non-null value").withValue(ListBuilder.of("!EMPTY")));
+
       return (rs);
    }
 
@@ -870,6 +883,10 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
       {
          componentExamples.keySet().stream().filter(s -> s.startsWith("criteriaBoolean")).forEach(exampleRefs::add);
       }
+      else if(tableApiField.getType().equals(QFieldType.BLOB))
+      {
+         componentExamples.keySet().stream().filter(s -> s.startsWith("criteriaBlob")).forEach(exampleRefs::add);
+      }
 
       Map<String, Example> rs = new LinkedHashMap<>();
 
@@ -910,10 +927,16 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
     *******************************************************************************/
    private Schema getFieldSchema(QTableMetaData table, QFieldMetaData field)
    {
+      String description = field.getLabel() + " for the " + table.getLabel() + ".";
+      if(field.getType().equals(QFieldType.BLOB))
+      {
+         description = "Base64 encoded " + description;
+      }
+
       Schema fieldSchema = new Schema()
          .withType(getFieldType(field))
          .withFormat(getFieldFormat(field))
-         .withDescription(field.getLabel() + " for the " + table.getLabel() + ".");
+         .withDescription(description);
 
       if(!field.getIsEditable())
       {
diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java
index 687fe2aa..ab9e982c 100644
--- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java
+++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java
@@ -24,6 +24,7 @@ package com.kingsrook.qqq.api.actions;
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -38,6 +39,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -79,14 +81,23 @@ public class QRecordApiAdapter
       {
          ApiFieldMetaData apiFieldMetaData = ObjectUtils.tryAndRequireNonNullElse(() -> ApiFieldMetaDataContainer.of(field).getApiFieldMetaData(apiName), new ApiFieldMetaData());
          String           apiFieldName     = ApiFieldMetaData.getEffectiveApiFieldName(apiName, field);
+
+         Serializable value = null;
          if(StringUtils.hasContent(apiFieldMetaData.getReplacedByFieldName()))
          {
-            outputRecord.put(apiFieldName, record.getValue(apiFieldMetaData.getReplacedByFieldName()));
+            value = record.getValue(apiFieldMetaData.getReplacedByFieldName());
          }
          else
          {
-            outputRecord.put(apiFieldName, record.getValue(field.getName()));
+            value = record.getValue(field.getName());
          }
+
+         if(field.getType().equals(QFieldType.BLOB) && value instanceof byte[] bytes)
+         {
+            value = Base64.getEncoder().encodeToString(bytes);
+         }
+
+         outputRecord.put(apiFieldName, value);
       }
 
       //////////////////////////////////////////////////////////////////////////////////////////////////
@@ -142,6 +153,11 @@ public class QRecordApiAdapter
             QFieldMetaData field = apiFieldsMap.get(jsonKey);
             Object         value = jsonObject.isNull(jsonKey) ? null : jsonObject.get(jsonKey);
 
+            if(field.getType().equals(QFieldType.BLOB) && value instanceof String s)
+            {
+               value = Base64.getDecoder().decode(s);
+            }
+
             ////////////////////////////////////////////////////////////////////////////////////////////////////////
             // generally, omit non-editable fields -                                                              //
             // however - if we're asked to include the primary key (and this is the primary key), then include it //
diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java
index 0ba12b72..cb1b9660 100644
--- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java
+++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/TestUtils.java
@@ -22,8 +22,8 @@
 package com.kingsrook.qqq.api;
 
 
-import java.time.LocalDate;
 import java.util.List;
+import java.util.function.Consumer;
 import com.kingsrook.qqq.api.model.APIVersion;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaData;
 import com.kingsrook.qqq.api.model.metadata.ApiInstanceMetaDataContainer;
@@ -191,7 +191,8 @@ public class TestUtils
          // .withField(new QFieldMetaData("customValue", QFieldType.INTEGER).withPossibleValueSourceName(POSSIBLE_VALUE_SOURCE_CUSTOM))
          .withField(new QFieldMetaData("noOfShoes", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS))
          .withField(new QFieldMetaData("cost", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
-         .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY));
+         .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
+         .withField(new QFieldMetaData("photo", QFieldType.BLOB));
 
       table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD.getRole(), new QCodeReference(PersonPreInsertCustomizer.class, QCodeUsage.CUSTOMIZER));
 
@@ -378,11 +379,16 @@ public class TestUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static void insertPersonRecord(Integer id, String firstName, String lastName, LocalDate birthDate) throws QException
+   public static void insertPersonRecord(Integer id, String firstName, String lastName, Consumer<QRecord> recordCustomizer) throws QException
    {
       InsertInput insertInput = new InsertInput();
       insertInput.setTableName(TestUtils.TABLE_NAME_PERSON);
-      insertInput.setRecords(List.of(new QRecord().withValue("id", id).withValue("firstName", firstName).withValue("lastName", lastName).withValue("birthDate", birthDate)));
+      QRecord record = new QRecord().withValue("id", id).withValue("firstName", firstName).withValue("lastName", lastName);
+      if(recordCustomizer != null)
+      {
+         recordCustomizer.accept(record);
+      }
+      insertInput.setRecords(List.of(record));
       new InsertAction().execute(insertInput);
    }
 
diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecActionTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecActionTest.java
index f4795b56..acd4e4f1 100644
--- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecActionTest.java
+++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/GenerateOpenApiSpecActionTest.java
@@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -92,7 +93,13 @@ class GenerateOpenApiSpecActionTest extends BaseTest
                   .withTableName(table.getName())
                   .withVersion(supportedVersion.toString())
                   .withApiName(apiInstanceMetaData.getName()));
-               // System.out.println(output.getYaml());
+
+               if(table.getName().equals(TestUtils.TABLE_NAME_PERSON))
+               {
+                  assertThat(output.getYaml())
+                     .contains("Query on the First Name field.  Can prefix value with an operator")
+                     .contains("Query on the Photo field.  Can only query for EMPTY or !EMPTY");
+               }
             }
          }
       }
diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java
index c53227ff..4319dfe1 100644
--- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java
+++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java
@@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 
@@ -61,12 +62,14 @@ class QRecordApiAdapterTest extends BaseTest
          .withValue("noOfShoes", 2)
          .withValue("birthDate", LocalDate.of(1980, Month.MAY, 31))
          .withValue("cost", new BigDecimal("3.50"))
-         .withValue("price", new BigDecimal("9.99"));
+         .withValue("price", new BigDecimal("9.99"))
+         .withValue("photo", "ABCD".getBytes());
 
       Map<String, Serializable> pastApiRecord = QRecordApiAdapter.qRecordToApiMap(person, TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2022_Q4);
       assertEquals(2, pastApiRecord.get("shoeCount")); // old field name - not currently in the QTable, but we can still get its value!
       assertFalse(pastApiRecord.containsKey("noOfShoes")); // current field name - doesn't appear in old api-version
       assertFalse(pastApiRecord.containsKey("cost")); // a current field name, but also not in this old api version
+      assertEquals("QUJDRA==", pastApiRecord.get("photo")); // base64 version of "ABCD".getBytes()
 
       Map<String, Serializable> currentApiRecord = QRecordApiAdapter.qRecordToApiMap(person, TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2023_Q1);
       assertFalse(currentApiRecord.containsKey("shoeCount")); // old field name - not in this current api version
@@ -109,9 +112,10 @@ class QRecordApiAdapterTest extends BaseTest
       // past version took shoeCount - so we still take that, but now put it in noOfShoes field of qRecord //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////
       QRecord recordFromOldApi = QRecordApiAdapter.apiJsonObjectToQRecord(new JSONObject("""
-         {"firstName": "Tim", "shoeCount": 2}
+         {"firstName": "Tim", "shoeCount": 2, "photo": "QUJDRA=="}
          """), TestUtils.TABLE_NAME_PERSON, TestUtils.API_NAME, TestUtils.V2022_Q4, true);
       assertEquals(2, recordFromOldApi.getValueInteger("noOfShoes"));
+      assertArrayEquals("ABCD".getBytes(), recordFromOldApi.getValueByteArray("photo"));
 
       ///////////////////////////////////////////
       // current version takes it as noOfShoes //
diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java
index 6dc6f93f..2a140627 100644
--- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java
+++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java
@@ -64,6 +64,7 @@ import org.junit.jupiter.api.Test;
 import static com.kingsrook.qqq.api.TestUtils.insertPersonRecord;
 import static com.kingsrook.qqq.api.TestUtils.insertSimpsons;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -194,7 +195,7 @@ class QJavalinApiHandlerTest extends BaseTest
    @Test
    void testGet200() throws QException
    {
-      insertPersonRecord(1, "Homer", "Simpson");
+      insertPersonRecord(1, "Homer", "Simpson", qRecord -> qRecord.withValue("photo", "12345".getBytes()));
 
       HttpResponse<String> response = Unirest.get(BASE_URL + "/api/" + VERSION + "/person/1").asString();
       assertEquals(HttpStatus.OK_200, response.getStatus());
@@ -202,6 +203,7 @@ class QJavalinApiHandlerTest extends BaseTest
       assertEquals(1, jsonObject.getInt("id"));
       assertEquals("Homer", jsonObject.getString("firstName"));
       assertEquals("Simpson", jsonObject.getString("lastName"));
+      assertEquals("MTIzNDU=", jsonObject.getString("photo")); // base64 of "12345".getBytes()
       assertTrue(jsonObject.isNull("noOfShoes"));
       assertFalse(jsonObject.has("someNonField"));
    }
@@ -335,7 +337,7 @@ class QJavalinApiHandlerTest extends BaseTest
    @Test
    void testFieldDifferencesBetweenApis() throws QException
    {
-      insertPersonRecord(1, "Homer", "Simpson", LocalDate.of(1970, Month.JANUARY, 1));
+      insertPersonRecord(1, "Homer", "Simpson", qRecord -> qRecord.withValue("birthDate", LocalDate.of(1970, Month.JANUARY, 1)));
 
       /////////////////////////////////////////////////////////////
       // on the main api, birthDate has been renamed to birthDay //
@@ -364,7 +366,7 @@ class QJavalinApiHandlerTest extends BaseTest
    @Test
    void testQuery200SomethingFound() throws QException
    {
-      insertPersonRecord(1, "Homer", "Simpson");
+      insertPersonRecord(1, "Homer", "Simpson", qRecord -> qRecord.withValue("photo", "12345".getBytes()));
 
       HttpResponse<String> response = Unirest.get(BASE_URL + "/api/" + VERSION + "/person/query").asString();
       assertEquals(HttpStatus.OK_200, response.getStatus());
@@ -378,6 +380,7 @@ class QJavalinApiHandlerTest extends BaseTest
       assertEquals(1, jsonObject.getInt("id"));
       assertEquals("Homer", jsonObject.getString("firstName"));
       assertEquals("Simpson", jsonObject.getString("lastName"));
+      assertEquals("MTIzNDU=", jsonObject.getString("photo")); // base64 of "12345".getBytes()
       assertTrue(jsonObject.isNull("noOfShoes"));
       assertFalse(jsonObject.has("someNonField"));
    }
@@ -470,8 +473,11 @@ class QJavalinApiHandlerTest extends BaseTest
       assertPersonQueryFindsFirstNames(List.of(), "noOfShoes=!EMPTY");
       assertPersonQueryFindsFirstNames(List.of("Homer", "Marge", "Bart", "Lisa", "Maggie"), "id=!EMPTY&orderBy=id");
       assertPersonQueryFindsFirstNames(List.of(), "id=EMPTY");
+      assertPersonQueryFindsFirstNames(List.of("Homer", "Marge", "Bart", "Lisa", "Maggie"), "photo=EMPTY&orderBy=id");
+      assertPersonQueryFindsFirstNames(List.of(), "photo=!EMPTY");
 
       assertError("Unexpected value after operator EMPTY for field id", BASE_URL + "/api/" + VERSION + "/person/query?id=EMPTY 3");
+      assertError("Operator = may not be used for field photo (blob fields only support operators EMPTY or !EMPTY)", BASE_URL + "/api/" + VERSION + "/person/query?photo=ABCD");
    }
 
 
@@ -547,7 +553,7 @@ class QJavalinApiHandlerTest extends BaseTest
    {
       HttpResponse<String> response = Unirest.post(BASE_URL + "/api/" + VERSION + "/person/")
          .body("""
-            {"firstName": "Moe"}
+            {"firstName": "Moe", "photo": "MTIzNDU="}
             """)
          .asString();
       assertEquals(HttpStatus.CREATED_201, response.getStatus());
@@ -556,6 +562,7 @@ class QJavalinApiHandlerTest extends BaseTest
 
       QRecord record = getRecord(TestUtils.TABLE_NAME_PERSON, 1);
       assertEquals("Moe", record.getValueString("firstName"));
+      assertArrayEquals("12345".getBytes(), record.getValueByteArray("photo"));
    }
 
 
@@ -1293,6 +1300,7 @@ class QJavalinApiHandlerTest extends BaseTest
       getInput.setTableName(tableName);
       getInput.setPrimaryKey(id);
       getInput.setIncludeAssociations(true);
+      getInput.setShouldFetchHeavyFields(true);
       GetOutput getOutput = new GetAction().execute(getInput);
       QRecord   record    = getOutput.getRecord();
       return record;

From b67813b7adc16c6d2ca7b2b64b1894529356c57c Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Thu, 1 Jun 2023 16:31:54 -0500
Subject: [PATCH 08/11] Update test to handle blob correctly

---
 .../kingsrook/qqq/api/actions/QRecordApiAdapterTest.java  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java
index 4319dfe1..6bc34538 100644
--- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java
+++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/actions/QRecordApiAdapterTest.java
@@ -95,6 +95,14 @@ class QRecordApiAdapterTest extends BaseTest
          Map<String, Serializable> alternativeApiRecord = QRecordApiAdapter.qRecordToApiMap(person, TestUtils.TABLE_NAME_PERSON, TestUtils.ALTERNATIVE_API_NAME, version);
          for(String key : person.getValues().keySet())
          {
+            if(key.equals("photo"))
+            {
+               ////////////////////////////////////////////////////////////////////////////////////////
+               // ok, well, skip the blob field (should be base64 version, and is covered elsewhere) //
+               ////////////////////////////////////////////////////////////////////////////////////////
+               continue;
+            }
+
             assertEquals(person.getValueString(key), ValueUtils.getValueAsString(alternativeApiRecord.get(key)));
          }
       }

From 499d59ade2a1fb2c44ff2ede6a0f3b53477bce4f Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Thu, 1 Jun 2023 16:32:34 -0500
Subject: [PATCH 09/11] Add overload of loadTableToMap that takes
 Consumer<QueryInput> (e.g., to request heavy fields)

---
 .../processes/utils/GeneralProcessUtils.java    | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java
index 98a41a4f..7c7afc3a 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/GeneralProcessUtils.java
@@ -28,6 +28,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -394,9 +395,25 @@ public class GeneralProcessUtils
     ** Note - null values from the key field are NOT put in the map.
     *******************************************************************************/
    public static <T extends QRecordEntity> Map<Serializable, T> loadTableToMap(AbstractActionInput parentActionInput, String tableName, String keyFieldName, Class<T> entityClass) throws QException
+   {
+      return (loadTableToMap(tableName, keyFieldName, entityClass, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Note - null values from the key field are NOT put in the map.
+    *******************************************************************************/
+   public static <T extends QRecordEntity> Map<Serializable, T> loadTableToMap(String tableName, String keyFieldName, Class<T> entityClass, Consumer<QueryInput> queryInputCustomizer) throws QException
    {
       QueryInput queryInput = new QueryInput();
       queryInput.setTableName(tableName);
+
+      if(queryInputCustomizer != null)
+      {
+         queryInputCustomizer.accept(queryInput);
+      }
+
       QueryOutput   queryOutput = new QueryAction().execute(queryInput);
       List<QRecord> records     = queryOutput.getRecords();
 

From fdbc7510485fc548fc0f19b06be4c0aeb5c22f05 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Mon, 5 Jun 2023 08:17:45 -0500
Subject: [PATCH 10/11] more support for blobs - fetch heavy flag for process
 extract - ignore blobs in bulk edit, load - set download urls in child-list
 render and javalin via shared method in QValueFormatter

also, make ETL load step aware of transform step, and more flexible process summary between them
---
 .../widgets/ChildRecordListRenderer.java      |  23 ++--
 .../core/actions/values/QValueFormatter.java  | 128 ++++++++++++++++++
 .../core/instances/QInstanceEnricher.java     |   3 +-
 .../processes/RunBackendStepOutput.java       |  22 +++
 .../AbstractProcessMetaDataBuilder.java       |  11 ++
 .../AbstractLoadStep.java                     |  33 +++++
 .../ExtractViaQueryStep.java                  |   5 +
 .../StreamedETLExecuteStep.java               |  12 +-
 .../StreamedETLWithFrontendProcess.java       |   2 +
 .../javalin/QJavalinImplementation.java       | 128 +-----------------
 10 files changed, 227 insertions(+), 140 deletions(-)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java
index fdb7b9df..bf1cea59 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ChildRecordListRenderer.java
@@ -33,6 +33,7 @@ import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
@@ -160,10 +161,12 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
    @Override
    public RenderWidgetOutput render(RenderWidgetInput input) throws QException
    {
-      String        widgetLabel = input.getQueryParams().get("widgetLabel");
-      String        joinName    = input.getQueryParams().get("joinName");
-      QJoinMetaData join        = input.getInstance().getJoin(joinName);
-      String        id          = input.getQueryParams().get("id");
+      String         widgetLabel = input.getQueryParams().get("widgetLabel");
+      String         joinName    = input.getQueryParams().get("joinName");
+      QJoinMetaData  join        = input.getInstance().getJoin(joinName);
+      String         id          = input.getQueryParams().get("id");
+      QTableMetaData leftTable   = input.getInstance().getTable(join.getLeftTable());
+      QTableMetaData rightTable  = input.getInstance().getTable(join.getRightTable());
 
       Integer maxRows = null;
       if(StringUtils.hasContent(input.getQueryParams().get("maxRows")))
@@ -187,8 +190,7 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
 
       if(record == null)
       {
-         QTableMetaData table = input.getInstance().getTable(join.getLeftTable());
-         throw (new QNotFoundException("Could not find " + (table == null ? "" : table.getLabel()) + " with primary key " + id));
+         throw (new QNotFoundException("Could not find " + (leftTable == null ? "" : leftTable.getLabel()) + " with primary key " + id));
       }
 
       ////////////////////////////////////////////////////////////////////
@@ -209,6 +211,8 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
       queryInput.setFilter(filter);
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
 
+      QValueFormatter.setBlobValuesToDownloadUrls(rightTable, queryOutput.getRecords());
+
       int totalRows = queryOutput.getRecords().size();
       if(maxRows != null && (queryOutput.getRecords().size() == maxRows))
       {
@@ -222,11 +226,10 @@ public class ChildRecordListRenderer extends AbstractWidgetRenderer
          totalRows = new CountAction().execute(countInput).getCount();
       }
 
-      QTableMetaData table       = input.getInstance().getTable(join.getRightTable());
-      String         tablePath   = input.getInstance().getTablePath(table.getName());
-      String         viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
+      String tablePath   = input.getInstance().getTablePath(rightTable.getName());
+      String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset()));
 
-      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, table, tablePath, viewAllLink, totalRows);
+      ChildRecordListData widgetData = new ChildRecordListData(widgetLabel, queryOutput, rightTable, tablePath, viewAllLink, totalRows);
 
       if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(input.getQueryParams().get("canAddChildRecord"))))
       {
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
index 3356fbae..c58d78cc 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java
@@ -29,13 +29,18 @@ import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@@ -434,4 +439,127 @@ public class QValueFormatter
       }
    }
 
+
+
+   /*******************************************************************************
+    ** For any BLOB type fields in the list of records, change their value to
+    ** the URL where they can be downloaded, and set their display value to a file name.
+    *******************************************************************************/
+   public static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
+   {
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         if(field.getType().equals(QFieldType.BLOB))
+         {
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // file name comes from:                                                                                            //
+            // if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
+            // - unless it was empty - then we do the "default thing":                                                          //
+            // else - the "default thing" is:                                                                                   //
+            // - tableLabel primaryKey fieldLabel                                                                               //
+            // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
+            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
+            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
+
+            if(fileDownloadAdornment.isPresent())
+            {
+               adornmentValues = fileDownloadAdornment.get().getValues();
+            }
+
+            String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
+            String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
+            String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
+
+            for(QRecord record : records)
+            {
+               if(!doesFieldHaveValue(field, record))
+               {
+                  continue;
+               }
+
+               Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
+               String       fileName   = null;
+
+               //////////////////////////////////////////////////
+               // try to make file name from the fileNameField //
+               //////////////////////////////////////////////////
+               if(StringUtils.hasContent(fileNameField))
+               {
+                  fileName = record.getValueString(fileNameField);
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  if(StringUtils.hasContent(fileNameFormat))
+                  {
+                     @SuppressWarnings("unchecked") // instance validation should make this safe!
+                     List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
+                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
+                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
+                  }
+               }
+
+               if(!StringUtils.hasContent(fileName))
+               {
+                  //////////////////////////////////
+                  // make default name if missing //
+                  //////////////////////////////////
+                  fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
+
+                  if(StringUtils.hasContent(defaultExtension))
+                  {
+                     //////////////////////////////////////////
+                     // add default extension if we have one //
+                     //////////////////////////////////////////
+                     fileName += "." + defaultExtension;
+                  }
+               }
+
+               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               record.setDisplayValue(field.getName(), fileName);
+            }
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
+   {
+      boolean fieldHasValue = false;
+
+      try
+      {
+         if(record.getValue(field.getName()) != null)
+         {
+            fieldHasValue = true;
+         }
+         else if(field.getIsHeavy())
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
+            if(heavyFieldLengths != null)
+            {
+               Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
+               if(fieldLength != null && fieldLength > 0)
+               {
+                  fieldHasValue = true;
+               }
+            }
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
+      }
+
+      return fieldHasValue;
+   }
+
 }
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
index 465434bc..816c9257 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java
@@ -716,7 +716,7 @@ public class QInstanceEnricher
             try
             {
                QFieldMetaData field = table.getField(fieldName);
-               if(field.getIsEditable())
+               if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB))
                {
                   editableFields.add(field);
                }
@@ -774,6 +774,7 @@ public class QInstanceEnricher
 
       List<QFieldMetaData> editableFields = table.getFields().values().stream()
          .filter(QFieldMetaData::getIsEditable)
+         .filter(f -> !f.getType().equals(QFieldType.BLOB))
          .toList();
 
       QFrontendStepMetaData editScreen = new QFrontendStepMetaData()
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
index fac4b6c4..9a915141 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepOutput.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -290,4 +291,25 @@ public class RunBackendStepOutput extends AbstractActionOutput implements Serial
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void addAuditSingleInput(AuditSingleInput auditSingleInput)
+   {
+      if(getAuditInputList() == null)
+      {
+         setAuditInputList(new ArrayList<>());
+      }
+
+      if(getAuditInputList().isEmpty())
+      {
+         getAuditInputList().add(new AuditInput());
+      }
+
+      AuditInput auditInput = getAuditInputList().get(0);
+      auditInput.addAuditSingleInput(auditSingleInput);
+   }
+
 }
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
index 64d2580e..c1fe43f9 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/AbstractProcessMetaDataBuilder.java
@@ -70,6 +70,17 @@ public class AbstractProcessMetaDataBuilder
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public AbstractProcessMetaDataBuilder withInputFieldDefaultValue(String fieldName, Serializable value)
+   {
+      setInputFieldDefaultValue(fieldName, value);
+      return (this);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java
index fa7b086a..835e79d7 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/AbstractLoadStep.java
@@ -47,6 +47,8 @@ public abstract class AbstractLoadStep implements BackendStep
    private   Optional<QBackendTransaction> transaction = Optional.empty();
    protected QSession                      session;
 
+   private AbstractTransformStep transformStep;
+
 
 
    /*******************************************************************************
@@ -122,4 +124,35 @@ public abstract class AbstractLoadStep implements BackendStep
       return (null);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for transformStep
+    *******************************************************************************/
+   public AbstractTransformStep getTransformStep()
+   {
+      return (this.transformStep);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for transformStep
+    *******************************************************************************/
+   public void setTransformStep(AbstractTransformStep transformStep)
+   {
+      this.transformStep = transformStep;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for transformStep
+    *******************************************************************************/
+   public AbstractLoadStep withTransformStep(AbstractTransformStep transformStep)
+   {
+      this.transformStep = transformStep;
+      return (this);
+   }
+
 }
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java
index 8f38b8fd..c4d7ff89 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/ExtractViaQueryStep.java
@@ -109,6 +109,11 @@ public class ExtractViaQueryStep extends AbstractExtractStep
       queryInput.setRecordPipe(getRecordPipe());
       queryInput.setAsyncJobCallback(runBackendStepInput.getAsyncJobCallback());
 
+      if(runBackendStepInput.getValuePrimitiveBoolean(StreamedETLWithFrontendProcess.FIELD_FETCH_HEAVY_FIELDS))
+      {
+         queryInput.setShouldFetchHeavyFields(true);
+      }
+
       customizeInputPreQuery(queryInput);
 
       new QueryAction().execute(queryInput);
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
index 2b61ead3..3894ea24 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
@@ -73,6 +73,8 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          AbstractTransformStep transformStep = getTransformStep(runBackendStepInput);
          AbstractLoadStep      loadStep      = getLoadStep(runBackendStepInput);
 
+         loadStep.setTransformStep(transformStep);
+
          /////////////////////////////////////////////////////////////////////////////
          // let the load step override the capacity for the record pipe.            //
          // this is useful for slower load steps - so that the extract step doesn't //
@@ -140,13 +142,17 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
          // get the process summary from the load step, if it's a summary-provider -- else, use the transform step (which is always a provider) //
          /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         ArrayList<ProcessSummaryLineInterface> processSummaryLines = null;
          if(loadStep instanceof ProcessSummaryProviderInterface provider)
          {
-            runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, provider.doGetProcessSummary(runBackendStepOutput, true));
+            processSummaryLines = provider.doGetProcessSummary(runBackendStepOutput, true);
+            runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryLines);
          }
-         else
+
+         if(CollectionUtils.nullSafeIsEmpty(processSummaryLines))
          {
-            runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, transformStep.doGetProcessSummary(runBackendStepOutput, true));
+            processSummaryLines = transformStep.doGetProcessSummary(runBackendStepOutput, true);
+            runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummaryLines);
          }
 
          /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
index 7b31440d..126735ef 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java
@@ -81,6 +81,7 @@ public class StreamedETLWithFrontendProcess
    public static final String FIELD_DESTINATION_TABLE    = "destinationTable"; // String
    public static final String FIELD_RECORD_COUNT         = "recordCount"; // Integer
    public static final String FIELD_DEFAULT_QUERY_FILTER = "defaultQueryFilter"; // QQueryFilter or String (json, of q QQueryFilter)
+   public static final String FIELD_FETCH_HEAVY_FIELDS   = "fetchHeavyFields"; // Boolean
 
    public static final String FIELD_SUPPORTS_FULL_VALIDATION = "supportsFullValidation"; // Boolean
    public static final String FIELD_DO_FULL_VALIDATION       = "doFullValidation"; // Boolean
@@ -142,6 +143,7 @@ public class StreamedETLWithFrontendProcess
          .withCode(new QCodeReference(StreamedETLPreviewStep.class))
          .withInputData(new QFunctionInputMetaData()
             .withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_SOURCE_TABLE)))
+            .withField(new QFieldMetaData(FIELD_FETCH_HEAVY_FIELDS, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_FETCH_HEAVY_FIELDS, false)))
             .withField(new QFieldMetaData(FIELD_DESTINATION_TABLE, QFieldType.STRING).withDefaultValue(defaultFieldValues.get(FIELD_DESTINATION_TABLE)))
             .withField(new QFieldMetaData(FIELD_SUPPORTS_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.getOrDefault(FIELD_SUPPORTS_FULL_VALIDATION, true)))
             .withField(new QFieldMetaData(FIELD_DO_FULL_VALIDATION, QFieldType.BOOLEAN).withDefaultValue(defaultFieldValues.get(FIELD_DO_FULL_VALIDATION)))
diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
index 9c8174eb..93f0c4c7 100644
--- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
+++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
@@ -32,7 +32,6 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -858,7 +857,7 @@ public class QJavalinImplementation
                + table.getFields().get(table.getPrimaryKeyField()).getLabel() + " of " + primaryKey));
          }
 
-         setBlobValuesToDownloadUrls(table, List.of(record));
+         QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record));
 
          QJavalinAccessLogger.logEndSuccess();
          context.result(JsonUtils.toJson(record));
@@ -872,129 +871,6 @@ public class QJavalinImplementation
 
 
 
-   /*******************************************************************************
-    ** For any BLOB type fields in the list of records, change their value to
-    ** the URL where they can be downloaded, and set their display value to a file name.
-    *******************************************************************************/
-   private static void setBlobValuesToDownloadUrls(QTableMetaData table, List<QRecord> records)
-   {
-      for(QFieldMetaData field : table.getFields().values())
-      {
-         if(field.getType().equals(QFieldType.BLOB))
-         {
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // file name comes from:                                                                                            //
-            // if there's a FILE_DOWNLOAD adornment, with a FILE_NAME_FIELD value, then the full filename comes from that field //
-            // - unless it was empty - then we do the "default thing":                                                          //
-            // else - the "default thing" is:                                                                                   //
-            // - tableLabel primaryKey fieldLabel                                                                               //
-            // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
-            //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
-            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
-
-            if(fileDownloadAdornment.isPresent())
-            {
-               adornmentValues = fileDownloadAdornment.get().getValues();
-            }
-
-            String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
-            String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
-            String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION));
-
-            for(QRecord record : records)
-            {
-               if(!doesFieldHaveValue(field, record))
-               {
-                  continue;
-               }
-
-               Serializable primaryKey = record.getValue(table.getPrimaryKeyField());
-               String       fileName   = null;
-
-               //////////////////////////////////////////////////
-               // try to make file name from the fileNameField //
-               //////////////////////////////////////////////////
-               if(StringUtils.hasContent(fileNameField))
-               {
-                  fileName = record.getValueString(fileNameField);
-               }
-
-               if(!StringUtils.hasContent(fileName))
-               {
-                  if(StringUtils.hasContent(fileNameFormat))
-                  {
-                     @SuppressWarnings("unchecked") // instance validation should make this safe!
-                     List<String> fileNameFormatFields = (List<String>) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS);
-                     List<String> values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList();
-                     fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values);
-                  }
-               }
-
-               if(!StringUtils.hasContent(fileName))
-               {
-                  //////////////////////////////////
-                  // make default name if missing //
-                  //////////////////////////////////
-                  fileName = table.getLabel() + " " + primaryKey + " " + field.getLabel();
-
-                  if(StringUtils.hasContent(defaultExtension))
-                  {
-                     //////////////////////////////////////////
-                     // add default extension if we have one //
-                     //////////////////////////////////////////
-                     fileName += "." + defaultExtension;
-                  }
-               }
-
-               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
-               record.setDisplayValue(field.getName(), fileName);
-            }
-         }
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record)
-   {
-      boolean fieldHasValue = false;
-
-      try
-      {
-         if(record.getValue(field.getName()) != null)
-         {
-            fieldHasValue = true;
-         }
-         else if(field.getIsHeavy())
-         {
-            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) //
-            /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            Map<String, Serializable> heavyFieldLengths = (Map<String, Serializable>) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS);
-            if(heavyFieldLengths != null)
-            {
-               Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName()));
-               if(fieldLength != null && fieldLength > 0)
-               {
-                  fieldHasValue = true;
-               }
-            }
-         }
-      }
-      catch(Exception e)
-      {
-         LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record));
-      }
-
-      return fieldHasValue;
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1199,7 +1075,7 @@ public class QJavalinImplementation
          QueryAction queryAction = new QueryAction();
          QueryOutput queryOutput = queryAction.execute(queryInput);
 
-         setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
+         QValueFormatter.setBlobValuesToDownloadUrls(QContext.getQInstance().getTable(table), queryOutput.getRecords());
 
          QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS), logPairIfSlow("joins", queryJoins, SLOW_LOG_THRESHOLD_MS));
          context.result(JsonUtils.toJson(queryOutput));

From e4ae717f7f2a05b7f7f8831df621f3333c12a591 Mon Sep 17 00:00:00 2001
From: Darin Kelkhoff <darin.kelkhoff@gmail.com>
Date: Mon, 5 Jun 2023 08:30:04 -0500
Subject: [PATCH 11/11] fix missing import

---
 .../etl/streamedwithfrontend/StreamedETLExecuteStep.java        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
index 3894ea24..96933097 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java
@@ -34,6 +34,8 @@ import com.kingsrook.qqq.backend.core.actions.reporting.RecordPipe;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.audits.AuditInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;