Merged feature/sftp-import-support into integration

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java

@@ -35,6 +35,7 @@ import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
 import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
 import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
 import org.jsoup.Jsoup;
+import org.jsoup.helper.W3CDom;
 import org.jsoup.nodes.Document;
 
 
@@ -67,6 +68,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
          //////////////////////////////////////////////////////////////////
          Document document = Jsoup.parse(input.getHtml());
          document.outputSettings().syntax(Document.OutputSettings.Syntax.xml);
+         org.w3c.dom.Document w3cDoc = new W3CDom().fromJsoup(document);
 
          //////////////////////////////
          // convert the XHTML to PDF //
@@ -74,7 +76,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
          PdfRendererBuilder builder = new PdfRendererBuilder();
          builder.toStream(input.getOutputStream());
          builder.useFastMode();
-         builder.withHtmlContent(document.html(), input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
+         builder.withW3cDocument(w3cDoc, input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
 
          try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java

@@ -0,0 +1,91 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.values;
+
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
+
+
+/*******************************************************************************
+ ** Basic implementation of a possible value provider, for where there's a limited
+ ** set of possible source objects - so you just have to define how to make one
+ ** PV from a source object, how to list all of the source objects, and how to
+ ** look up a PV from an id.
+ *******************************************************************************/
+public abstract class BasicCustomPossibleValueProvider<S, ID extends Serializable> implements QCustomPossibleValueProvider<ID>
+{
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract QPossibleValue<ID> makePossibleValue(S sourceObject);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract S getSourceObject(Serializable id);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected abstract List<S> getAllSourceObjects();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public QPossibleValue<ID> getPossibleValue(Serializable idValue)
+   {
+      S sourceObject = getSourceObject(idValue);
+      if(sourceObject == null)
+      {
+         return (null);
+      }
+
+      return makePossibleValue(sourceObject);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public List<QPossibleValue<ID>> search(SearchPossibleValueSourceInput input) throws QException
+   {
+      List<QPossibleValue<ID>> allPossibleValues = new ArrayList<>();
+      List<S>                  allSourceObjects  = getAllSourceObjects();
+      for(S sourceObject : allSourceObjects)
+      {
+         allPossibleValues.add(makePossibleValue(sourceObject));
+      }
+
+      return completeCustomPVSSearch(input, allPossibleValues);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java

@@ -66,6 +66,7 @@ public enum WidgetType
    // record view/edit widgets //
    //////////////////////////////
    CHILD_RECORD_LIST("childRecordList"),
+   CUSTOM_COMPONENT("customComponent"),
    DYNAMIC_FORM("dynamicForm"),
    DATA_BAG_VIEWER("dataBagViewer"),
    PIVOT_TABLE_SETUP("pivotTableSetup"),

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java

@@ -27,11 +27,9 @@ import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
 import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
-import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
+import com.kingsrook.qqq.backend.core.actions.values.BasicCustomPossibleValueProvider;
 import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -40,26 +38,16 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
  ** possible-value source provider for the `Tables` PVS - a list of all tables
  ** in an application/qInstance.
  *******************************************************************************/
-public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider<String>
+public class TablesCustomPossibleValueProvider extends BasicCustomPossibleValueProvider<QTableMetaData, String>
 {
 
    /***************************************************************************
     **
     ***************************************************************************/
    @Override
-   public QPossibleValue<String> getPossibleValue(Serializable idValue)
+   protected QPossibleValue<String> makePossibleValue(QTableMetaData sourceObject)
    {
-      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue));
+      return (new QPossibleValue<>(sourceObject.getName(), sourceObject.getLabel()));
-      if(table != null && !table.getIsHidden())
-      {
-         PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
-         if(PermissionCheckResult.ALLOW.equals(permissionCheckResult))
-         {
-            return (new QPossibleValue<>(table.getName(), table.getLabel()));
-         }
-      }
-
-      return null;
    }
 
 
@@ -68,22 +56,54 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr
     **
     ***************************************************************************/
    @Override
-   public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
+   protected QTableMetaData getSourceObject(Serializable id)
    {
-      /////////////////////////////////////////////////////////////////////////////////////
+      QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(id));
-      // build all of the possible values (note, will be filtered by user's permissions) //
+      return isTableAllowed(table) ? table : null;
-      /////////////////////////////////////////////////////////////////////////////////////
+   }
-      List<QPossibleValue<String>> allPossibleValues = new ArrayList<>();
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   protected List<QTableMetaData> getAllSourceObjects()
+   {
+      ArrayList<QTableMetaData> rs = new ArrayList<>();
       for(QTableMetaData table : QContext.getQInstance().getTables().values())
       {
-         QPossibleValue<String> possibleValue = getPossibleValue(table.getName());
+         if(isTableAllowed(table))
-         if(possibleValue != null)
          {
-            allPossibleValues.add(possibleValue);
+            rs.add(table);
          }
       }
+      return rs;
+   }
 
-      return completeCustomPVSSearch(input, allPossibleValues);
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private boolean isTableAllowed(QTableMetaData table)
+   {
+      if(table == null)
+      {
+         return (false);
+      }
+
+      if(table.getIsHidden())
+      {
+         return (false);
+      }
+
+      PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
+      if(!PermissionCheckResult.ALLOW.equals(permissionCheckResult))
+      {
+         return (false);
+      }
+
+      return (true);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java

@@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
 
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
 import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
@@ -45,6 +46,7 @@ public class TablesPossibleValueSourceMetaDataProvider
    {
       QPossibleValueSource possibleValueSource = new QPossibleValueSource()
          .withName(NAME)
+         .withIdType(QFieldType.STRING)
          .withType(QPossibleValueSourceType.CUSTOM)
          .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class))
          .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java

@@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
-      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
-
       int rowsAdded     = 0;
       int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java

@@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
    @Override
    public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
    {
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // for headless-bulk load (e.g., sftp import), set up the process tracer's key record //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
+
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
       // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) //
       ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java

@@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
 import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 import org.apache.commons.lang3.BooleanUtils;
 
@@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             //////////////////////////////////////////////////////////////////////////////
             if(savedBulkLoadProfileRecord == null)
             {
-               throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
+               throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input.  Unable to perform headless bulk load"));
             }
 
             SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
+            if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson()))
+            {
+               throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty.  Unable to perform headless bulk load"));
+            }
 
             try
             {
@@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             }
             catch(Exception e)
             {
-               throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
+               throw (new QUserFacingException("Error processing Bulk Load Profile record.  Unable to perform headless bulk load", e));
             }
          }
          else
@@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
             }
          }
       }
+      catch(QUserFacingException ufe)
+      {
+         LOG.warn("User-facing error in bulk insert receive mapping", ufe);
+         throw ufe;
+      }
       catch(Exception e)
       {
          LOG.warn("Error in bulk insert receive mapping", e);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java

@@ -79,6 +79,7 @@ class ConvertHtmlToPdfActionTest extends BaseTest
                      </h1>
                      <div class="myclass">
                          <p>This is a test of converting HTML to PDF!!</p>
+                         <p>This is &nbsp; a line with &bull; some entities &lt;</p>
                          <p style="font-family: SF-Pro; font-size: 24px;">(btw, is this in SF-Pro???)</p>
                      </div>
                  </div>