Adding filesystem writing - used by javalin to store uploaded files; done async, via new base class for actions

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionBiConsumer.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions;
+
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
+
+
+/*******************************************************************************
+ ** Base class for QQQ Actions (both framework and application defined) that
+ ** have a signature like a BiConsumer - taking both Input and Output objects as
+ ** parameters, with void output.
+ *******************************************************************************/
+public abstract class AbstractQActionBiConsumer<I extends AbstractActionInput, O extends AbstractActionOutput>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract void execute(I input, O output) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Future<Void> executeAsync(I input, O output)
+   {
+      CompletableFuture<Void> completableFuture = new CompletableFuture<>();
+      Executors.newCachedThreadPool().submit(() ->
+      {
+         try
+         {
+            execute(input, output);
+            completableFuture.complete(null);
+         }
+         catch(QException e)
+         {
+            completableFuture.completeExceptionally(e);
+         }
+      });
+      return (completableFuture);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/AbstractQActionFunction.java

@@ -0,0 +1,69 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions;
+
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionOutput;
+
+
+/*******************************************************************************
+ ** Base class for QQQ Actions (both framework and application defined) that
+ ** have a signature like a Function - taking an Input object as a parameter,
+ ** and returning an Output object.
+ *******************************************************************************/
+public abstract class AbstractQActionFunction<I extends AbstractActionInput, O extends AbstractActionOutput>
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public abstract O execute(I input) throws QException;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public Future<O> executeAsync(I input)
+   {
+      CompletableFuture<O> completableFuture = new CompletableFuture<>();
+      Executors.newCachedThreadPool().submit(() ->
+      {
+         try
+         {
+            O output = execute(input);
+            completableFuture.complete(output);
+         }
+         catch(QException e)
+         {
+            completableFuture.completeExceptionally(e);
+         }
+      });
+      return (completableFuture);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
@@ -39,7 +40,7 @@ import org.apache.logging.log4j.Logger;
  ** Action to insert one or more records.
  **
  *******************************************************************************/
-public class InsertAction
+public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOutput>
 {
    private static final Logger LOG = LogManager.getLogger(InsertAction.class);
 
@@ -48,6 +49,7 @@ public class InsertAction
    /*******************************************************************************
     **
     *******************************************************************************/
+   @Override
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
       ActionHelper.validateSession(insertInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -130,6 +130,8 @@ public class QInstanceValidator
          validateApps(qInstance);
          validatePossibleValueSources(qInstance);
          validateQueuesAndProviders(qInstance);
+
+         validateUniqueTopLevelNames(qInstance);
       }
       catch(Exception e)
       {
@@ -146,6 +148,45 @@ public class QInstanceValidator
 
 
 
+   /*******************************************************************************
+    ** there can be some unexpected bad-times if you have a table and process, or
+    ** table and app (etc) with the same name (e.g., in app tree building).  So,
+    ** just go ahead and make sure those are all unique.
+    *******************************************************************************/
+   private void validateUniqueTopLevelNames(QInstance qInstance)
+   {
+      String      suffix    = " is not unique across tables, processes, and apps (but it needs to be)";
+      Set<String> usedNames = new HashSet<>();
+      if(qInstance.getTables() != null)
+      {
+         for(String tableName : qInstance.getTables().keySet())
+         {
+            assertCondition(!usedNames.contains(tableName), "Table name " + tableName + suffix);
+            usedNames.add(tableName);
+         }
+      }
+
+      if(qInstance.getProcesses() != null)
+      {
+         for(String processName : qInstance.getProcesses().keySet())
+         {
+            assertCondition(!usedNames.contains(processName), "Process name " + processName + suffix);
+            usedNames.add(processName);
+         }
+      }
+
+      if(qInstance.getApps() != null)
+      {
+         for(String appName : qInstance.getApps().keySet())
+         {
+            assertCondition(!usedNames.contains(appName), "App name " + appName + suffix);
+            usedNames.add(appName);
+         }
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -311,7 +352,7 @@ public class QInstanceValidator
             ///////////////////////////////
             // validate the record label //
             ///////////////////////////////
-            if(table.getRecordLabelFields() != null)
+            if(table.getRecordLabelFields() != null && table.getFields() != null)
             {
                for(String recordLabelField : table.getRecordLabelFields())
                {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -406,6 +406,16 @@ public class QRecord implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public byte[] getValueByteArray(String fieldName)
+   {
+      return (ValueUtils.getValueAsByteArray(values.get(fieldName)));
+   }
+
+
+
    /*******************************************************************************
     ** Getter for backendDetails
     **

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/metadata/Auth0AuthenticationMetaData.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.modules.authentication.metadata;
 
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
 
 
@@ -32,6 +33,11 @@ public class Auth0AuthenticationMetaData extends QAuthenticationMetaData
 {
    private String baseUrl;
    private String clientId;
+
+   ////////////////////////////////////////////////////////////////////////////////////////
+   // keep this secret, on the server - don't let it be serialized and sent to a client! //
+   ////////////////////////////////////////////////////////////////////////////////////////
+   @JsonIgnore
    private String clientSecret;
 
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java

@@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.utils;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -559,4 +560,29 @@ public class ValueUtils
          throw (new QValueException("Value [" + value + "] could not be converted to a LocalTime.", e));
       }
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static byte[] getValueAsByteArray(Serializable value)
+   {
+      if(value == null)
+      {
+         return (null);
+      }
+      else if(value instanceof byte[] ba)
+      {
+         return (ba);
+      }
+      else if(value instanceof String s)
+      {
+         return (s.getBytes(StandardCharsets.UTF_8));
+      }
+      else
+      {
+         throw (new QValueException("Unsupported class " + value.getClass().getName() + " for converting to ByteArray."));
+      }
+   }
 }

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/QInstanceTest.java

@@ -0,0 +1,84 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.metadata;
+
+
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for QInstance
+ *******************************************************************************/
+class QInstanceTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetTablePath() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      GetInput getInput = new GetInput(qInstance);
+      getInput.setSession(new QSession());
+
+      String tablePath = qInstance.getTablePath(getInput, TestUtils.TABLE_NAME_PERSON);
+      assertEquals("/peopleApp/person", tablePath);
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // call again (to make sure getting from memoization works - verify w/ breakpoint...) //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      tablePath = qInstance.getTablePath(getInput, TestUtils.TABLE_NAME_PERSON);
+      assertEquals("/peopleApp/person", tablePath);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetTablePathNotInAnyApp() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+
+      GetInput getInput = new GetInput(qInstance);
+      getInput.setSession(new QSession());
+
+      String tablePath = qInstance.getTablePath(getInput, "notATable");
+      assertNull(tablePath);
+
+      ////////////////////////////////////////////////////////////////////////////////////////
+      // call again (to make sure getting from memoization works - verify w/ breakpoint...) //
+      ////////////////////////////////////////////////////////////////////////////////////////
+      tablePath = qInstance.getTablePath(getInput, "notATable");
+      assertNull(tablePath);
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/LoadViaInsertOrUpdateStepTest.java

@@ -0,0 +1,85 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.utils.TestUtils;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertOrUpdateStep
+ *******************************************************************************/
+class LoadViaInsertOrUpdateStepTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @BeforeEach
+   @AfterEach
+   void beforeAndAfterEach()
+   {
+      MemoryRecordStore.getInstance().reset();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws QException
+   {
+      QInstance qInstance = TestUtils.defineInstance();
+      List<QRecord> existingRecordList = List.of(
+         new QRecord().withValue("id", 47).withValue("firstName", "Tom")
+      );
+      TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), existingRecordList);
+
+      List<QRecord> inputRecordList = List.of(
+         new QRecord().withValue("id", 47).withValue("firstName", "Tim"),
+         new QRecord().withValue("firstName", "John")
+      );
+      RunBackendStepInput input = new RunBackendStepInput(qInstance);
+      input.setSession(new QSession());
+      input.setRecords(inputRecordList);
+      input.addValue(LoadViaInsertOrUpdateStep.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_MEMORY);
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      new LoadViaInsertOrUpdateStep().run(input, output);
+
+      List<QRecord> qRecords = TestUtils.queryTable(qInstance, TestUtils.TABLE_NAME_PERSON_MEMORY);
+      assertEquals(2, qRecords.size());
+   }
+
+}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -129,7 +129,7 @@ public class TestUtils
    public static final String PROCESS_NAME_GREET_PEOPLE_INTERACTIVE = "greetInteractive";
    public static final String PROCESS_NAME_INCREASE_BIRTHDATE       = "increaseBirthdate";
    public static final String PROCESS_NAME_ADD_TO_PEOPLES_AGE       = "addToPeoplesAge";
-   public static final String PROCESS_NAME_BASEPULL                 = "basepullTest";
+   public static final String PROCESS_NAME_BASEPULL                 = "basepullTestProcess";
    public static final String PROCESS_NAME_RUN_SHAPES_PERSON_REPORT = "runShapesPersonReport";
    public static final String TABLE_NAME_PERSON_FILE                = "personFile";
    public static final String TABLE_NAME_PERSON_MEMORY              = "personMemory";

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -33,6 +33,8 @@ import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -45,6 +47,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
+import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.NotImplementedException;
@@ -328,4 +331,40 @@ public abstract class AbstractBaseFilesystemAction<FILE>
       }
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected InsertOutput executeInsert(InsertInput insertInput) throws QException
+   {
+      try
+      {
+         InsertOutput     output  = new InsertOutput();
+         QTableMetaData   table   = insertInput.getTable();
+         QBackendMetaData backend = insertInput.getBackend();
+
+         AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table);
+         if(tableDetails.getCardinality().equals(Cardinality.ONE))
+         {
+            for(QRecord record : insertInput.getRecords())
+            {
+               String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + record.getValueString("fileName"));
+               writeFile(backend, fullPath, record.getValueByteArray("contents"));
+               record.addBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fullPath);
+               output.addRecord(record);
+            }
+         }
+         else
+         {
+            throw (new NotImplementedException("Insert is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality()));
+         }
+
+         return (output);
+      }
+      catch(Exception e)
+      {
+         throw new QException("Error executing insert: " + e.getMessage(), e);
+      }
+   }
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertAction.java

@@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
-import org.apache.commons.lang.NotImplementedException;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class FilesystemInsertAction implements InsertInterface
+public class FilesystemInsertAction extends AbstractFilesystemAction implements InsertInterface
 {
 
    /*******************************************************************************
@@ -40,23 +39,7 @@ public class FilesystemInsertAction implements InsertInterface
     *******************************************************************************/
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
-      throw new NotImplementedException("Filesystem insert not implemented");
+      return (super.executeInsert(insertInput));
-      /*
-      try
-      {
-         InsertResult rs = new InsertResult();
-         QTableMetaData table = insertRequest.getTable();
-
-         List<QRecord> recordsWithStatus = new ArrayList<>();
-         rs.setRecords(recordsWithStatus);
-
-         // return rs;
-      }
-      catch(Exception e)
-      {
-         throw new QException("Error executing insert: " + e.getMessage(), e);
-      }
-      */
    }
 
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertAction.java

@@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
-import org.apache.commons.lang.NotImplementedException;
 
 
 /*******************************************************************************
  **
  *******************************************************************************/
-public class S3InsertAction implements InsertInterface
+public class S3InsertAction extends AbstractS3Action implements InsertInterface
 {
 
    /*******************************************************************************
@@ -40,24 +39,7 @@ public class S3InsertAction implements InsertInterface
     *******************************************************************************/
    public InsertOutput execute(InsertInput insertInput) throws QException
    {
-      throw new NotImplementedException("S3 insert not implemented");
+      return (super.executeInsert(insertInput));
-      /*
-      try
-      {
-         InsertResult rs = new InsertResult();
-         QTableMetaData table = insertRequest.getTable();
-
-         List<QRecord> recordsWithStatus = new ArrayList<>();
-         rs.setRecords(recordsWithStatus);
-
-
-         // return rs;
-      }
-      catch(Exception e)
-      {
-         throw new QException("Error executing insert: " + e.getMessage(), e);
-      }
-      */
    }
 
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java

@@ -62,7 +62,9 @@ public class TestUtils
 
    public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json";
    public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV  = "person-local-csv";
+   public static final String TABLE_NAME_BLOB_LOCAL_FS        = "local-blob";
    public static final String TABLE_NAME_PERSON_S3            = "person-s3";
+   public static final String TABLE_NAME_BLOB_S3              = "s3-blob";
    public static final String TABLE_NAME_PERSON_MOCK          = "person-mock";
 
    public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed";
@@ -132,8 +134,10 @@ public class TestUtils
       qInstance.addBackend(defineLocalFilesystemBackend());
       qInstance.addTable(defineLocalFilesystemJSONPersonTable());
       qInstance.addTable(defineLocalFilesystemCSVPersonTable());
+      qInstance.addTable(defineLocalFilesystemBlobTable());
       qInstance.addBackend(defineS3Backend());
       qInstance.addTable(defineS3CSVPersonTable());
+      qInstance.addTable(defineS3BlobTable());
       qInstance.addBackend(defineMockBackend());
       qInstance.addTable(defineMockPersonTable());
       qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
@@ -230,6 +234,46 @@ public class TestUtils
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineLocalFilesystemBlobTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_BLOB_LOCAL_FS)
+         .withLabel("Blob")
+         .withBackendName(defineLocalFilesystemBackend().getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB))
+         .withBackendDetails(new FilesystemTableBackendDetails()
+            .withBasePath("blobs")
+            .withCardinality(Cardinality.ONE)
+         );
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static QTableMetaData defineS3BlobTable()
+   {
+      return new QTableMetaData()
+         .withName(TABLE_NAME_BLOB_S3)
+         .withLabel("Blob S3")
+         .withBackendName(defineS3Backend().getName())
+         .withPrimaryKeyField("fileName")
+         .withField(new QFieldMetaData("fileName", QFieldType.STRING))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB))
+         .withBackendDetails(new S3TableBackendDetails()
+            .withBasePath("blobs")
+            .withCardinality(Cardinality.ONE)
+         );
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -296,7 +340,7 @@ public class TestUtils
    {
       QProcessMetaData qProcessMetaData = new StreamedETLProcess().defineProcessMetaData();
       qProcessMetaData.setName(PROCESS_NAME_STREAMED_ETL);
-      QBackendStepMetaData backendStep  = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
+      QBackendStepMetaData backendStep = qProcessMetaData.getBackendStep(StreamedETLProcess.FUNCTION_NAME_ETL);
       backendStep.setCode(new QCodeReference(StreamedETLFilesystemBackendStep.class));
 
       backendStep.getInputMetaData().getFieldThrowing(StreamedETLProcess.FIELD_SOURCE_TABLE).setDefaultValue(TABLE_NAME_PERSON_LOCAL_FS_CSV);

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertActionTest.java

@@ -22,11 +22,24 @@
 package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.NotImplementedException;
 import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -39,9 +52,40 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
     **
     *******************************************************************************/
    @Test
-   public void test() throws QException
+   public void testCardinalityOne() throws QException, IOException
    {
-      assertThrows(NotImplementedException.class, () -> new FilesystemInsertAction().execute(new InsertInput()));
+      QInstance   qInstance   = TestUtils.defineInstance();
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("fileName", "file1.txt").withValue("contents", "Hello, World")
+      ));
+      InsertOutput insertOutput = new InsertAction().execute(insertInput);
+      assertThat(insertOutput.getRecords())
+         .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH))
+         .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
+
+      assertEquals("Hello, World", FileUtils.readFileToString(new File(insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testCardinalityMany() throws QException, IOException
+   {
+      QInstance   qInstance   = TestUtils.defineInstance();
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("id", "1").withValue("firstName", "Bob")
+      ));
+      assertThatThrownBy(() -> new InsertAction().execute(insertInput))
+         .hasRootCauseInstanceOf(NotImplementedException.class);
    }
 
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java

@@ -22,12 +22,25 @@
 package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
 
 
+import java.io.IOException;
+import java.util.List;
+import com.amazonaws.services.s3.model.S3Object;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.NotImplementedException;
 import org.junit.jupiter.api.Test;
-import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 
 /*******************************************************************************
@@ -40,9 +53,46 @@ public class S3InsertActionTest extends BaseS3Test
     **
     *******************************************************************************/
    @Test
-   public void test() throws QException
+   public void testCardinalityOne() throws QException, IOException
    {
-      assertThrows(NotImplementedException.class, () -> new S3InsertAction().execute(new InsertInput()));
+      QInstance qInstance = TestUtils.defineInstance();
+
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
+      ));
+
+      S3InsertAction insertAction = new S3InsertAction();
+      insertAction.setS3Utils(getS3Utils());
+
+      InsertOutput insertOutput = insertAction.execute(insertInput);
+      assertThat(insertOutput.getRecords())
+         .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
+
+      String   fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
+      S3Object object   = getAmazonS3().getObject(BUCKET_NAME, fullPath);
+      List     lines    = IOUtils.readLines(object.getObjectContent());
+      assertEquals("Hi, Bob.", lines.get(0));
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testCardinalityMany() throws QException, IOException
+   {
+      QInstance   qInstance   = TestUtils.defineInstance();
+      InsertInput insertInput = new InsertInput(qInstance);
+      insertInput.setSession(new QSession());
+      insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_S3);
+      insertInput.setRecords(List.of(
+         new QRecord().withValue("id", "1").withValue("firstName", "Bob")
+      ));
+      assertThatThrownBy(() -> new InsertAction().execute(insertInput))
+         .hasRootCauseInstanceOf(NotImplementedException.class);
+   }
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java

@@ -83,7 +83,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
       rs.setRecords(outputRecords);
 
       /////////////////////////////////////////////////////////////////////////////////////////////
-      // we want to do batch updates.  But, since we only update the columns columns that        //
+      // we want to do batch updates.  But, since we only update the columns that                //
       // are present in each record, it means we may have different update SQL for each          //
       // record.  So, we will first "hash" up the records by their list of fields being updated. //
       /////////////////////////////////////////////////////////////////////////////////////////////

qqq-dev-tools/bin/createTableToEntityFields.groovy

@@ -12,6 +12,7 @@ boolean writeTableMetaData = args.length > 2 ? args[2] : false;
 def reader = new BufferedReader(new InputStreamReader(System.in))
 String line
 String allFieldNames = ""
+String dataFieldNames = ""
 List<String> fieldNameList = new ArrayList<>();
 List<String> fieldTypeList = new ArrayList<>();
 
@@ -64,6 +65,15 @@ while((line = reader.readLine()) != null)
    """.formatted(attributes, fieldType, fieldName))
 
    allFieldNames += '"' + fieldName + '",'
+   if(!fieldName.equals("id") && !fieldName.equals("createDate") && !fieldName.equals("modifyDate"))
+   {
+      dataFieldNames += '"' + fieldName + '",'
+   }
+}
+
+if(dataFieldNames.length() > 0)
+{
+   dataFieldNames = dataFieldNames.substring(0, dataFieldNames.length() - 1);
 }
 
 if(writeWholeClass)
@@ -162,22 +172,35 @@ if(writeTableMetaData)
          .withRecordLabelFields("TODO")
          .withBackendName(TODO)
          .withPrimaryKeyField("id")
+         .withFieldsFromEntity({className}.class)
          .withBackendDetails(new RDBMSTableBackendDetails()
             .withTableName("{tableName}")
-         );
+         )
+         .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
+         .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of({dataFieldNames})))
+         .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));
+         
+      QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData);
+         
       return (qTableMetaData);
    }
 
    """
       .replaceAll("\\{className}", className)
       .replaceAll("\\{tableName}", tableName)
+      .replaceAll("\\{dataFieldNames}", dataFieldNames)
    );
 }
 
 println(output);
 println()
-println("All field names (e.g., for sections):")
+
-println(allFieldNames);
+if(!writeTableMetaData)
+{
+   println("All field names (e.g., for sections):")
+   allFieldNames = allFieldNames.substring(0, allFieldNames.length() - 1);
+   println(allFieldNames);
+}
 println();
 
 

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -123,7 +123,8 @@ public class QJavalinImplementation
    private static final int    SESSION_COOKIE_AGE     = 60 * 60 * 24;
    private static final String SESSION_ID_COOKIE_NAME = "sessionId";
 
-   static QInstance qInstance;
+   static QInstance        qInstance;
+   static QJavalinMetaData javalinMetaData = new QJavalinMetaData();
 
    private static Supplier<QInstance> qInstanceHotSwapSupplier;
    private static long                lastQInstanceHotSwapMillis;
@@ -1063,4 +1064,25 @@ public class QJavalinImplementation
       QJavalinImplementation.qInstanceHotSwapSupplier = qInstanceHotSwapSupplier;
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for javalinMetaData
+    **
+    *******************************************************************************/
+   public QJavalinMetaData getJavalinMetaData()
+   {
+      return javalinMetaData;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for javalinMetaData
+    **
+    *******************************************************************************/
+   public void setJavalinMetaData(QJavalinMetaData javalinMetaData)
+   {
+      QJavalinImplementation.javalinMetaData = javalinMetaData;
+   }
 }

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinMetaData.java

@@ -0,0 +1,66 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.javalin;
+
+
+/*******************************************************************************
+ ** MetaData specific to a QQQ Javalin server.
+ *******************************************************************************/
+public class QJavalinMetaData
+{
+   private String uploadedFileArchiveTableName;
+
+
+
+   /*******************************************************************************
+    ** Getter for uploadedFileArchiveTableName
+    **
+    *******************************************************************************/
+   public String getUploadedFileArchiveTableName()
+   {
+      return uploadedFileArchiveTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for uploadedFileArchiveTableName
+    **
+    *******************************************************************************/
+   public void setUploadedFileArchiveTableName(String uploadedFileArchiveTableName)
+   {
+      this.uploadedFileArchiveTableName = uploadedFileArchiveTableName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for uploadedFileArchiveTableName
+    **
+    *******************************************************************************/
+   public QJavalinMetaData withUploadedFileArchiveTableName(String uploadedFileArchiveTableName)
+   {
+      this.uploadedFileArchiveTableName = uploadedFileArchiveTableName;
+      return (this);
+   }
+
+}

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java

@@ -22,10 +22,12 @@
 package com.kingsrook.qqq.backend.javalin;
 
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -42,6 +44,8 @@ import com.kingsrook.qqq.backend.core.actions.async.AsyncJobStatus;
 import com.kingsrook.qqq.backend.core.actions.async.JobGoingAsyncException;
 import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
 import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
 import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
@@ -49,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState;
 import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
 import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -290,6 +295,8 @@ public class QJavalinProcessHandler
          TempFileStateProvider.getInstance().put(key, qUploadedFile);
          LOG.info("Stored uploaded file in TempFileStateProvider under key: " + key);
          runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
+
+         archiveUploadedFile(runProcessInput, qUploadedFile);
       }
 
       /////////////////////////////////////////////////////////////
@@ -319,6 +326,28 @@ public class QJavalinProcessHandler
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static void archiveUploadedFile(RunProcessInput runProcessInput, QUploadedFile qUploadedFile)
+   {
+      String fileName = new QValueFormatter().formatDate(LocalDate.now())
+         + File.separator + runProcessInput.getProcessName()
+         + File.separator + qUploadedFile.getFilename();
+
+      InsertInput insertInput = new InsertInput(QJavalinImplementation.qInstance);
+      insertInput.setSession(runProcessInput.getSession());
+      insertInput.setTableName(QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName());
+      insertInput.setRecords(List.of(new QRecord()
+         .withValue("fileName", fileName)
+         .withValue("contents", qUploadedFile.getBytes())
+      ));
+
+      new InsertAction().executeAsync(insertInput);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/