CE-881 - Add new StorageAction to backends - e.g., for streaming data into a backend's data store. implemented for in-memory, filesystem, and s3

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QStorageInterface.java

@@ -0,0 +1,49 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.interfaces;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+
+
+/*******************************************************************************
+ ** Interface for actions that a backend can perform, based on streaming data
+ ** into the backend's storage. 
+ *******************************************************************************/
+public interface QStorageInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   OutputStream createOutputStream(StorageInput storageInput) throws QException;
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   InputStream getInputStream(StorageInput storageInput) throws QException;
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/StorageAction.java

@@ -0,0 +1,96 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.actions.tables;
+
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import com.kingsrook.qqq.backend.core.actions.ActionHelper;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
+import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
+
+
+/*******************************************************************************
+ ** Action to do (generally, "mass") storage operations in a backend.
+ **
+ ** e.g., store a (potentially large) file - specifically - by working with it
+ ** as either an InputStream or OutputStream.
+ **
+ ** May not be implemented in all backends.
+ **
+ *******************************************************************************/
+public class StorageAction
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public OutputStream createOutputStream(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.createOutputStream(storageInput));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public InputStream getInputStream(StorageInput storageInput) throws QException
+   {
+      QBackendModuleInterface qBackendModuleInterface = preAction(storageInput);
+      QStorageInterface       storageInterface        = qBackendModuleInterface.getStorageInterface();
+      return (storageInterface.getInputStream(storageInput));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private QBackendModuleInterface preAction(StorageInput storageInput) throws QException
+   {
+      ActionHelper.validateSession(storageInput);
+
+      if(storageInput.getTableName() == null)
+      {
+         throw (new QException("Table name was not specified in query input"));
+      }
+
+      QTableMetaData table = storageInput.getTable();
+      if(table == null)
+      {
+         throw (new QException("A table named [" + storageInput.getTableName() + "] was not found in the active QInstance"));
+      }
+
+      QBackendMetaData         backend                  = storageInput.getBackend();
+      QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
+      QBackendModuleInterface  qModule                  = qBackendModuleDispatcher.getQBackendModule(backend);
+      return (qModule);
+   }
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java

@@ -0,0 +1,77 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.model.actions.tables.storage;
+
+
+import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
+
+
+/*******************************************************************************
+ **
+ *******************************************************************************/
+public class StorageInput extends AbstractTableActionInput
+{
+   private String reference;
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public StorageInput(String storageTableName)
+   {
+      super();
+      setTableName(storageTableName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for reference
+    *******************************************************************************/
+   public String getReference()
+   {
+      return (this.reference);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for reference
+    *******************************************************************************/
+   public void setReference(String reference)
+   {
+      this.reference = reference;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for reference
+    *******************************************************************************/
+   public StorageInput withReference(String reference)
+   {
+      this.reference = reference;
+      return (this);
+   }
+
+}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/QBackendModuleInterface.java

@@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -129,6 +130,16 @@ public interface QBackendModuleInterface
       return null;
    }
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   default QStorageInterface getStorageInterface()
+   {
+      throwNotImplemented("StorageInterface");
+      return null;
+   }
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryBackendModule.java

@@ -26,6 +26,7 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
 import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@@ -117,4 +118,14 @@ public class MemoryBackendModule implements QBackendModuleInterface
       return (new MemoryDeleteAction());
    }
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QStorageInterface getStorageInterface()
+   {
+      return (new MemoryStorageAction());
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryStorageAction.java

@@ -0,0 +1,149 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.modules.backend.implementations.memory;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
+import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** implementation of bulk-storage interface, for the memory backend module.
+ **
+ ** Requires table to have (at least?) 2 fields - a STRING primary key and a
+ ** BLOB to store bytes.
+ *******************************************************************************/
+public class MemoryStorageAction implements QStorageInterface
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public OutputStream createOutputStream(StorageInput storageInput)
+   {
+      return new MemoryStorageOutputStream(storageInput.getTableName(), storageInput.getReference());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public InputStream getInputStream(StorageInput storageInput) throws QException
+   {
+      QRecord record = new GetAction().executeForRecord(new GetInput(storageInput.getTableName()).withPrimaryKey(storageInput.getReference()));
+      if(record == null)
+      {
+         throw (new QNotFoundException("Could not find input stream for [" + storageInput.getTableName() + "][" + storageInput.getReference() + "]"));
+      }
+
+      QFieldMetaData blobField = getBlobField(storageInput.getTableName());
+      return (new ByteArrayInputStream(record.getValueByteArray(blobField.getName())));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QFieldMetaData getBlobField(String tableName) throws QException
+   {
+      Optional<QFieldMetaData> firstBlobField = QContext.getQInstance().getTable(tableName).getFields().values().stream().filter(f -> QFieldType.BLOB.equals(f.getType())).findFirst();
+      if(firstBlobField.isEmpty())
+      {
+         throw (new QException("Could not find a blob field in table [" + tableName + "]"));
+      }
+      return firstBlobField.get();
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static class MemoryStorageOutputStream extends ByteArrayOutputStream
+   {
+      private final String tableName;
+      private final String reference;
+
+
+
+      /*******************************************************************************
+       ** Constructor
+       **
+       *******************************************************************************/
+      public MemoryStorageOutputStream(String tableName, String reference)
+      {
+         this.tableName = tableName;
+         this.reference = reference;
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void close() throws IOException
+      {
+         super.close();
+
+         try
+         {
+            QFieldMetaData blobField = getBlobField(tableName);
+            InsertOutput insertOutput = new InsertAction().execute(new InsertInput(tableName).withRecord(new QRecord()
+               .withValue(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), reference)
+               .withValue(blobField.getName(), toByteArray())));
+
+            if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors()))
+            {
+               throw(new IOException("Error storing stream into memory storage: " + StringUtils.joinWithCommasAndAnd(insertOutput.getRecords().get(0).getErrors().stream().map(e -> e.getMessage()).toList())));
+            }
+         }
+         catch(Exception e)
+         {
+            throw new IOException("Wrapped QException", e);
+         }
+      }
+   }
+}