CE-881 - Tests for storage action

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageAction.java

@@ -52,10 +52,13 @@ public class FilesystemStorageAction extends AbstractFilesystemAction implements
       {
          String fullPath = getFullPath(storageInput);
          File file = new File(fullPath);
+         if(!file.getParentFile().exists())
+         {
             if(!file.getParentFile().mkdirs())
             {
                throw (new QException("Could not make directory required for storing file: " + fullPath));
             }
+         }
 
          return (new FileOutputStream(fullPath));
       }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -113,7 +113,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
    /*******************************************************************************
     ** Internal accessor for the s3Utils object - should always use this, not the field.
     *******************************************************************************/
-   private S3Utils getS3Utils()
+   protected S3Utils getS3Utils()
    {
       if(s3Utils == null)
       {
@@ -257,4 +257,5 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
       getS3Utils().moveObject(bucketName, source, destination);
    }
 
+
 }

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageAction.java

@@ -53,7 +53,9 @@ public class S3StorageAction extends AbstractS3Action implements QStorageInterfa
       try
       {
          S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend();
-         AmazonS3             amazonS3             = buildAmazonS3ClientFromBackendMetaData(backend);
+         preAction(backend);
+
+         AmazonS3             amazonS3             = getS3Utils().getAmazonS3();
          String               fullPath             = getFullPath(storageInput);
          S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(amazonS3, backend.getBucketName(), fullPath);
          return (s3UploadOutputStream);
@@ -97,7 +99,9 @@ public class S3StorageAction extends AbstractS3Action implements QStorageInterfa
       try
       {
          S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend();
-         AmazonS3            amazonS3         = buildAmazonS3ClientFromBackendMetaData(backend);
+         preAction(backend);
+
+         AmazonS3            amazonS3         = getS3Utils().getAmazonS3();
          String              fullPath         = getFullPath(storageInput);
          GetObjectRequest    getObjectRequest = new GetObjectRequest(backend.getBucketName(), fullPath);
          S3Object            s3Object         = amazonS3.getObject(getObjectRequest);

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemStorageActionTest.java

@@ -0,0 +1,63 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.local.actions;
+
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemStorageAction 
+ *******************************************************************************/
+class FilesystemStorageActionTest extends FilesystemActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws Exception
+   {
+      String       data         = "Hellooo, Storage.";
+      StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withReference("test.txt");
+
+      OutputStream outputStream = new StorageAction().createOutputStream(storageInput);
+      outputStream.write(data.getBytes(StandardCharsets.UTF_8));
+      outputStream.close();
+
+      InputStream           inputStream           = new StorageAction().getInputStream(storageInput);
+      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+      inputStream.transferTo(byteArrayOutputStream);
+
+      assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8));
+
+   }
+
+}

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageActionTest.java

@@ -0,0 +1,68 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+
+/*******************************************************************************
+ ** Unit test for FilesystemStorageAction 
+ *******************************************************************************/
+public class S3StorageActionTest extends BaseS3Test
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test() throws Exception
+   {
+      String       data         = "Hellooo, Storage.";
+      StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_BLOB_S3).withReference("test.txt");
+
+      /////////////////////////////////////////////////////////////////////////
+      // work directly w/ s3 action class here, so we can set s3 utils in it //
+      /////////////////////////////////////////////////////////////////////////
+      S3StorageAction s3StorageAction = new S3StorageAction();
+      s3StorageAction.setS3Utils(getS3Utils());
+      OutputStream outputStream = s3StorageAction.createOutputStream(storageInput);
+      outputStream.write(data.getBytes(StandardCharsets.UTF_8));
+      outputStream.close();
+
+      InputStream           inputStream           = s3StorageAction.getInputStream(storageInput);
+      ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+      inputStream.transferTo(byteArrayOutputStream);
+
+      assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8));
+
+   }
+
+}

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java

@@ -0,0 +1,80 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2024.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
+
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
+import io.github.cdimascio.dotenv.Dotenv;
+import org.junit.jupiter.api.Test;
+
+
+/*******************************************************************************
+ ** Unit test for S3UploadOutputStream 
+ *******************************************************************************/
+class S3UploadOutputStreamTest extends BaseTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void test() throws IOException
+   {
+      Dotenv dotenv = Dotenv.load();
+
+      BasicAWSCredentials credentials = new BasicAWSCredentials(dotenv.get("NFONE_S3_ACCESS_KEY"), dotenv.get("NFONE_S3_SECRET_KEY"));
+      AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard()
+         .withCredentials(new AWSStaticCredentialsProvider(credentials))
+         .withRegion(dotenv.get("NFONE_S3_REGION"))
+         .build();
+
+      String bucketName = "bucket-upload-archive-nf-one-dev";
+      String key        = "uploader-tests/" + Instant.now().toString() + ".txt";
+
+      // S3UploadOutputStream outputStream = new S3UploadOutputStream(amazonS3, bucketName, key);
+      // FileOutputStream outputStream = new FileOutputStream("/tmp/file.json");
+      ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+
+      outputStream.write("[\n1".getBytes(StandardCharsets.UTF_8));
+      for(int i = 2; i <= 1_000_000; i++)
+      {
+         outputStream.write((",\n" + i).getBytes(StandardCharsets.UTF_8));
+      }
+      outputStream.write("\n]\n".getBytes(StandardCharsets.UTF_8));
+      outputStream.close();
+
+      S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(amazonS3, bucketName, key);
+      s3UploadOutputStream.write(outputStream.toByteArray(), 0, 5 * 1024 * 1024);
+      s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024);
+      s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024);
+      s3UploadOutputStream.close();
+   }
+
+}