Adding jacoco coverage reporting; switching to run mvn verify in circleci; increasing test coverage

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java

@@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
  *******************************************************************************/
 public interface FilesystemBackendModuleInterface<FILE>
 {
+   String CUSTOMIZER_FILE_POST_FILE_READ = "postFileRead";
+
    /*******************************************************************************
     ** For filesystem backends, get the module-specific action base-class, that helps
     ** with functions like listing and deleting files.

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
@@ -269,7 +270,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
    {
-      Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
+      Optional<QCodeReference> optionalCustomizer = table.getCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ);
       if(optionalCustomizer.isEmpty())
       {
          return (fileContents);

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -27,6 +27,7 @@ import java.io.InputStream;
 import java.util.List;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -67,6 +68,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
          return;
       }
 
+      s3Utils = new S3Utils();
+      s3Utils.setAmazonS3(buildAmazonS3ClientFromBackendMetaData(backendMetaData));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData backendMetaData)
+   {
       S3BackendMetaData     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
       AmazonS3ClientBuilder clientBuilder     = AmazonS3ClientBuilder.standard();
 
@@ -81,8 +93,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
          clientBuilder.setRegion(s3BackendMetaData.getRegion());
       }
 
-      s3Utils = new S3Utils();
-      s3Utils.setAmazonS3(clientBuilder.build());
+      return (clientBuilder.build());
    }
 
 

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -22,11 +22,17 @@
 package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -44,8 +50,10 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    @Test
    public void testQuery1() throws QException
    {
-      QueryRequest queryRequest = initQueryRequest();
-      QueryResult  queryResult  = new FilesystemQueryAction().execute(queryRequest);
+      QueryRequest queryRequest = new QueryRequest();
+      queryRequest.setInstance(TestUtils.defineInstance());
+      queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
+      QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
       Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
       Assertions.assertTrue(queryResult.getRecords().stream()
             .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)),
@@ -57,12 +65,36 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QueryRequest initQueryRequest() throws QInstanceValidationException
+   @Test
+   public void testQueryWithFileCustomizer() throws QException
    {
       QueryRequest queryRequest = new QueryRequest();
-      queryRequest.setInstance(TestUtils.defineInstance());
+      QInstance    instance     = TestUtils.defineInstance();
+
+      QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
+         .withName(ValueUpshifter.class.getName())
+         .withCodeType(QCodeType.JAVA)
+         .withCodeUsage(QCodeUsage.CUSTOMIZER));
+
+      queryRequest.setInstance(instance);
       queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
-      return queryRequest;
+      QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
+      Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertTrue(
+         queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
+         "All records should have their email addresses up-shifted.");
+   }
+
+
+
+   public static class ValueUpshifter implements Function<String, String>
+   {
+      @Override
+      public String apply(String s)
+      {
+         return (s.replaceAll("kingsrook.com", "KINGSROOK.COM"));
+      }
    }
 
 }

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java

@@ -0,0 +1,115 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.util.Arrays;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for BasicETLCollectSourceFileNamesFunction
+ *******************************************************************************/
+class BasicETLCollectSourceFileNamesFunctionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testOneFile() throws Exception
+   {
+      String file   = "/tmp/test1.csv";
+      String result = runTest(file);
+      assertEquals(file, result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testTwoFiles() throws Exception
+   {
+      String file1  = "/tmp/test1.csv";
+      String file2  = "/tmp/test2.csv";
+      String result = runTest(file1, file2);
+
+      //////////////////////////////////////////////////////////////////////
+      // the names go into a set, so they can come out in either order... //
+      //////////////////////////////////////////////////////////////////////
+      assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDuplicatedFile() throws Exception
+   {
+      String file   = "/tmp/test1.csv";
+      String result = runTest(file, file);
+      assertEquals(file, result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String runTest(String... fileNames) throws Exception
+   {
+      QInstance            qInstance           = TestUtils.defineInstance();
+      QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesFunction().defineStepMetaData();
+      QProcessMetaData     qProcessMetaData    = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
+      qInstance.addProcess(qProcessMetaData);
+
+      List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
+         new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
+
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setStepName(backendStepMetaData.getName());
+      runBackendStepRequest.setProcessName(qProcessMetaData.getName());
+      runBackendStepRequest.setRecords(records);
+
+      RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
+      RunBackendStepResult result               = runBackendStepAction.execute(runBackendStepRequest);
+
+      return ((String) result.getValues().get(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS));
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+
+/*******************************************************************************
+ ** Unit test for AbstractS3Action
+ *******************************************************************************/
+class AbstractS3ActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBuildAmazonS3ClientFromBackendMetaData()
+   {
+      String regionName = Regions.AP_SOUTHEAST_3.getName();
+      S3BackendMetaData s3BackendMetaData = new S3BackendMetaData()
+         .withAccessKey("Not a real access key")
+         .withSecretKey("Also not a real key")
+         .withRegion(regionName);
+      AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData);
+      assertEquals(regionName, amazonS3.getRegionName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBuildAmazonS3ClientFromBackendMetaDataWrongType()
+   {
+      assertThrows(IllegalArgumentException.class, () ->
+      {
+         new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData());
+      });
+   }
+}