Adding jacoco coverage reporting; switching to run mvn verify in circleci; increasing test coverage

.circleci/config.yml

@@ -57,7 +57,7 @@ jobs:
       - localstack/startup
       - install_java17
       - run_maven:
-          maven_subcommand: test
+          maven_subcommand: verify
       - slack/notify:
           event: fail
 

pom.xml

@@ -44,6 +44,8 @@
       <maven.compiler.target>17</maven.compiler.target>
       <maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
       <maven.compiler.showWarnings>true</maven.compiler.showWarnings>
+      <coverage.haltOnFailure>true</coverage.haltOnFailure>
+      <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
    </properties>
 
    <dependencies>
@@ -115,6 +117,9 @@
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-surefire-plugin</artifactId>
             <version>3.0.0-M5</version>
+            <configuration>
+               <argLine>@{jaCoCoArgLine}</argLine>
+            </configuration>
          </plugin>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
@@ -164,7 +169,84 @@
                <versionDigitToIncrement>1</versionDigitToIncrement> <!-- In general, we update the minor -->
             </configuration>
          </plugin>
-
+         <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>0.8.8</version>
+            <executions>
+               <execution>
+                  <id>pre-unit-test</id>
+                  <goals>
+                     <goal>prepare-agent</goal>
+                  </goals>
+                  <configuration>
+                     <propertyName>jaCoCoArgLine</propertyName>
+                  </configuration>
+               </execution>
+               <execution>
+                  <id>unit-test-check</id>
+                  <goals>
+                     <goal>check</goal>
+                  </goals>
+                  <configuration>
+                     <!-- Gives us the ability to pass a parameter to not fail due to coverage E.g. -Dcoverage.haltOnFailure=false -->
+                     <haltOnFailure>${coverage.haltOnFailure}</haltOnFailure>
+                     <rules>
+                        <rule>
+                           <element>BUNDLE</element>
+                           <limits>
+                              <limit>
+                                 <counter>INSTRUCTION</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>${coverage.instructionCoveredRatioMinimum}</minimum>
+                              </limit>
+                           </limits>
+                        </rule>
+                     </rules>
+                  </configuration>
+               </execution>
+               <execution>
+                  <id>post-unit-test</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>report</goal>
+                  </goals>
+               </execution>
+            </executions>
+         </plugin>
+         <plugin>
+            <artifactId>exec-maven-plugin</artifactId>
+            <groupId>org.codehaus.mojo</groupId>
+            <version>3.0.0</version>
+            <executions>
+               <execution>
+                  <id>test-coverage-summary</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>exec</goal>
+                  </goals>
+                  <configuration>
+                     <executable>sh</executable>
+                     <arguments>
+                        <argument>-c</argument>
+                        <argument>
+                           <![CDATA[
+echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+echo
+echo "Jacoco coverage summary report:"
+echo "   See also target/site/jacoco/index.html"
+echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
+echo "------------------------------------------------------------"
+paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
+rm /tmp/$$.headers /tmp/$$.values
+                           ]]>
+                        </argument>
+                     </arguments>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
       </plugins>
    </build>
 

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java

@@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile
  *******************************************************************************/
 public interface FilesystemBackendModuleInterface<FILE>
 {
+   String CUSTOMIZER_FILE_POST_FILE_READ = "postFileRead";
+
    /*******************************************************************************
     ** For filesystem backends, get the module-specific action base-class, that helps
     ** with functions like listing and deleting files.

src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java

@@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails;
 import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
@@ -269,7 +270,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
     *******************************************************************************/
    private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException
    {
-      Optional<QCodeReference> optionalCustomizer = table.getCustomizer("postFileRead");
+      Optional<QCodeReference> optionalCustomizer = table.getCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ);
       if(optionalCustomizer.isEmpty())
       {
          return (fileContents);

src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java

@@ -27,6 +27,7 @@ import java.io.InputStream;
 import java.util.List;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
@@ -67,6 +68,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
          return;
       }
 
+      s3Utils = new S3Utils();
+      s3Utils.setAmazonS3(buildAmazonS3ClientFromBackendMetaData(backendMetaData));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData backendMetaData)
+   {
       S3BackendMetaData     s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendMetaData);
       AmazonS3ClientBuilder clientBuilder     = AmazonS3ClientBuilder.standard();
 
@@ -81,8 +93,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction<S3ObjectSumma
          clientBuilder.setRegion(s3BackendMetaData.getRegion());
       }
 
-      s3Utils = new S3Utils();
+      return (clientBuilder.build());
-      s3Utils.setAmazonS3(clientBuilder.build());
    }
 
 

src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemQueryActionTest.java

@@ -22,11 +22,17 @@
 package com.kingsrook.qqq.backend.module.filesystem.local.actions;
 
 
+import java.util.function.Function;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryRequest;
 import com.kingsrook.qqq.backend.core.model.actions.query.QueryResult;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
+import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
 import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface;
 import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -44,7 +50,9 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    @Test
    public void testQuery1() throws QException
    {
-      QueryRequest queryRequest = initQueryRequest();
+      QueryRequest queryRequest = new QueryRequest();
+      queryRequest.setInstance(TestUtils.defineInstance());
+      queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
       QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
       Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
       Assertions.assertTrue(queryResult.getRecords().stream()
@@ -57,12 +65,36 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QueryRequest initQueryRequest() throws QInstanceValidationException
+   @Test
+   public void testQueryWithFileCustomizer() throws QException
    {
       QueryRequest queryRequest = new QueryRequest();
-      queryRequest.setInstance(TestUtils.defineInstance());
+      QInstance    instance     = TestUtils.defineInstance();
+
+      QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
+      table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference()
+         .withName(ValueUpshifter.class.getName())
+         .withCodeType(QCodeType.JAVA)
+         .withCodeUsage(QCodeUsage.CUSTOMIZER));
+
+      queryRequest.setInstance(instance);
       queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
-      return queryRequest;
+      QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest);
+      Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows");
+      Assertions.assertTrue(
+         queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")),
+         "All records should have their email addresses up-shifted.");
+   }
+
+
+
+   public static class ValueUpshifter implements Function<String, String>
+   {
+      @Override
+      public String apply(String s)
+      {
+         return (s.replaceAll("kingsrook.com", "KINGSROOK.COM"));
+      }
    }
 
 }

src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java

@@ -0,0 +1,115 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
+
+
+import java.util.Arrays;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
+import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+/*******************************************************************************
+ ** Unit test for BasicETLCollectSourceFileNamesFunction
+ *******************************************************************************/
+class BasicETLCollectSourceFileNamesFunctionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testOneFile() throws Exception
+   {
+      String file   = "/tmp/test1.csv";
+      String result = runTest(file);
+      assertEquals(file, result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testTwoFiles() throws Exception
+   {
+      String file1  = "/tmp/test1.csv";
+      String file2  = "/tmp/test2.csv";
+      String result = runTest(file1, file2);
+
+      //////////////////////////////////////////////////////////////////////
+      // the names go into a set, so they can come out in either order... //
+      //////////////////////////////////////////////////////////////////////
+      assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1)));
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void testDuplicatedFile() throws Exception
+   {
+      String file   = "/tmp/test1.csv";
+      String result = runTest(file, file);
+      assertEquals(file, result);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private String runTest(String... fileNames) throws Exception
+   {
+      QInstance            qInstance           = TestUtils.defineInstance();
+      QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesFunction().defineStepMetaData();
+      QProcessMetaData     qProcessMetaData    = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
+      qInstance.addProcess(qProcessMetaData);
+
+      List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
+         new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
+
+      RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance);
+      runBackendStepRequest.setSession(TestUtils.getMockSession());
+      runBackendStepRequest.setStepName(backendStepMetaData.getName());
+      runBackendStepRequest.setProcessName(qProcessMetaData.getName());
+      runBackendStepRequest.setRecords(records);
+
+      RunBackendStepAction runBackendStepAction = new RunBackendStepAction();
+      RunBackendStepResult result               = runBackendStepAction.execute(runBackendStepRequest);
+
+      return ((String) result.getValues().get(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS));
+   }
+}

src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
+
+
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.*;
+
+
+/*******************************************************************************
+ ** Unit test for AbstractS3Action
+ *******************************************************************************/
+class AbstractS3ActionTest
+{
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBuildAmazonS3ClientFromBackendMetaData()
+   {
+      String regionName = Regions.AP_SOUTHEAST_3.getName();
+      S3BackendMetaData s3BackendMetaData = new S3BackendMetaData()
+         .withAccessKey("Not a real access key")
+         .withSecretKey("Also not a real key")
+         .withRegion(regionName);
+      AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData);
+      assertEquals(regionName, amazonS3.getRegionName());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testBuildAmazonS3ClientFromBackendMetaDataWrongType()
+   {
+      assertThrows(IllegalArgumentException.class, () ->
+      {
+         new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData());
+      });
+   }
+}