Merged dev into feature/workflows-support

.circleci/collect-jacoco-reports.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+############################################################################
+## Script to collect all JaCoCo reports from different modules into a
+## single directory for easier artifact storage in CI.
+############################################################################
+
+mkdir -p /home/circleci/jacoco-reports/
+
+##############################################################
+## Find all module directories that have target/site/jacoco ##
+##############################################################
+for module_dir in */; do
+  if [ -d "${module_dir}target/site/jacoco" ]; then
+    module_name=$(basename "${module_dir%/}")
+    target_dir="/home/circleci/jacoco-reports/${module_name}"
+    
+    echo "Collecting JaCoCo reports for module: ${module_name}"
+    
+    cp -r "${module_dir}target/site/jacoco" "${target_dir}"
+    
+    echo "Copied JaCoCo reports for ${module_name} to ${target_dir}"
+  fi
+done
+
+echo "All JaCoCo reports collected to /home/circleci/jacoco-reports/" 

.circleci/concatenate-test-output.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+############################################################################
+## Script to concatenate all .txt files in the surefire-reports directory
+## into a single artifact that can be stored in CI.
+############################################################################
+
+mkdir -p /home/circleci/test-output-artifacts/
+
+###################################################################
+## Find all module directories that have target/surefire-reports ##
+###################################################################
+for module_dir in */; do
+  if [ -d "${module_dir}target/surefire-reports" ]; then
+    module_name=$(basename "${module_dir%/}")
+    output_file="/home/circleci/test-output-artifacts/${module_name}-test-output.txt"
+    
+    echo "Processing module: ${module_name}"
+    echo "Output file: ${output_file}"
+    
+    ##################################################################
+    ## Concatenate all .txt files in the surefire-reports directory ##
+    ##################################################################
+    if [ -n "$(find "${module_dir}target/surefire-reports" -name "*.txt" -type f)" ]; then
+      echo "=== Test Output for ${module_name} ===" > "${output_file}"
+      echo "Generated at: $(date)" >> "${output_file}"
+      echo "==========================================" >> "${output_file}"
+      echo "" >> "${output_file}"
+      
+      ##############################################
+      ## Sort files to ensure consistent ordering ##
+      ##############################################
+      find "${module_dir}target/surefire-reports" -name "*.txt" -type f | sort | while read -r txt_file; do
+        echo "--- File: $(basename "${txt_file}") ---" >> "${output_file}"
+        cat "${txt_file}" >> "${output_file}"
+        echo "" >> "${output_file}"
+        echo "--- End of $(basename "${txt_file}") ---" >> "${output_file}"
+        echo "" >> "${output_file}"
+        echo "" >> "${output_file}"
+        echo "" >> "${output_file}"
+      done
+      
+      echo "Concatenated test output for ${module_name} to ${output_file}"
+    else
+      echo "No .txt files found in ${module_dir}target/surefire-reports"
+    fi
+  fi
+done 

.circleci/config.yml

@@ -5,35 +5,7 @@ orbs:
   browser-tools: circleci/browser-tools@1.4.7
 
 commands:
-  store_jacoco_site:
+  mvn_build:
-    parameters:
-      module:
-        type: string
-    steps:
-      - store_artifacts:
-          path: << parameters.module >>/target/site/jacoco/index.html
-          when: always
-      - store_artifacts:
-          path: << parameters.module >>/target/site/jacoco/jacoco-resources
-          when: always
-
-  install_java17:
-    steps:
-      - run:
-          name: Install Java 17
-          command: |
-            sudo apt-get update
-            sudo apt install -y openjdk-17-jdk
-            sudo rm /etc/alternatives/java
-            sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
-      - run:
-          ## used by jacoco uncovered class reporting in pom.xml
-          name: Install html2text
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y html2text
-
-  mvn_verify:
     steps:
       - checkout
       - restore_cache:
@@ -45,30 +17,41 @@ commands:
           name: Write .env
           command: |
             echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env
+      - run:
+          name: Run Maven Compile
+          command: |
+            mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress compile
+      - save_cache:
+          paths:
+            - ~/.m2
+          key: v1-dependencies-{{ checksum "pom.xml" }}
+
+  mvn_verify:
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "pom.xml" }}
       - run:
           name: Run Maven Verify
           command: |
-            mvn -s .circleci/mvn-settings.xml -T4 verify
+            mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress verify
-      - store_jacoco_site:
+      - run:
-          module: qqq-backend-core
+          name: Collect JaCoCo reports
-      - store_jacoco_site:
+          command: .circleci/collect-jacoco-reports.sh
-          module: qqq-backend-module-filesystem
+          when: always
-      - store_jacoco_site:
+      - store_artifacts:
-          module: qqq-backend-module-rdbms
+          path: /home/circleci/jacoco-reports
-      - store_jacoco_site:
+          destination: jacoco-reports
-          module: qqq-backend-module-api
+          when: always
-      - store_jacoco_site:
+      - run:
-          module: qqq-middleware-api
+          name: Concatenate test output files
-      - store_jacoco_site:
+          command: .circleci/concatenate-test-output.sh
-          module: qqq-middleware-javalin
+          when: always
-      - store_jacoco_site:
+      - store_artifacts:
-          module: qqq-middleware-picocli
+          path: /home/circleci/test-output-artifacts
-      - store_jacoco_site:
+          destination: test-output
-          module: qqq-middleware-slack
+          when: always
-      - store_jacoco_site:
-          module: qqq-language-support-javascript
-      - store_jacoco_site:
-          module: qqq-sample-project
       - run:
           name: Save test results
           command: |
@@ -77,10 +60,6 @@ commands:
           when: always
       - store_test_results:
           path: ~/test-results
-      - save_cache:
-          paths:
-            - ~/.m2
-          key: v1-dependencies-{{ checksum "pom.xml" }}
 
   check_middleware_api_versions:
       steps:
@@ -91,8 +70,8 @@ commands:
         - run:
             name: Build and Run ValidateApiVersions
             command: |
-              mvn -s .circleci/mvn-settings.xml -T4 install -DskipTests
+              mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress install -DskipTests
-              mvn -s .circleci/mvn-settings.xml -pl qqq-middleware-javalin package appassembler:assemble -DskipTests
+              mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress -pl qqq-middleware-javalin package appassembler:assemble -DskipTests
               qqq-middleware-javalin/target/appassembler/bin/ValidateApiVersions -r $(pwd)
 
   mvn_jar_deploy:
@@ -108,7 +87,7 @@ commands:
       - run:
           name: Run Maven Jar Deploy
           command: |
-            mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy
+            mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress flatten:flatten jar:jar deploy:deploy
       - save_cache:
           paths:
             - ~/.m2
@@ -135,19 +114,25 @@ commands:
           when: always
 
 jobs:
-  mvn_test:
+  build:
+    executor: localstack/default
+    steps:
+      - mvn_build
+
+  test:
     executor: localstack/default
     steps:
-      ## - localstack/startup
-      - install_java17
       - mvn_verify
+
+  api_version_check:
+    executor: localstack/default
+    steps:
       - check_middleware_api_versions
 
   mvn_deploy:
     executor: localstack/default
     steps:
-      ## - localstack/startup
+      - mvn_build
-      - install_java17
       - mvn_verify
       - check_middleware_api_versions
       - mvn_jar_deploy
@@ -161,13 +146,31 @@ jobs:
 workflows:
   test_only:
     jobs:
-      - mvn_test:
+      - build:
           context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
           filters:
             branches:
               ignore: /(dev|integration.*)/
             tags:
               ignore: /(version|snapshot)-.*/
+      - test:
+          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /(dev|integration.*)/
+            tags:
+              ignore: /(version|snapshot)-.*/
+      - api_version_check:
+          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /(dev|integration.*)/
+            tags:
+              ignore: /(version|snapshot)-.*/
 
   deploy:
     jobs:

README.md

@@ -30,6 +30,20 @@ There are a few useful IntelliJ settings files, under `qqq-dev-tools/intellij`:
 One will likely also want the [Kingsrook Commentator
 Plugin](https://plugins.jetbrains.com/plugin/19325-kingsrook-commentator).
 
+## Test Logging
+By default, when ran from the command line, mvn surefire will make each test's
+output (e.g., System.out, err, printStackTrace, and all logger calls) go into a 
+file under target/surefire-reports/${className}.txt. 
+
+The system property `-DtestOutputToFile=false` can be given on the command line
+to get all of this output on the console.
+
+In the IDE (e.g,. IntelliJ), output goes to the Console.
+
+In CircleCI, output goes to files, and those files are concatenated together and 
+stored as artifacts.
+
+
 ## License
 QQQ - Low-code Application Framework for Engineers. \
 Copyright (C) 2020-2024.  Kingsrook, LLC \

pom.xml

@@ -59,6 +59,7 @@
       <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
       <coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
       <plugin.shade.phase>none</plugin.shade.phase>
+      <testOutputToFile>true</testOutputToFile>
    </properties>
 
    <profiles>
@@ -141,6 +142,8 @@
             <configuration>
                <!-- Sets the VM argument line used when integration tests are run. -->
                <argLine>@{jaCoCoArgLine}</argLine>
+               <!-- Reduce console output for cleaner JUnit output -->
+               <redirectTestOutputToFile>${testOutputToFile}</redirectTestOutputToFile>
             </configuration>
          </plugin>
          <plugin>
@@ -244,30 +247,29 @@ if [ ! -e target/site/jacoco/index.html ]; then
 fi
 
 echo
-echo "Jacoco coverage summary report:"
+echo "Jacoco coverage summary report for module: ${project.artifactId}"
 echo "   See also target/site/jacoco/index.html"
 echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
 echo "------------------------------------------------------------"
 
-if which xpath > /dev/null 2>&1; then
+# Parse Jacoco HTML coverage summary 
-   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+if [ -f target/site/jacoco/index.html ]; then
-   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+   echo -e "Instructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+   sed 's/<\/\w\+>/&\n/g' target/site/jacoco/index.html | grep -A 12 '<tfoot>' | grep '<td' | sed 's/<td class="\w\+\d*">\([^<]*\)<\/td>/\1/' | grep -v Total > /tmp/$$.values
    paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
    rm /tmp/$$.headers /tmp/$$.values
 else
-   echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
+   echo "Jacoco coverage summary was not found.";
 fi
+echo "-----------------------------"
+echo
 
-if which html2text > /dev/null 2>&1; then
+echo "Untested classes, per Jacoco for module: ${project.artifactId}"
-   echo "Untested classes, per Jacoco:"
+echo "-----------------------------"
-   echo "-----------------------------"
+# Parse Jacoco XML reports directly to find classes with 0% coverage
-   for i in target/site/jacoco/*/index.html; do
+sed 's/<classs .*\?>/&\n/g;s/<\/class>/&\n/g' target/site/jacoco/jacoco.xml | grep -v 'counter type="CLASS" missed="0"' | sed 's/>.*//;s/.*\///;s/".*//'
-      html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1;
+echo "-----------------------------"
-   done;
+echo
-   echo
-else
-   echo "html2text is not installed.  Untested classes from Jacoco will not be printed here...";
-fi
 
                            ]]>
                         </argument>

qqq-backend-module-filesystem/pom.xml

@@ -55,11 +55,6 @@
          <artifactId>sshd-sftp</artifactId>
          <version>2.14.0</version>
       </dependency>
-      <dependency>
-         <groupId>org.apache.sshd</groupId>
-         <artifactId>sshd-sftp</artifactId>
-         <version>2.14.0</version>
-      </dependency>
 
       <dependency>
          <groupId>cloud.localstack</groupId>

qqq-middleware-javalin/pom.xml

@@ -122,6 +122,18 @@
    <build>
       <sourceDirectory>src/main/java</sourceDirectory>
       <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>3.5.3</version>
+            <configuration>
+               <additionalClasspathElements>
+                  <additionalClasspathElement>
+                     ${project.basedir}/src/test/resources/static-site.jar
+                  </additionalClasspathElement>
+               </additionalClasspathElements>
+            </configuration>
+         </plugin>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-compiler-plugin</artifactId>

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java

@@ -1932,6 +1932,7 @@ public class QJavalinImplementation
    {
       String searchTerm = context.queryParam("searchTerm");
       String ids        = context.queryParam("ids");
+      String labels     = context.queryParam("labels");
 
       SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput();
       setupSession(context, input);
@@ -1945,6 +1946,11 @@ public class QJavalinImplementation
          List<Serializable> idList = new ArrayList<>(Arrays.asList(ids.split(",")));
          input.setIdList(idList);
       }
+      else if(StringUtils.hasContent(labels))
+      {
+         List<String> labelList = new ArrayList<>(Arrays.asList(labels.split(",")));
+         input.setLabelList(labelList);
+      }
 
       SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input);
 

qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java

@@ -49,16 +49,14 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInterface
 {
    private static final QLogger LOG = QLogger.getLogger(SimpleFileSystemDirectoryRouter.class);
+   public static boolean loadStaticFilesFromJar = false;
+
 
    private final String hostedPath;
    private final String fileSystemPath;
-
    private QCodeReference routeAuthenticator;
-
    private QInstance qInstance;
 
-
-
    /*******************************************************************************
     ** Constructor
     **
@@ -67,6 +65,24 @@ public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInt
    {
       this.hostedPath = hostedPath;
       this.fileSystemPath = fileSystemPath;
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      // read the property to see if we should load static files from the jar file or from the file system //
+      // Javan only supports loading via one method per path, so its a choice of one or the other...       //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         String propertyName  = "qqq.javalin.enableStaticFilesFromJar";  // TODO: make a more general way to handle properties like this system-wide via a central config class
+         String propertyValue = System.getProperty(propertyName, "");
+         if(propertyValue.equals("true"))
+         {
+            loadStaticFilesFromJar = true;
+         }
+      }
+      catch(Exception e)
+      {
+         e.printStackTrace();
+      }
    }
 
 
@@ -98,25 +114,41 @@ public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInt
     ***************************************************************************/
    private void handleJavalinStaticFileConfig(StaticFileConfig staticFileConfig)
    {
-      URL resource = getClass().getClassLoader().getResource(fileSystemPath);
-      if(resource == null)
-      {
-         String message = "Could not find file system path: " + fileSystemPath;
-         if(fileSystemPath.startsWith("/") && getClass().getClassLoader().getResource(fileSystemPath.replaceFirst("^/+", "")) != null)
-         {
-            message += ".  For non-absolute paths, do not prefix with a leading slash.";
-         }
-         throw new RuntimeException(message);
-      }
 
       if(!hostedPath.startsWith("/"))
       {
          LOG.warn("hostedPath [" + hostedPath + "] should probably start with a leading slash...");
       }
 
-      staticFileConfig.directory = resource.getFile();
+      /// /////////////////////////////////////////////////////////////////////////////////////
-      staticFileConfig.hostedPath = hostedPath;
+      // Handle loading static files from the jar OR the filesystem based on system property //
-      staticFileConfig.location = Location.EXTERNAL;
+      /// /////////////////////////////////////////////////////////////////////////////////////
+      if(SimpleFileSystemDirectoryRouter.loadStaticFilesFromJar)
+      {
+         staticFileConfig.directory = fileSystemPath;
+         staticFileConfig.hostedPath = hostedPath;
+         staticFileConfig.location = Location.CLASSPATH;
+         LOG.info("Static File Config : hostedPath [" + hostedPath + "] : directory [" + staticFileConfig.directory + "] : location [CLASSPATH]");
+      }
+      else
+      {
+         URL resource = getClass().getClassLoader().getResource(fileSystemPath);
+         if(resource == null)
+         {
+            String message = "Could not find file system path: " + fileSystemPath;
+            if(fileSystemPath.startsWith("/") && getClass().getClassLoader().getResource(fileSystemPath.replaceFirst("^/+", "")) != null)
+            {
+               message += ".  For non-absolute paths, do not prefix with a leading slash.";
+            }
+            throw new RuntimeException(message);
+         }
+
+         staticFileConfig.directory = resource.getFile();
+         staticFileConfig.hostedPath = hostedPath;
+         staticFileConfig.location = Location.EXTERNAL;
+         LOG.info("Static File Config : hostedPath [" + hostedPath + "] : directory [" + staticFileConfig.directory + "] : location [EXTERNAL]");
+      }
+
    }
 
 

qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.javalin.TestUtils;
+import com.kingsrook.qqq.middleware.javalin.routeproviders.SimpleFileSystemDirectoryRouter;
 import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1;
 import io.javalin.http.HttpStatus;
 import kong.unirest.HttpResponse;
@@ -52,6 +53,16 @@ class QApplicationJavalinServerTest
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static AbstractQQQApplication getQqqApplication()
+   {
+      return new TestApplication();
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -60,6 +71,7 @@ class QApplicationJavalinServerTest
    {
       javalinServer.stop();
       TestApplication.callCount = 0;
+      System.clearProperty("qqq.javalin.enableStaticFilesFromJar");
    }
 
 
@@ -196,6 +208,48 @@ class QApplicationJavalinServerTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testStaticRouterFilesFromExternal() throws Exception
+   {
+      System.setProperty("qqq.javalin.enableStaticFilesFromJar", "false");
+
+      javalinServer = new QApplicationJavalinServer(getQqqApplication())
+         .withServeFrontendMaterialDashboard(false)
+         .withPort(PORT);
+      javalinServer.start();
+
+      Unirest.config().setDefaultResponseEncoding("UTF-8");
+      HttpResponse<String> response = Unirest.get("http://localhost:" + PORT + "/statically-served/foo.html").asString();
+      assertEquals("Foo? Bar!", response.getBody());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testStaticRouterFilesFromClasspath() throws Exception
+   {
+      System.setProperty("qqq.javalin.enableStaticFilesFromJar", "true");
+
+      javalinServer = new QApplicationJavalinServer(new QApplicationJavalinServerTest.TestApplication())
+         .withServeFrontendMaterialDashboard(false)
+         .withPort(PORT)
+         .withAdditionalRouteProvider(new SimpleFileSystemDirectoryRouter("/statically-served-from-jar", "static-site-from-jar/"));
+
+      javalinServer.start();
+
+      Unirest.config().setDefaultResponseEncoding("UTF-8");
+      HttpResponse<String> response = Unirest.get("http://localhost:" + PORT + "/statically-served-from-jar/foo-in-jar.html").asString();
+      assertEquals("Foo in a Jar!\n", response.getBody());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -296,16 +350,6 @@ class QApplicationJavalinServerTest
 
 
 
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private static AbstractQQQApplication getQqqApplication()
-   {
-      return new TestApplication();
-   }
-
-
-
    /***************************************************************************
     **
     ***************************************************************************/