Merged dev into feature/workflows-support

2025-07-17 20:50:44 +00:00 · 2025-07-14 19:44:38 -05:00
parent 104dd2fff8 d311e28e60
commit 78b893616b
11 changed files with 292 additions and 110 deletions
--- a/.circleci/collect-jacoco-reports.sh
+++ b/.circleci/collect-jacoco-reports.sh
@ -0,0 +1,26 @@
+#!/bin/bash
+
+############################################################################
+## Script to collect all JaCoCo reports from different modules into a
+## single directory for easier artifact storage in CI.
+############################################################################
+
+mkdir -p /home/circleci/jacoco-reports/
+
+##############################################################
+## Find all module directories that have target/site/jacoco ##
+##############################################################
+for module_dir in */; do
+  if [ -d "${module_dir}target/site/jacoco" ]; then
+    module_name=$(basename "${module_dir%/}")
+    target_dir="/home/circleci/jacoco-reports/${module_name}"
+    
+    echo "Collecting JaCoCo reports for module: ${module_name}"
+    
+    cp -r "${module_dir}target/site/jacoco" "${target_dir}"
+    
+    echo "Copied JaCoCo reports for ${module_name} to ${target_dir}"
+  fi
+done
+
+echo "All JaCoCo reports collected to /home/circleci/jacoco-reports/" 
--- a/.circleci/concatenate-test-output.sh
+++ b/.circleci/concatenate-test-output.sh
@ -0,0 +1,48 @@
+#!/bin/bash
+
+############################################################################
+## Script to concatenate all .txt files in the surefire-reports directory
+## into a single artifact that can be stored in CI.
+############################################################################
+
+mkdir -p /home/circleci/test-output-artifacts/
+
+###################################################################
+## Find all module directories that have target/surefire-reports ##
+###################################################################
+for module_dir in */; do
+  if [ -d "${module_dir}target/surefire-reports" ]; then
+    module_name=$(basename "${module_dir%/}")
+    output_file="/home/circleci/test-output-artifacts/${module_name}-test-output.txt"
+    
+    echo "Processing module: ${module_name}"
+    echo "Output file: ${output_file}"
+    
+    ##################################################################
+    ## Concatenate all .txt files in the surefire-reports directory ##
+    ##################################################################
+    if [ -n "$(find "${module_dir}target/surefire-reports" -name "*.txt" -type f)" ]; then
+      echo "=== Test Output for ${module_name} ===" > "${output_file}"
+      echo "Generated at: $(date)" >> "${output_file}"
+      echo "==========================================" >> "${output_file}"
+      echo "" >> "${output_file}"
+      
+      ##############################################
+      ## Sort files to ensure consistent ordering ##
+      ##############################################
+      find "${module_dir}target/surefire-reports" -name "*.txt" -type f | sort | while read -r txt_file; do
+        echo "--- File: $(basename "${txt_file}") ---" >> "${output_file}"
+        cat "${txt_file}" >> "${output_file}"
+        echo "" >> "${output_file}"
+        echo "--- End of $(basename "${txt_file}") ---" >> "${output_file}"
+        echo "" >> "${output_file}"
+        echo "" >> "${output_file}"
+        echo "" >> "${output_file}"
+      done
+      
+      echo "Concatenated test output for ${module_name} to ${output_file}"
+    else
+      echo "No .txt files found in ${module_dir}target/surefire-reports"
+    fi
+  fi
+done 
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -5,35 +5,7 @@ orbs:
  browser-tools: circleci/browser-tools@1.4.7

 commands:
-  store_jacoco_site:
-    parameters:
-      module:
-        type: string
-    steps:
-      - store_artifacts:
-          path: << parameters.module >>/target/site/jacoco/index.html
-          when: always
-      - store_artifacts:
-          path: << parameters.module >>/target/site/jacoco/jacoco-resources
-          when: always
-
-  install_java17:
-    steps:
-      - run:
-          name: Install Java 17
-          command: |
-            sudo apt-get update
-            sudo apt install -y openjdk-17-jdk
-            sudo rm /etc/alternatives/java
-            sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
-      - run:
-          ## used by jacoco uncovered class reporting in pom.xml
-          name: Install html2text
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y html2text
-
-  mvn_verify:
+  mvn_build:
    steps:
      - checkout
      - restore_cache:
@ -45,30 +17,41 @@ commands:
          name: Write .env
          command: |
            echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env
+      - run:
+          name: Run Maven Compile
+          command: |
+            mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress compile
+      - save_cache:
+          paths:
+            - ~/.m2
+          key: v1-dependencies-{{ checksum "pom.xml" }}
+
+  mvn_verify:
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "pom.xml" }}
      - run:
          name: Run Maven Verify
          command: |
-            mvn -s .circleci/mvn-settings.xml -T4 verify
-      - store_jacoco_site:
-          module: qqq-backend-core
-      - store_jacoco_site:
-          module: qqq-backend-module-filesystem
-      - store_jacoco_site:
-          module: qqq-backend-module-rdbms
-      - store_jacoco_site:
-          module: qqq-backend-module-api
-      - store_jacoco_site:
-          module: qqq-middleware-api
-      - store_jacoco_site:
-          module: qqq-middleware-javalin
-      - store_jacoco_site:
-          module: qqq-middleware-picocli
-      - store_jacoco_site:
-          module: qqq-middleware-slack
-      - store_jacoco_site:
-          module: qqq-language-support-javascript
-      - store_jacoco_site:
-          module: qqq-sample-project
+            mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress verify
+      - run:
+          name: Collect JaCoCo reports
+          command: .circleci/collect-jacoco-reports.sh
+          when: always
+      - store_artifacts:
+          path: /home/circleci/jacoco-reports
+          destination: jacoco-reports
+          when: always
+      - run:
+          name: Concatenate test output files
+          command: .circleci/concatenate-test-output.sh
+          when: always
+      - store_artifacts:
+          path: /home/circleci/test-output-artifacts
+          destination: test-output
+          when: always
      - run:
          name: Save test results
          command: |
@ -77,10 +60,6 @@ commands:
          when: always
      - store_test_results:
          path: ~/test-results
-      - save_cache:
-          paths:
-            - ~/.m2
-          key: v1-dependencies-{{ checksum "pom.xml" }}

  check_middleware_api_versions:
      steps:
@ -91,8 +70,8 @@ commands:
        - run:
            name: Build and Run ValidateApiVersions
            command: |
-              mvn -s .circleci/mvn-settings.xml -T4 install -DskipTests
-              mvn -s .circleci/mvn-settings.xml -pl qqq-middleware-javalin package appassembler:assemble -DskipTests
+              mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress install -DskipTests
+              mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress -pl qqq-middleware-javalin package appassembler:assemble -DskipTests
              qqq-middleware-javalin/target/appassembler/bin/ValidateApiVersions -r $(pwd)

  mvn_jar_deploy:
@ -108,7 +87,7 @@ commands:
      - run:
          name: Run Maven Jar Deploy
          command: |
-            mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy
+            mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress flatten:flatten jar:jar deploy:deploy
      - save_cache:
          paths:
            - ~/.m2
@ -135,19 +114,25 @@ commands:
          when: always

 jobs:
-  mvn_test:
+  build:
+    executor: localstack/default
+    steps:
+      - mvn_build
+
+  test:
    executor: localstack/default
    steps:
-      ## - localstack/startup
-      - install_java17
      - mvn_verify
+
+  api_version_check:
+    executor: localstack/default
+    steps:
      - check_middleware_api_versions

  mvn_deploy:
    executor: localstack/default
    steps:
-      ## - localstack/startup
-      - install_java17
+      - mvn_build
      - mvn_verify
      - check_middleware_api_versions
      - mvn_jar_deploy
@ -161,13 +146,31 @@ jobs:
 workflows:
  test_only:
    jobs:
-      - mvn_test:
+      - build:
          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
          filters:
            branches:
              ignore: /(dev|integration.*)/
            tags:
              ignore: /(version|snapshot)-.*/
+      - test:
+          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /(dev|integration.*)/
+            tags:
+              ignore: /(version|snapshot)-.*/
+      - api_version_check:
+          context: [ qqq-maven-registry-credentials, build-qqq-sample-app ]
+          requires:
+            - build
+          filters:
+            branches:
+              ignore: /(dev|integration.*)/
+            tags:
+              ignore: /(version|snapshot)-.*/

  deploy:
    jobs:
--- a/README.md
+++ b/README.md
@ -30,6 +30,20 @@ There are a few useful IntelliJ settings files, under `qqq-dev-tools/intellij`:
 One will likely also want the [Kingsrook Commentator
 Plugin](https://plugins.jetbrains.com/plugin/19325-kingsrook-commentator).

+## Test Logging
+By default, when ran from the command line, mvn surefire will make each test's
+output (e.g., System.out, err, printStackTrace, and all logger calls) go into a 
+file under target/surefire-reports/${className}.txt. 
+
+The system property `-DtestOutputToFile=false` can be given on the command line
+to get all of this output on the console.
+
+In the IDE (e.g,. IntelliJ), output goes to the Console.
+
+In CircleCI, output goes to files, and those files are concatenated together and 
+stored as artifacts.
+
+
 ## License
 QQQ - Low-code Application Framework for Engineers. \
 Copyright (C) 2020-2024.  Kingsrook, LLC \
--- a/pom.xml
+++ b/pom.xml
@ -59,6 +59,7 @@
      <coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
      <coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
      <plugin.shade.phase>none</plugin.shade.phase>
+      <testOutputToFile>true</testOutputToFile>
   </properties>

   <profiles>
@ -141,6 +142,8 @@
            <configuration>
               <!-- Sets the VM argument line used when integration tests are run. -->
               <argLine>@{jaCoCoArgLine}</argLine>
+               <!-- Reduce console output for cleaner JUnit output -->
+               <redirectTestOutputToFile>${testOutputToFile}</redirectTestOutputToFile>
            </configuration>
         </plugin>
         <plugin>
@ -244,30 +247,29 @@ if [ ! -e target/site/jacoco/index.html ]; then
 fi

 echo
-echo "Jacoco coverage summary report:"
+echo "Jacoco coverage summary report for module: ${project.artifactId}"
 echo "   See also target/site/jacoco/index.html"
 echo "   and https://www.jacoco.org/jacoco/trunk/doc/counters.html"
 echo "------------------------------------------------------------"

-if which xpath > /dev/null 2>&1; then
-   echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
-   xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values
+# Parse Jacoco HTML coverage summary 
+if [ -f target/site/jacoco/index.html ]; then
+   echo -e "Instructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers
+   sed 's/<\/\w\+>/&\n/g' target/site/jacoco/index.html | grep -A 12 '<tfoot>' | grep '<td' | sed 's/<td class="\w\+\d*">\([^<]*\)<\/td>/\1/' | grep -v Total > /tmp/$$.values
   paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}'
   rm /tmp/$$.headers /tmp/$$.values
 else
-   echo "xpath is not installed.  Jacoco coverage summary will not be produced here...";
+   echo "Jacoco coverage summary was not found.";
 fi
+echo "-----------------------------"
+echo

-if which html2text > /dev/null 2>&1; then
-   echo "Untested classes, per Jacoco:"
-   echo "-----------------------------"
-   for i in target/site/jacoco/*/index.html; do
-      html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1;
-   done;
-   echo
-else
-   echo "html2text is not installed.  Untested classes from Jacoco will not be printed here...";
-fi
+echo "Untested classes, per Jacoco for module: ${project.artifactId}"
+echo "-----------------------------"
+# Parse Jacoco XML reports directly to find classes with 0% coverage
+sed 's/<classs .*\?>/&\n/g;s/<\/class>/&\n/g' target/site/jacoco/jacoco.xml | grep -v 'counter type="CLASS" missed="0"' | sed 's/>.*//;s/.*\///;s/".*//'
+echo "-----------------------------"
+echo

                           ]]>
                        </argument>
--- a/qqq-backend-module-filesystem/pom.xml
+++ b/qqq-backend-module-filesystem/pom.xml
@ -55,11 +55,6 @@
         <artifactId>sshd-sftp</artifactId>
         <version>2.14.0</version>
      </dependency>
-      <dependency>
-         <groupId>org.apache.sshd</groupId>
-         <artifactId>sshd-sftp</artifactId>
-         <version>2.14.0</version>
-      </dependency>

      <dependency>
         <groupId>cloud.localstack</groupId>
--- a/qqq-middleware-javalin/pom.xml
+++ b/qqq-middleware-javalin/pom.xml
@ -122,6 +122,18 @@
   <build>
      <sourceDirectory>src/main/java</sourceDirectory>
      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>3.5.3</version>
+            <configuration>
+               <additionalClasspathElements>
+                  <additionalClasspathElement>
+                     ${project.basedir}/src/test/resources/static-site.jar
+                  </additionalClasspathElement>
+               </additionalClasspathElements>
+            </configuration>
+         </plugin>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
--- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
+++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java
@ -1932,6 +1932,7 @@ public class QJavalinImplementation
   {
      String searchTerm = context.queryParam("searchTerm");
      String ids        = context.queryParam("ids");
+      String labels     = context.queryParam("labels");

      SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput();
      setupSession(context, input);
@ -1945,6 +1946,11 @@ public class QJavalinImplementation
         List<Serializable> idList = new ArrayList<>(Arrays.asList(ids.split(",")));
         input.setIdList(idList);
      }
+      else if(StringUtils.hasContent(labels))
+      {
+         List<String> labelList = new ArrayList<>(Arrays.asList(labels.split(",")));
+         input.setLabelList(labelList);
+      }

      SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input);

--- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java
+++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java
@ -49,16 +49,14 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInterface
 {
   private static final QLogger LOG = QLogger.getLogger(SimpleFileSystemDirectoryRouter.class);
+   public static boolean loadStaticFilesFromJar = false;
+

   private final String hostedPath;
   private final String fileSystemPath;
-
   private QCodeReference routeAuthenticator;
-
   private QInstance qInstance;

-
-
   /*******************************************************************************
    ** Constructor
    **
@ -67,6 +65,24 @@ public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInt
   {
      this.hostedPath = hostedPath;
      this.fileSystemPath = fileSystemPath;
+
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      // read the property to see if we should load static files from the jar file or from the file system //
+      // Javan only supports loading via one method per path, so its a choice of one or the other...       //
+      ///////////////////////////////////////////////////////////////////////////////////////////////////////
+      try
+      {
+         String propertyName  = "qqq.javalin.enableStaticFilesFromJar";  // TODO: make a more general way to handle properties like this system-wide via a central config class
+         String propertyValue = System.getProperty(propertyName, "");
+         if(propertyValue.equals("true"))
+         {
+            loadStaticFilesFromJar = true;
+         }
+      }
+      catch(Exception e)
+      {
+         e.printStackTrace();
+      }
   }


@ -98,25 +114,41 @@ public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInt
    ***************************************************************************/
   private void handleJavalinStaticFileConfig(StaticFileConfig staticFileConfig)
   {
-      URL resource = getClass().getClassLoader().getResource(fileSystemPath);
-      if(resource == null)
-      {
-         String message = "Could not find file system path: " + fileSystemPath;
-         if(fileSystemPath.startsWith("/") && getClass().getClassLoader().getResource(fileSystemPath.replaceFirst("^/+", "")) != null)
-         {
-            message += ".  For non-absolute paths, do not prefix with a leading slash.";
-         }
-         throw new RuntimeException(message);
-      }

      if(!hostedPath.startsWith("/"))
      {
         LOG.warn("hostedPath [" + hostedPath + "] should probably start with a leading slash...");
      }

-      staticFileConfig.directory = resource.getFile();
-      staticFileConfig.hostedPath = hostedPath;
-      staticFileConfig.location = Location.EXTERNAL;
+      /// /////////////////////////////////////////////////////////////////////////////////////
+      // Handle loading static files from the jar OR the filesystem based on system property //
+      /// /////////////////////////////////////////////////////////////////////////////////////
+      if(SimpleFileSystemDirectoryRouter.loadStaticFilesFromJar)
+      {
+         staticFileConfig.directory = fileSystemPath;
+         staticFileConfig.hostedPath = hostedPath;
+         staticFileConfig.location = Location.CLASSPATH;
+         LOG.info("Static File Config : hostedPath [" + hostedPath + "] : directory [" + staticFileConfig.directory + "] : location [CLASSPATH]");
+      }
+      else
+      {
+         URL resource = getClass().getClassLoader().getResource(fileSystemPath);
+         if(resource == null)
+         {
+            String message = "Could not find file system path: " + fileSystemPath;
+            if(fileSystemPath.startsWith("/") && getClass().getClassLoader().getResource(fileSystemPath.replaceFirst("^/+", "")) != null)
+            {
+               message += ".  For non-absolute paths, do not prefix with a leading slash.";
+            }
+            throw new RuntimeException(message);
+         }
+
+         staticFileConfig.directory = resource.getFile();
+         staticFileConfig.hostedPath = hostedPath;
+         staticFileConfig.location = Location.EXTERNAL;
+         LOG.info("Static File Config : hostedPath [" + hostedPath + "] : directory [" + staticFileConfig.directory + "] : location [EXTERNAL]");
+      }
+
   }


--- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java
+++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java
@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.javalin.TestUtils;
+import com.kingsrook.qqq.middleware.javalin.routeproviders.SimpleFileSystemDirectoryRouter;
 import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1;
 import io.javalin.http.HttpStatus;
 import kong.unirest.HttpResponse;
@ -52,6 +53,16 @@ class QApplicationJavalinServerTest



+   /***************************************************************************
+    **
+    ***************************************************************************/
+   private static AbstractQQQApplication getQqqApplication()
+   {
+      return new TestApplication();
+   }
+
+
+
   /*******************************************************************************
    **
    *******************************************************************************/
@ -60,6 +71,7 @@ class QApplicationJavalinServerTest
   {
      javalinServer.stop();
      TestApplication.callCount = 0;
+      System.clearProperty("qqq.javalin.enableStaticFilesFromJar");
   }


@ -196,6 +208,48 @@ class QApplicationJavalinServerTest



+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testStaticRouterFilesFromExternal() throws Exception
+   {
+      System.setProperty("qqq.javalin.enableStaticFilesFromJar", "false");
+
+      javalinServer = new QApplicationJavalinServer(getQqqApplication())
+         .withServeFrontendMaterialDashboard(false)
+         .withPort(PORT);
+      javalinServer.start();
+
+      Unirest.config().setDefaultResponseEncoding("UTF-8");
+      HttpResponse<String> response = Unirest.get("http://localhost:" + PORT + "/statically-served/foo.html").asString();
+      assertEquals("Foo? Bar!", response.getBody());
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testStaticRouterFilesFromClasspath() throws Exception
+   {
+      System.setProperty("qqq.javalin.enableStaticFilesFromJar", "true");
+
+      javalinServer = new QApplicationJavalinServer(new QApplicationJavalinServerTest.TestApplication())
+         .withServeFrontendMaterialDashboard(false)
+         .withPort(PORT)
+         .withAdditionalRouteProvider(new SimpleFileSystemDirectoryRouter("/statically-served-from-jar", "static-site-from-jar/"));
+
+      javalinServer.start();
+
+      Unirest.config().setDefaultResponseEncoding("UTF-8");
+      HttpResponse<String> response = Unirest.get("http://localhost:" + PORT + "/statically-served-from-jar/foo-in-jar.html").asString();
+      assertEquals("Foo in a Jar!\n", response.getBody());
+   }
+
+
+
   /*******************************************************************************
    **
    *******************************************************************************/
@ -296,16 +350,6 @@ class QApplicationJavalinServerTest



-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private static AbstractQQQApplication getQqqApplication()
-   {
-      return new TestApplication();
-   }
-
-
-
   /***************************************************************************
    **
    ***************************************************************************/
--- a/qqq-middleware-javalin/src/test/resources/static-site.jar
+++ b/qqq-middleware-javalin/src/test/resources/static-site.jar