diff --git a/.circleci/collect-jacoco-reports.sh b/.circleci/collect-jacoco-reports.sh new file mode 100755 index 00000000..dca6bbad --- /dev/null +++ b/.circleci/collect-jacoco-reports.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +############################################################################ +## Script to collect all JaCoCo reports from different modules into a +## single directory for easier artifact storage in CI. +############################################################################ + +mkdir -p /home/circleci/jacoco-reports/ + +############################################################## +## Find all module directories that have target/site/jacoco ## +############################################################## +for module_dir in */; do + if [ -d "${module_dir}target/site/jacoco" ]; then + module_name=$(basename "${module_dir%/}") + target_dir="/home/circleci/jacoco-reports/${module_name}" + + echo "Collecting JaCoCo reports for module: ${module_name}" + + cp -r "${module_dir}target/site/jacoco" "${target_dir}" + + echo "Copied JaCoCo reports for ${module_name} to ${target_dir}" + fi +done + +echo "All JaCoCo reports collected to /home/circleci/jacoco-reports/" \ No newline at end of file diff --git a/.circleci/concatenate-test-output.sh b/.circleci/concatenate-test-output.sh new file mode 100755 index 00000000..e86efd8e --- /dev/null +++ b/.circleci/concatenate-test-output.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +############################################################################ +## Script to concatenate all .txt files in the surefire-reports directory +## into a single artifact that can be stored in CI. +############################################################################ + +mkdir -p /home/circleci/test-output-artifacts/ + +################################################################### +## Find all module directories that have target/surefire-reports ## +################################################################### +for module_dir in */; do + if [ -d "${module_dir}target/surefire-reports" ]; then + module_name=$(basename "${module_dir%/}") + output_file="/home/circleci/test-output-artifacts/${module_name}-test-output.txt" + + echo "Processing module: ${module_name}" + echo "Output file: ${output_file}" + + ################################################################## + ## Concatenate all .txt files in the surefire-reports directory ## + ################################################################## + if [ -n "$(find "${module_dir}target/surefire-reports" -name "*.txt" -type f)" ]; then + echo "=== Test Output for ${module_name} ===" > "${output_file}" + echo "Generated at: $(date)" >> "${output_file}" + echo "==========================================" >> "${output_file}" + echo "" >> "${output_file}" + + ############################################## + ## Sort files to ensure consistent ordering ## + ############################################## + find "${module_dir}target/surefire-reports" -name "*.txt" -type f | sort | while read -r txt_file; do + echo "--- File: $(basename "${txt_file}") ---" >> "${output_file}" + cat "${txt_file}" >> "${output_file}" + echo "" >> "${output_file}" + echo "--- End of $(basename "${txt_file}") ---" >> "${output_file}" + echo "" >> "${output_file}" + echo "" >> "${output_file}" + echo "" >> "${output_file}" + done + + echo "Concatenated test output for ${module_name} to ${output_file}" + else + echo "No .txt files found in ${module_dir}target/surefire-reports" + fi + fi +done \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index 7433d19e..748c1a64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,35 +5,7 @@ orbs: browser-tools: circleci/browser-tools@1.4.7 commands: - store_jacoco_site: - parameters: - module: - type: string - steps: - - store_artifacts: - path: << parameters.module >>/target/site/jacoco/index.html - when: always - - store_artifacts: - path: << parameters.module >>/target/site/jacoco/jacoco-resources - when: always - - install_java17: - steps: - - run: - name: Install Java 17 - command: | - sudo apt-get update - sudo apt install -y openjdk-17-jdk - sudo rm /etc/alternatives/java - sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java - - run: - ## used by jacoco uncovered class reporting in pom.xml - name: Install html2text - command: | - sudo apt-get update - sudo apt-get install -y html2text - - mvn_verify: + mvn_build: steps: - checkout - restore_cache: @@ -45,30 +17,41 @@ commands: name: Write .env command: | echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env + - run: + name: Run Maven Compile + command: | + mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress compile + - save_cache: + paths: + - ~/.m2 + key: v1-dependencies-{{ checksum "pom.xml" }} + + mvn_verify: + steps: + - checkout + - restore_cache: + keys: + - v1-dependencies-{{ checksum "pom.xml" }} - run: name: Run Maven Verify command: | - mvn -s .circleci/mvn-settings.xml -T4 verify - - store_jacoco_site: - module: qqq-backend-core - - store_jacoco_site: - module: qqq-backend-module-filesystem - - store_jacoco_site: - module: qqq-backend-module-rdbms - - store_jacoco_site: - module: qqq-backend-module-api - - store_jacoco_site: - module: qqq-middleware-api - - store_jacoco_site: - module: qqq-middleware-javalin - - store_jacoco_site: - module: qqq-middleware-picocli - - store_jacoco_site: - module: qqq-middleware-slack - - store_jacoco_site: - module: qqq-language-support-javascript - - store_jacoco_site: - module: qqq-sample-project + mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress verify + - run: + name: Collect JaCoCo reports + command: .circleci/collect-jacoco-reports.sh + when: always + - store_artifacts: + path: /home/circleci/jacoco-reports + destination: jacoco-reports + when: always + - run: + name: Concatenate test output files + command: .circleci/concatenate-test-output.sh + when: always + - store_artifacts: + path: /home/circleci/test-output-artifacts + destination: test-output + when: always - run: name: Save test results command: | @@ -77,10 +60,6 @@ commands: when: always - store_test_results: path: ~/test-results - - save_cache: - paths: - - ~/.m2 - key: v1-dependencies-{{ checksum "pom.xml" }} check_middleware_api_versions: steps: @@ -91,8 +70,8 @@ commands: - run: name: Build and Run ValidateApiVersions command: | - mvn -s .circleci/mvn-settings.xml -T4 install -DskipTests - mvn -s .circleci/mvn-settings.xml -pl qqq-middleware-javalin package appassembler:assemble -DskipTests + mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress install -DskipTests + mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress -pl qqq-middleware-javalin package appassembler:assemble -DskipTests qqq-middleware-javalin/target/appassembler/bin/ValidateApiVersions -r $(pwd) mvn_jar_deploy: @@ -108,7 +87,7 @@ commands: - run: name: Run Maven Jar Deploy command: | - mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy + mvn -s .circleci/mvn-settings.xml -T4 --no-transfer-progress flatten:flatten jar:jar deploy:deploy - save_cache: paths: - ~/.m2 @@ -135,19 +114,25 @@ commands: when: always jobs: - mvn_test: + build: + executor: localstack/default + steps: + - mvn_build + + test: executor: localstack/default steps: - ## - localstack/startup - - install_java17 - mvn_verify + + api_version_check: + executor: localstack/default + steps: - check_middleware_api_versions mvn_deploy: executor: localstack/default steps: - ## - localstack/startup - - install_java17 + - mvn_build - mvn_verify - check_middleware_api_versions - mvn_jar_deploy @@ -161,13 +146,31 @@ jobs: workflows: test_only: jobs: - - mvn_test: + - build: context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] filters: branches: ignore: /(dev|integration.*)/ tags: ignore: /(version|snapshot)-.*/ + - test: + context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] + requires: + - build + filters: + branches: + ignore: /(dev|integration.*)/ + tags: + ignore: /(version|snapshot)-.*/ + - api_version_check: + context: [ qqq-maven-registry-credentials, build-qqq-sample-app ] + requires: + - build + filters: + branches: + ignore: /(dev|integration.*)/ + tags: + ignore: /(version|snapshot)-.*/ deploy: jobs: diff --git a/README.md b/README.md index 797f0e0b..8a810ee7 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,20 @@ There are a few useful IntelliJ settings files, under `qqq-dev-tools/intellij`: One will likely also want the [Kingsrook Commentator Plugin](https://plugins.jetbrains.com/plugin/19325-kingsrook-commentator). +## Test Logging +By default, when ran from the command line, mvn surefire will make each test's +output (e.g., System.out, err, printStackTrace, and all logger calls) go into a +file under target/surefire-reports/${className}.txt. + +The system property `-DtestOutputToFile=false` can be given on the command line +to get all of this output on the console. + +In the IDE (e.g,. IntelliJ), output goes to the Console. + +In CircleCI, output goes to files, and those files are concatenated together and +stored as artifacts. + + ## License QQQ - Low-code Application Framework for Engineers. \ Copyright (C) 2020-2024. Kingsrook, LLC \ diff --git a/pom.xml b/pom.xml index a8d48433..0f55dd18 100644 --- a/pom.xml +++ b/pom.xml @@ -59,6 +59,7 @@ 0.80 0.95 none + true @@ -141,6 +142,8 @@ @{jaCoCoArgLine} + + ${testOutputToFile} @@ -244,30 +247,29 @@ if [ ! -e target/site/jacoco/index.html ]; then fi echo -echo "Jacoco coverage summary report:" +echo "Jacoco coverage summary report for module: ${project.artifactId}" echo " See also target/site/jacoco/index.html" echo " and https://www.jacoco.org/jacoco/trunk/doc/counters.html" echo "------------------------------------------------------------" -if which xpath > /dev/null 2>&1; then - echo "Element\nInstructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers - xpath -n -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values +# Parse Jacoco HTML coverage summary +if [ -f target/site/jacoco/index.html ]; then + echo -e "Instructions Missed\nInstruction Coverage\nBranches Missed\nBranch Coverage\nComplexity Missed\nComplexity Hit\nLines Missed\nLines Hit\nMethods Missed\nMethods Hit\nClasses Missed\nClasses Hit\n" > /tmp/$$.headers + sed 's/<\/\w\+>/&\n/g' target/site/jacoco/index.html | grep -A 12 '' | grep '\([^<]*\)<\/td>/\1/' | grep -v Total > /tmp/$$.values paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' rm /tmp/$$.headers /tmp/$$.values else - echo "xpath is not installed. Jacoco coverage summary will not be produced here..."; + echo "Jacoco coverage summary was not found."; fi +echo "-----------------------------" +echo -if which html2text > /dev/null 2>&1; then - echo "Untested classes, per Jacoco:" - echo "-----------------------------" - for i in target/site/jacoco/*/index.html; do - html2text -width 500 -nobs $i | sed '1,/^Total/d;' | grep -v Created | sed 's/ \+/ /g' | sed 's/ [[:digit:]]$//' | grep -v 0$ | cut -d' ' -f1; - done; - echo -else - echo "html2text is not installed. Untested classes from Jacoco will not be printed here..."; -fi +echo "Untested classes, per Jacoco for module: ${project.artifactId}" +echo "-----------------------------" +# Parse Jacoco XML reports directly to find classes with 0% coverage +sed 's//&\n/g;s/<\/class>/&\n/g' target/site/jacoco/jacoco.xml | grep -v 'counter type="CLASS" missed="0"' | sed 's/>.*//;s/.*\///;s/".*//' +echo "-----------------------------" +echo ]]> diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index c389a018..ec34d48e 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -55,11 +55,6 @@ sshd-sftp 2.14.0 - - org.apache.sshd - sshd-sftp - 2.14.0 - cloud.localstack diff --git a/qqq-middleware-javalin/pom.xml b/qqq-middleware-javalin/pom.xml index 3cfe058d..1099db78 100644 --- a/qqq-middleware-javalin/pom.xml +++ b/qqq-middleware-javalin/pom.xml @@ -122,6 +122,18 @@ src/main/java + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.3 + + + + ${project.basedir}/src/test/resources/static-site.jar + + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 0213141b..ad6ccb23 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -1932,6 +1932,7 @@ public class QJavalinImplementation { String searchTerm = context.queryParam("searchTerm"); String ids = context.queryParam("ids"); + String labels = context.queryParam("labels"); SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); setupSession(context, input); @@ -1945,6 +1946,11 @@ public class QJavalinImplementation List idList = new ArrayList<>(Arrays.asList(ids.split(","))); input.setIdList(idList); } + else if(StringUtils.hasContent(labels)) + { + List labelList = new ArrayList<>(Arrays.asList(labels.split(","))); + input.setLabelList(labelList); + } SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java index c6504be1..e30c94a9 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/routeproviders/SimpleFileSystemDirectoryRouter.java @@ -49,16 +49,14 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInterface { private static final QLogger LOG = QLogger.getLogger(SimpleFileSystemDirectoryRouter.class); + public static boolean loadStaticFilesFromJar = false; + private final String hostedPath; private final String fileSystemPath; - private QCodeReference routeAuthenticator; - private QInstance qInstance; - - /******************************************************************************* ** Constructor ** @@ -67,6 +65,24 @@ public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInt { this.hostedPath = hostedPath; this.fileSystemPath = fileSystemPath; + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // read the property to see if we should load static files from the jar file or from the file system // + // Javan only supports loading via one method per path, so its a choice of one or the other... // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + try + { + String propertyName = "qqq.javalin.enableStaticFilesFromJar"; // TODO: make a more general way to handle properties like this system-wide via a central config class + String propertyValue = System.getProperty(propertyName, ""); + if(propertyValue.equals("true")) + { + loadStaticFilesFromJar = true; + } + } + catch(Exception e) + { + e.printStackTrace(); + } } @@ -98,25 +114,41 @@ public class SimpleFileSystemDirectoryRouter implements QJavalinRouteProviderInt ***************************************************************************/ private void handleJavalinStaticFileConfig(StaticFileConfig staticFileConfig) { - URL resource = getClass().getClassLoader().getResource(fileSystemPath); - if(resource == null) - { - String message = "Could not find file system path: " + fileSystemPath; - if(fileSystemPath.startsWith("/") && getClass().getClassLoader().getResource(fileSystemPath.replaceFirst("^/+", "")) != null) - { - message += ". For non-absolute paths, do not prefix with a leading slash."; - } - throw new RuntimeException(message); - } if(!hostedPath.startsWith("/")) { LOG.warn("hostedPath [" + hostedPath + "] should probably start with a leading slash..."); } - staticFileConfig.directory = resource.getFile(); - staticFileConfig.hostedPath = hostedPath; - staticFileConfig.location = Location.EXTERNAL; + /// ///////////////////////////////////////////////////////////////////////////////////// + // Handle loading static files from the jar OR the filesystem based on system property // + /// ///////////////////////////////////////////////////////////////////////////////////// + if(SimpleFileSystemDirectoryRouter.loadStaticFilesFromJar) + { + staticFileConfig.directory = fileSystemPath; + staticFileConfig.hostedPath = hostedPath; + staticFileConfig.location = Location.CLASSPATH; + LOG.info("Static File Config : hostedPath [" + hostedPath + "] : directory [" + staticFileConfig.directory + "] : location [CLASSPATH]"); + } + else + { + URL resource = getClass().getClassLoader().getResource(fileSystemPath); + if(resource == null) + { + String message = "Could not find file system path: " + fileSystemPath; + if(fileSystemPath.startsWith("/") && getClass().getClassLoader().getResource(fileSystemPath.replaceFirst("^/+", "")) != null) + { + message += ". For non-absolute paths, do not prefix with a leading slash."; + } + throw new RuntimeException(message); + } + + staticFileConfig.directory = resource.getFile(); + staticFileConfig.hostedPath = hostedPath; + staticFileConfig.location = Location.EXTERNAL; + LOG.info("Static File Config : hostedPath [" + hostedPath + "] : directory [" + staticFileConfig.directory + "] : location [EXTERNAL]"); + } + } diff --git a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java index a6bf4085..a9524022 100644 --- a/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java +++ b/qqq-middleware-javalin/src/test/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServerTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.javalin.TestUtils; +import com.kingsrook.qqq.middleware.javalin.routeproviders.SimpleFileSystemDirectoryRouter; import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; import io.javalin.http.HttpStatus; import kong.unirest.HttpResponse; @@ -52,6 +53,16 @@ class QApplicationJavalinServerTest + /*************************************************************************** + ** + ***************************************************************************/ + private static AbstractQQQApplication getQqqApplication() + { + return new TestApplication(); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -60,6 +71,7 @@ class QApplicationJavalinServerTest { javalinServer.stop(); TestApplication.callCount = 0; + System.clearProperty("qqq.javalin.enableStaticFilesFromJar"); } @@ -196,6 +208,48 @@ class QApplicationJavalinServerTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStaticRouterFilesFromExternal() throws Exception + { + System.setProperty("qqq.javalin.enableStaticFilesFromJar", "false"); + + javalinServer = new QApplicationJavalinServer(getQqqApplication()) + .withServeFrontendMaterialDashboard(false) + .withPort(PORT); + javalinServer.start(); + + Unirest.config().setDefaultResponseEncoding("UTF-8"); + HttpResponse response = Unirest.get("http://localhost:" + PORT + "/statically-served/foo.html").asString(); + assertEquals("Foo? Bar!", response.getBody()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStaticRouterFilesFromClasspath() throws Exception + { + System.setProperty("qqq.javalin.enableStaticFilesFromJar", "true"); + + javalinServer = new QApplicationJavalinServer(new QApplicationJavalinServerTest.TestApplication()) + .withServeFrontendMaterialDashboard(false) + .withPort(PORT) + .withAdditionalRouteProvider(new SimpleFileSystemDirectoryRouter("/statically-served-from-jar", "static-site-from-jar/")); + + javalinServer.start(); + + Unirest.config().setDefaultResponseEncoding("UTF-8"); + HttpResponse response = Unirest.get("http://localhost:" + PORT + "/statically-served-from-jar/foo-in-jar.html").asString(); + assertEquals("Foo in a Jar!\n", response.getBody()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -296,16 +350,6 @@ class QApplicationJavalinServerTest - /*************************************************************************** - ** - ***************************************************************************/ - private static AbstractQQQApplication getQqqApplication() - { - return new TestApplication(); - } - - - /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-middleware-javalin/src/test/resources/static-site.jar b/qqq-middleware-javalin/src/test/resources/static-site.jar new file mode 100644 index 00000000..f5e203e4 Binary files /dev/null and b/qqq-middleware-javalin/src/test/resources/static-site.jar differ