Merged dev into feature/CE-1460-export-and-join-bugs

.github/actions/install_asciidoctor/action.yml

@@ -0,0 +1,10 @@
+name: install_asciidoctor
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4.1.0
+  - name: Install asciidoctor
+    run: |-
+      sudo apt-get update
+      sudo apt install -y asciidoctor
+    shell: bash

.github/actions/install_java17/action.yml

@@ -0,0 +1,16 @@
+name: install_java17
+runs:
+  using: composite
+  steps:
+  - name: Install Java 17
+    run: |-
+      sudo apt-get update
+      sudo apt install -y openjdk-17-jdk
+      sudo rm /etc/alternatives/java
+      sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
+    shell: bash
+  - name: Install html2text
+    run: |-
+      sudo apt-get update
+      sudo apt-get install -y html2text
+    shell: bash

.github/actions/mvn_jar_deploy/action.yml

@@ -0,0 +1,22 @@
+name: mvn_jar_deploy
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4.1.0
+  - name: Adjust pom version
+    run: ".circleci/adjust-pom-version.sh"
+    shell: bash
+  - name: restore_cache
+    uses: actions/cache@v3.3.2
+    with:
+      key: v1-dependencies-{{ checksum "pom.xml" }}
+      path: UPDATE_ME
+      restore-keys: v1-dependencies-{{ checksum "pom.xml" }}
+  - name: Run Maven Jar Deploy
+    run: mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy
+    shell: bash
+  - name: save_cache
+    uses: actions/cache@v3.3.2
+    with:
+      path: "~/.m2"
+      key: v1-dependencies-{{ checksum "pom.xml" }}

.github/actions/mvn_verify/action.yml

@@ -0,0 +1,61 @@
+name: mvn_verify
+runs:
+  using: composite
+  steps:
+  - uses: actions/checkout@v4.1.0
+  - name: restore_cache
+    uses: actions/cache@v3.3.2
+    with:
+      key: v1-dependencies-{{ checksum "pom.xml" }}
+      path: UPDATE_ME
+      restore-keys: v1-dependencies-{{ checksum "pom.xml" }}
+  - name: Write .env
+    run: echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env
+    shell: bash
+  - name: Run Maven Verify
+    run: mvn -s .circleci/mvn-settings.xml -T4 verify
+    shell: bash
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-core
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-module-filesystem
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-module-rdbms
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-backend-module-api
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-api
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-javalin
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-picocli
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-middleware-slack
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-language-support-javascript
+  - uses: "./.github/actions/store_jacoco_site"
+    with:
+      module: qqq-sample-project
+  - name: Save test results
+    run: |-
+      mkdir -p ~/test-results/junit/
+      find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} ~/test-results/junit/ \;
+    if: always()
+    shell: bash
+  - uses: actions/upload-artifact@v4.1.0
+    with:
+      path: "~/test-results"
+  - name: save_cache
+    uses: actions/cache@v3.3.2
+    with:
+      path: "~/.m2"
+      key: v1-dependencies-{{ checksum "pom.xml" }}

.github/actions/run_asciidoctor/action.yml

@@ -0,0 +1,9 @@
+name: run_asciidoctor
+runs:
+  using: composite
+  steps:
+  - name: Run asciidoctor
+    run: |-
+      cd docs
+      asciidoctor -a docinfo=shared index.adoc
+    shell: bash

.github/actions/store_jacoco_site/action.yml

@@ -0,0 +1,13 @@
+name: store_jacoco_site
+inputs:
+  module:
+    required: false
+runs:
+  using: composite
+  steps:
+  - uses: actions/upload-artifact@v4.1.0
+    with:
+      path: "${{ inputs.module }}/target/site/jacoco/index.html"
+  - uses: actions/upload-artifact@v4.1.0
+    with:
+      path: "${{ inputs.module }}/target/site/jacoco/jacoco-resources"

.github/actions/upload_docs_site/action.yml

@@ -0,0 +1,9 @@
+name: upload_docs_site
+runs:
+  using: composite
+  steps:
+  - name: scp html to justinsgotskinnylegs.com
+    run: |-
+      cd docs
+      scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
+    shell: bash

.github/workflows/codacy.yml

@@ -0,0 +1,61 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# This workflow checks out code, performs a Codacy security scan
+# and integrates the results with the
+# GitHub Advanced Security code scanning feature.  For more information on
+# the Codacy security scan action usage and parameters, see
+# https://github.com/codacy/codacy-analysis-cli-action.
+# For more information on Codacy Analysis CLI in general, see
+# https://github.com/codacy/codacy-analysis-cli.
+
+name: Codacy Security Scan
+
+on:
+  push:
+    branches: [ "security" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "security" ]
+  schedule:
+    - cron: '26 5 * * 4'
+
+permissions:
+  contents: read
+
+jobs:
+  codacy-security-scan:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
+    name: Codacy Security Scan
+    runs-on: ubuntu-latest
+    steps:
+      # Checkout the repository to the GitHub Actions runner
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis
+      - name: Run Codacy Analysis CLI
+        uses: codacy/codacy-analysis-cli-action@d840f886c4bd4edc059706d09c6a1586111c540b
+        with:
+          # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository
+          # You can also omit the token and run the tools that support default configurations
+          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+          verbose: true
+          output: results.sarif
+          format: sarif
+          # Adjust severity of non-security issues
+          gh-code-scanning-compat: true
+          # Force 0 exit code to allow SARIF file generation
+          # This will handover control about PR rejection to the GitHub side
+          max-allowed-issues: 2147483647
+
+      # Upload the SARIF file generated in the previous step
+      - name: Upload SARIF results file
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: results.sarif

.github/workflows/codeql.yml

@@ -0,0 +1,93 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "security" ]
+  pull_request:
+    branches: [ "security" ]
+  schedule:
+    - cron: '31 10 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners (GitHub.com only)
+    # Consider using larger runners or machines with greater resources for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      # required for all workflows
+      security-events: write
+
+      # required to fetch internal or private CodeQL packs
+      packages: read
+
+      # only required for workflows in private repositories
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - language: java-kotlin
+          build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too.
+        # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
+        # Use `c-cpp` to analyze code written in C, C++ or both
+        # Use 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
+        # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
+        # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
+        # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        build-mode: ${{ matrix.build-mode }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+    # If the analyze step fails for one of the languages you are analyzing with
+    # "We were unable to automatically build your code", modify the matrix above
+    # to set the build mode to "manual" for that language. Then modify this step
+    # to build your code.
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+    - if: matrix.build-mode == 'manual'
+      shell: bash
+      run: |
+        echo 'If you are using a "manual" build mode for one or more of the' \
+          'languages you are analyzing, replace this with the commands to build' \
+          'your code, for example:'
+        echo '  make bootstrap'
+        echo '  make release'
+        exit 1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/deploy.yml

@@ -0,0 +1,20 @@
+name: Kingsrook/qqq/deploy
+on:
+  push:
+    branches:
+    - release
+jobs:
+  mvn_deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/install_java17"
+    - uses: "./.github/actions/mvn_verify"
+    - uses: "./.github/actions/mvn_jar_deploy"
+  publish_asciidoc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/install_asciidoctor"
+    - uses: "./.github/actions/run_asciidoctor"
+    - uses: "./.github/actions/upload_docs_site"

.github/workflows/test_only.yml

@@ -0,0 +1,12 @@
+name: Kingsrook/qqq/test_only
+on:
+  push:
+    branches:
+    - release
+jobs:
+  mvn_test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/install_java17"
+    - uses: "./.github/actions/mvn_verify"

.gitignore

@@ -35,3 +35,5 @@ hs_err_pid*
 *.swp
 .flattened-pom.xml
 dependency-reduced-pom.xml
+/.env.local
+/.cache/

README.md

@@ -19,7 +19,7 @@ You can also use fine-grained jars:
 
 ## License
 QQQ - Low-code Application Framework for Engineers. \
-Copyright (C) 2022.  Kingsrook, LLC \
+Copyright (C) 2020-2024.  Kingsrook, LLC \
 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States \
 contact@kingsrook.com | https://github.com/Kingsrook/
 

SECURITY.md

@@ -0,0 +1,21 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 5.1.x   | :white_check_mark: |
+| 5.0.x   | :x:                |
+| 4.0.x   | :white_check_mark: |
+| < 4.0   | :x:                |
+
+## Reporting a Vulnerability
+
+Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc.

pom.xml

@@ -46,7 +46,7 @@
    </modules>
 
    <properties>
-      <revision>0.20.0-SNAPSHOT</revision>
+      <revision>0.21.0-SNAPSHOT</revision>
 
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -209,6 +209,7 @@
                   <productionBranch>main</productionBranch>
                   <developmentBranch>dev</developmentBranch>
                   <versionTagPrefix>version-</versionTagPrefix>
+                  <releaseBranchPrefix>rel/</releaseBranchPrefix>
                </gitFlowConfig>
                <skipFeatureVersion>true</skipFeatureVersion> <!-- Keep feature names out of versions -->
                <postReleaseGoals>install</postReleaseGoals>  <!-- Let CI run deploys -->

qodana.yaml

@@ -0,0 +1,31 @@
+#-------------------------------------------------------------------------------#
+#               Qodana analysis is configured by qodana.yaml file               #
+#             https://www.jetbrains.com/help/qodana/qodana-yaml.html            #
+#-------------------------------------------------------------------------------#
+version: "1.0"
+
+#Specify inspection profile for code analysis
+profile:
+  name: qodana.starter
+
+#Enable inspections
+#include:
+#  - name: <SomeEnabledInspectionId>
+
+#Disable inspections
+#exclude:
+#  - name: <SomeDisabledInspectionId>
+#    paths:
+#      - <path/where/not/run/inspection>
+
+projectJDK: 17 #(Applied in CI/CD pipeline)
+
+#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
+#bootstrap: sh ./prepare-qodana.sh
+
+#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
+#plugins:
+#  - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
+
+#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
+linter: jetbrains/qodana-jvm:latest

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java

@@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperat
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
@@ -430,6 +431,12 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
          queryInput.setAssociationNamesToInclude(associationNamesToInclude);
       }
 
+      Collection<QueryJoin> joins = getQueryJoins();
+      if(CollectionUtils.nullSafeHasContents(joins))
+      {
+         queryInput.setQueryJoins(getQueryJoins());
+      }
+
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
       return (buildExistingRecordsMap(destinationTableForeignKeyField, queryOutput.getRecords()));
    }
@@ -443,7 +450,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
     ** Note, if you're overriding this method, you'll likely also want & need to
     ** override getExistingRecord.
     *******************************************************************************/
-   protected Map<Pair<String, Serializable>, QRecord> buildExistingRecordsMap(String destinationTableForeignKeyField, List<QRecord> existingRecordList)
+   protected Map<Pair<String, Serializable>, QRecord> buildExistingRecordsMap(String destinationTableForeignKeyField, List<QRecord> existingRecordList) throws QException
    {
       Map<Pair<String, Serializable>, QRecord> existingRecordsByForeignKey = new HashMap<>();
       for(QRecord record : existingRecordList)
@@ -455,6 +462,16 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   protected List<QueryJoin> getQueryJoins()
+   {
+      return null;
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-dev-tools/CURRENT-SNAPSHOT-VERSION

@@ -1 +1 @@
-0.20.0
+0.21.0

qqq-middleware-picocli/src/test/java/com/kingsrook/qqq/frontend/picocli/QPicoCliImplementationTest.java

@@ -70,6 +70,7 @@ class QPicoCliImplementationTest
    @BeforeEach
    public void beforeEach() throws Exception
    {
+      System.setProperty("picocli.ansi", "false");
       TestUtils.primeTestDatabase();
       QContext.init(TestUtils.defineInstance(), new QSession());
    }

qqq-sample-project/pom.xml

@@ -68,7 +68,7 @@
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-frontend-material-dashboard</artifactId>
-         <version>0.20.0-20240418.180316-42</version>
+         <version>0.20.0</version>
       </dependency>
       <dependency>
          <groupId>com.h2database</groupId>