mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-21 22:48:44 +00:00
Compare commits
38 Commits
snapshot-f
...
version-0.
Author | SHA1 | Date | |
---|---|---|---|
31edb6a7fe | |||
338670118d | |||
32573bdf78 | |||
bd683253a5 | |||
75279c2e6c | |||
9e33ac564d | |||
7fe7c2d0a0 | |||
79b9f0e921 | |||
a75ec9a0c5 | |||
b11f1fb394 | |||
3c927693f1 | |||
cb41f239b8 | |||
8648c67a98 | |||
d11ae90ad6 | |||
8395dfaa52 | |||
3273e56b17 | |||
9b1786dc01 | |||
429513f337 | |||
6ee8dad45f | |||
f380d44dd2 | |||
2cf14e543c | |||
1669741d19 | |||
2be41d8714 | |||
3e7e416a2a | |||
b377af846a | |||
6b5b971368 | |||
f6e09f1d57 | |||
f069358764 | |||
c509b6da38 | |||
e788929d67 | |||
18c94943cb | |||
b24a990043 | |||
a4295df20d | |||
3398b812ce | |||
9e9f266878 | |||
7cbd6705e1 | |||
1eb078d916 | |||
82201286d4 |
10
.github/actions/install_asciidoctor/action.yml
vendored
Normal file
10
.github/actions/install_asciidoctor/action.yml
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
name: install_asciidoctor
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/checkout@v4.1.0
|
||||
- name: Install asciidoctor
|
||||
run: |-
|
||||
sudo apt-get update
|
||||
sudo apt install -y asciidoctor
|
||||
shell: bash
|
16
.github/actions/install_java17/action.yml
vendored
Normal file
16
.github/actions/install_java17/action.yml
vendored
Normal file
@ -0,0 +1,16 @@
|
||||
name: install_java17
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install Java 17
|
||||
run: |-
|
||||
sudo apt-get update
|
||||
sudo apt install -y openjdk-17-jdk
|
||||
sudo rm /etc/alternatives/java
|
||||
sudo ln -s /usr/lib/jvm/java-17-openjdk-amd64/bin/java /etc/alternatives/java
|
||||
shell: bash
|
||||
- name: Install html2text
|
||||
run: |-
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y html2text
|
||||
shell: bash
|
22
.github/actions/mvn_jar_deploy/action.yml
vendored
Normal file
22
.github/actions/mvn_jar_deploy/action.yml
vendored
Normal file
@ -0,0 +1,22 @@
|
||||
name: mvn_jar_deploy
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/checkout@v4.1.0
|
||||
- name: Adjust pom version
|
||||
run: ".circleci/adjust-pom-version.sh"
|
||||
shell: bash
|
||||
- name: restore_cache
|
||||
uses: actions/cache@v3.3.2
|
||||
with:
|
||||
key: v1-dependencies-{{ checksum "pom.xml" }}
|
||||
path: UPDATE_ME
|
||||
restore-keys: v1-dependencies-{{ checksum "pom.xml" }}
|
||||
- name: Run Maven Jar Deploy
|
||||
run: mvn -s .circleci/mvn-settings.xml -T4 flatten:flatten jar:jar deploy:deploy
|
||||
shell: bash
|
||||
- name: save_cache
|
||||
uses: actions/cache@v3.3.2
|
||||
with:
|
||||
path: "~/.m2"
|
||||
key: v1-dependencies-{{ checksum "pom.xml" }}
|
61
.github/actions/mvn_verify/action.yml
vendored
Normal file
61
.github/actions/mvn_verify/action.yml
vendored
Normal file
@ -0,0 +1,61 @@
|
||||
name: mvn_verify
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/checkout@v4.1.0
|
||||
- name: restore_cache
|
||||
uses: actions/cache@v3.3.2
|
||||
with:
|
||||
key: v1-dependencies-{{ checksum "pom.xml" }}
|
||||
path: UPDATE_ME
|
||||
restore-keys: v1-dependencies-{{ checksum "pom.xml" }}
|
||||
- name: Write .env
|
||||
run: echo "RDBMS_PASSWORD=$RDBMS_PASSWORD" >> qqq-sample-project/.env
|
||||
shell: bash
|
||||
- name: Run Maven Verify
|
||||
run: mvn -s .circleci/mvn-settings.xml -T4 verify
|
||||
shell: bash
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-backend-core
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-backend-module-filesystem
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-backend-module-rdbms
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-backend-module-api
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-middleware-api
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-middleware-javalin
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-middleware-picocli
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-middleware-slack
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-language-support-javascript
|
||||
- uses: "./.github/actions/store_jacoco_site"
|
||||
with:
|
||||
module: qqq-sample-project
|
||||
- name: Save test results
|
||||
run: |-
|
||||
mkdir -p ~/test-results/junit/
|
||||
find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} ~/test-results/junit/ \;
|
||||
if: always()
|
||||
shell: bash
|
||||
- uses: actions/upload-artifact@v4.1.0
|
||||
with:
|
||||
path: "~/test-results"
|
||||
- name: save_cache
|
||||
uses: actions/cache@v3.3.2
|
||||
with:
|
||||
path: "~/.m2"
|
||||
key: v1-dependencies-{{ checksum "pom.xml" }}
|
9
.github/actions/run_asciidoctor/action.yml
vendored
Normal file
9
.github/actions/run_asciidoctor/action.yml
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
name: run_asciidoctor
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Run asciidoctor
|
||||
run: |-
|
||||
cd docs
|
||||
asciidoctor -a docinfo=shared index.adoc
|
||||
shell: bash
|
13
.github/actions/store_jacoco_site/action.yml
vendored
Normal file
13
.github/actions/store_jacoco_site/action.yml
vendored
Normal file
@ -0,0 +1,13 @@
|
||||
name: store_jacoco_site
|
||||
inputs:
|
||||
module:
|
||||
required: false
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- uses: actions/upload-artifact@v4.1.0
|
||||
with:
|
||||
path: "${{ inputs.module }}/target/site/jacoco/index.html"
|
||||
- uses: actions/upload-artifact@v4.1.0
|
||||
with:
|
||||
path: "${{ inputs.module }}/target/site/jacoco/jacoco-resources"
|
9
.github/actions/upload_docs_site/action.yml
vendored
Normal file
9
.github/actions/upload_docs_site/action.yml
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
name: upload_docs_site
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: scp html to justinsgotskinnylegs.com
|
||||
run: |-
|
||||
cd docs
|
||||
scp index.html dkelkhoff@45.79.44.221:/mnt/first-volume/dkelkhoff/nginx/html/justinsgotskinnylegs.com/qqq-docs.html
|
||||
shell: bash
|
61
.github/workflows/codacy.yml
vendored
Normal file
61
.github/workflows/codacy.yml
vendored
Normal file
@ -0,0 +1,61 @@
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
# This workflow checks out code, performs a Codacy security scan
|
||||
# and integrates the results with the
|
||||
# GitHub Advanced Security code scanning feature. For more information on
|
||||
# the Codacy security scan action usage and parameters, see
|
||||
# https://github.com/codacy/codacy-analysis-cli-action.
|
||||
# For more information on Codacy Analysis CLI in general, see
|
||||
# https://github.com/codacy/codacy-analysis-cli.
|
||||
|
||||
name: Codacy Security Scan
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "security" ]
|
||||
pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
branches: [ "security" ]
|
||||
schedule:
|
||||
- cron: '26 5 * * 4'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
codacy-security-scan:
|
||||
permissions:
|
||||
contents: read # for actions/checkout to fetch code
|
||||
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
|
||||
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
|
||||
name: Codacy Security Scan
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout the repository to the GitHub Actions runner
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis
|
||||
- name: Run Codacy Analysis CLI
|
||||
uses: codacy/codacy-analysis-cli-action@d840f886c4bd4edc059706d09c6a1586111c540b
|
||||
with:
|
||||
# Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository
|
||||
# You can also omit the token and run the tools that support default configurations
|
||||
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
|
||||
verbose: true
|
||||
output: results.sarif
|
||||
format: sarif
|
||||
# Adjust severity of non-security issues
|
||||
gh-code-scanning-compat: true
|
||||
# Force 0 exit code to allow SARIF file generation
|
||||
# This will handover control about PR rejection to the GitHub side
|
||||
max-allowed-issues: 2147483647
|
||||
|
||||
# Upload the SARIF file generated in the previous step
|
||||
- name: Upload SARIF results file
|
||||
uses: github/codeql-action/upload-sarif@v2
|
||||
with:
|
||||
sarif_file: results.sarif
|
93
.github/workflows/codeql.yml
vendored
Normal file
93
.github/workflows/codeql.yml
vendored
Normal file
@ -0,0 +1,93 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "security" ]
|
||||
pull_request:
|
||||
branches: [ "security" ]
|
||||
schedule:
|
||||
- cron: '31 10 * * 3'
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
# Runner size impacts CodeQL analysis time. To learn more, please see:
|
||||
# - https://gh.io/recommended-hardware-resources-for-running-codeql
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners (GitHub.com only)
|
||||
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
|
||||
permissions:
|
||||
# required for all workflows
|
||||
security-events: write
|
||||
|
||||
# required to fetch internal or private CodeQL packs
|
||||
packages: read
|
||||
|
||||
# only required for workflows in private repositories
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- language: java-kotlin
|
||||
build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too.
|
||||
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
|
||||
# Use `c-cpp` to analyze code written in C, C++ or both
|
||||
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
|
||||
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
|
||||
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
|
||||
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
|
||||
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
|
||||
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
# If the analyze step fails for one of the languages you are analyzing with
|
||||
# "We were unable to automatically build your code", modify the matrix above
|
||||
# to set the build mode to "manual" for that language. Then modify this step
|
||||
# to build your code.
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
- if: matrix.build-mode == 'manual'
|
||||
shell: bash
|
||||
run: |
|
||||
echo 'If you are using a "manual" build mode for one or more of the' \
|
||||
'languages you are analyzing, replace this with the commands to build' \
|
||||
'your code, for example:'
|
||||
echo ' make bootstrap'
|
||||
echo ' make release'
|
||||
exit 1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
20
.github/workflows/deploy.yml
vendored
Normal file
20
.github/workflows/deploy.yml
vendored
Normal file
@ -0,0 +1,20 @@
|
||||
name: Kingsrook/qqq/deploy
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- release
|
||||
jobs:
|
||||
mvn_deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.1.0
|
||||
- uses: "./.github/actions/install_java17"
|
||||
- uses: "./.github/actions/mvn_verify"
|
||||
- uses: "./.github/actions/mvn_jar_deploy"
|
||||
publish_asciidoc:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.1.0
|
||||
- uses: "./.github/actions/install_asciidoctor"
|
||||
- uses: "./.github/actions/run_asciidoctor"
|
||||
- uses: "./.github/actions/upload_docs_site"
|
12
.github/workflows/test_only.yml
vendored
Normal file
12
.github/workflows/test_only.yml
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
name: Kingsrook/qqq/test_only
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- release
|
||||
jobs:
|
||||
mvn_test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4.1.0
|
||||
- uses: "./.github/actions/install_java17"
|
||||
- uses: "./.github/actions/mvn_verify"
|
2
.gitignore
vendored
2
.gitignore
vendored
@ -35,3 +35,5 @@ hs_err_pid*
|
||||
*.swp
|
||||
.flattened-pom.xml
|
||||
dependency-reduced-pom.xml
|
||||
/.env.local
|
||||
/.cache/
|
||||
|
@ -19,7 +19,7 @@ You can also use fine-grained jars:
|
||||
|
||||
## License
|
||||
QQQ - Low-code Application Framework for Engineers. \
|
||||
Copyright (C) 2022. Kingsrook, LLC \
|
||||
Copyright (C) 2020-2024. Kingsrook, LLC \
|
||||
651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States \
|
||||
contact@kingsrook.com | https://github.com/Kingsrook/
|
||||
|
||||
|
21
SECURITY.md
Normal file
21
SECURITY.md
Normal file
@ -0,0 +1,21 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Use this section to tell people about which versions of your project are
|
||||
currently being supported with security updates.
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| 5.1.x | :white_check_mark: |
|
||||
| 5.0.x | :x: |
|
||||
| 4.0.x | :white_check_mark: |
|
||||
| < 4.0 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Use this section to tell people how to report a vulnerability.
|
||||
|
||||
Tell them where to go, how often they can expect to get an update on a
|
||||
reported vulnerability, what to expect if the vulnerability is accepted or
|
||||
declined, etc.
|
@ -117,3 +117,19 @@ new QTableMetaData().withName("flights").withFields(List.of(
|
||||
.withBehavior(new DateTimeDisplayValueBehavior()
|
||||
.withDefaultZoneId("UTC"))
|
||||
----
|
||||
|
||||
===== CaseChangeBehavior
|
||||
A field can be made to always go through a toUpperCase or toLowerCase transformation, both before it is stored in a backend,
|
||||
and after it is read from a backend, by adding a CaseChangeBehavior to it:
|
||||
|
||||
[source,java]
|
||||
.Examples of using CaseChangeBehavior
|
||||
----
|
||||
new QTableMetaData().withName("item").withFields(List.of(
|
||||
|
||||
new QFieldMetaData("sku", QFieldType.STRING)
|
||||
.withBehavior(CaseChangeBehavior.TO_UPPER_CASE)),
|
||||
|
||||
new QFieldMetaData("username", QFieldType.STRING)
|
||||
.withBehavior(CaseChangeBehavior.TO_LOWER_CASE)),
|
||||
----
|
||||
|
5
pom.xml
5
pom.xml
@ -46,7 +46,7 @@
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<revision>0.20.0-SNAPSHOT</revision>
|
||||
<revision>0.20.0</revision>
|
||||
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
@ -55,7 +55,7 @@
|
||||
<maven.compiler.showDeprecation>true</maven.compiler.showDeprecation>
|
||||
<maven.compiler.showWarnings>true</maven.compiler.showWarnings>
|
||||
<coverage.haltOnFailure>true</coverage.haltOnFailure>
|
||||
<coverage.instructionCoveredRatioMinimum>0.75</coverage.instructionCoveredRatioMinimum>
|
||||
<coverage.instructionCoveredRatioMinimum>0.80</coverage.instructionCoveredRatioMinimum>
|
||||
<coverage.classCoveredRatioMinimum>0.95</coverage.classCoveredRatioMinimum>
|
||||
<plugin.shade.phase>none</plugin.shade.phase>
|
||||
</properties>
|
||||
@ -209,6 +209,7 @@
|
||||
<productionBranch>main</productionBranch>
|
||||
<developmentBranch>dev</developmentBranch>
|
||||
<versionTagPrefix>version-</versionTagPrefix>
|
||||
<releaseBranchPrefix>rel/</releaseBranchPrefix>
|
||||
</gitFlowConfig>
|
||||
<skipFeatureVersion>true</skipFeatureVersion> <!-- Keep feature names out of versions -->
|
||||
<postReleaseGoals>install</postReleaseGoals> <!-- Let CI run deploys -->
|
||||
|
31
qodana.yaml
Normal file
31
qodana.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
#-------------------------------------------------------------------------------#
|
||||
# Qodana analysis is configured by qodana.yaml file #
|
||||
# https://www.jetbrains.com/help/qodana/qodana-yaml.html #
|
||||
#-------------------------------------------------------------------------------#
|
||||
version: "1.0"
|
||||
|
||||
#Specify inspection profile for code analysis
|
||||
profile:
|
||||
name: qodana.starter
|
||||
|
||||
#Enable inspections
|
||||
#include:
|
||||
# - name: <SomeEnabledInspectionId>
|
||||
|
||||
#Disable inspections
|
||||
#exclude:
|
||||
# - name: <SomeDisabledInspectionId>
|
||||
# paths:
|
||||
# - <path/where/not/run/inspection>
|
||||
|
||||
projectJDK: 17 #(Applied in CI/CD pipeline)
|
||||
|
||||
#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
|
||||
#bootstrap: sh ./prepare-qodana.sh
|
||||
|
||||
#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
|
||||
#plugins:
|
||||
# - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
|
||||
|
||||
#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
|
||||
linter: jetbrains/qodana-jvm:latest
|
@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction;
|
||||
import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -185,8 +186,7 @@ public class AsyncRecordPipeLoop
|
||||
|
||||
if(recordCount > 0)
|
||||
{
|
||||
LOG.info(String.format("Processed %,d records", recordCount)
|
||||
+ String.format(" at end of job [%s] in %,d ms (%.2f records/second).", jobName, (endTime - jobStartTime), 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
|
||||
LOG.info("End of job summary", logPair("recordCount", recordCount), logPair("jobName", jobName), logPair("millis", endTime - jobStartTime), logPair("recordsPerSecond", 1000d * (recordCount / (.001d + (endTime - jobStartTime)))));
|
||||
}
|
||||
|
||||
return (recordCount);
|
||||
|
@ -208,7 +208,7 @@ public class GetAction
|
||||
return getFieldFilterBehaviorMemoization.getResult(key, (p) ->
|
||||
{
|
||||
List<FieldFilterBehavior<?>> rs = new ArrayList<>();
|
||||
for(FieldBehavior<?> fieldBehavior : tableMetaData.getFields().get(fieldName).getBehaviors())
|
||||
for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(tableMetaData.getFields().get(fieldName).getBehaviors()))
|
||||
{
|
||||
if(fieldBehavior instanceof FieldFilterBehavior<?> fieldFilterBehavior)
|
||||
{
|
||||
|
@ -159,6 +159,7 @@ public class ValueBehaviorApplier
|
||||
}
|
||||
}
|
||||
|
||||
QFilterCriteria criteriaToUse = criteria;
|
||||
if(field != null)
|
||||
{
|
||||
for(FieldBehavior<?> fieldBehavior : CollectionUtils.nonNullCollection(field.getBehaviors()))
|
||||
@ -175,23 +176,20 @@ public class ValueBehaviorApplier
|
||||
// call to apply the behavior on the criteria - which will return a //
|
||||
// new criteria if any values are changed, else the input criteria //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
QFilterCriteria newCriteria = apply(criteria, instance, table, field, filterBehavior);
|
||||
criteriaToUse = apply(criteriaToUse, instance, table, field, filterBehavior);
|
||||
|
||||
if(newCriteria != criteria)
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if the new criteria is not the same as the old criteria, mark that we need to make and return a clone. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(criteriaToUse != criteria)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if the new criteria is not the same as the old criteria, mark that we need to make and return a clone. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
newCriteriaList.add(newCriteria);
|
||||
needToUseClone = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
newCriteriaList.add(criteria);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newCriteriaList.add(criteriaToUse);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
@ -123,6 +123,18 @@ public class QInstanceEnricher
|
||||
*******************************************************************************/
|
||||
public void enrich()
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
// at one point, we did apps later - but - it was possible to put tables in an app's //
|
||||
// sections, but not its children list (enrichApp fixes this by adding such tables to //
|
||||
// the children list) so then when enrichTable runs, it looks for fields that are //
|
||||
// possible-values pointed at tables, for adding LINK adornments - and that could //
|
||||
// cause such links to be omitted, mysteriously! so, do app enrichment before tables. //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(qInstance.getApps() != null)
|
||||
{
|
||||
qInstance.getApps().values().forEach(this::enrichApp);
|
||||
}
|
||||
|
||||
if(qInstance.getTables() != null)
|
||||
{
|
||||
qInstance.getTables().values().forEach(this::enrichTable);
|
||||
@ -139,11 +151,6 @@ public class QInstanceEnricher
|
||||
qInstance.getBackends().values().forEach(this::enrichBackend);
|
||||
}
|
||||
|
||||
if(qInstance.getApps() != null)
|
||||
{
|
||||
qInstance.getApps().values().forEach(this::enrichApp);
|
||||
}
|
||||
|
||||
if(qInstance.getReports() != null)
|
||||
{
|
||||
qInstance.getReports().values().forEach(this::enrichReport);
|
||||
|
@ -107,8 +107,10 @@ public class AbstractActionInput
|
||||
/*******************************************************************************
|
||||
** Getter for instance
|
||||
**
|
||||
** Deprecated. Please use QContext.getInstance() instead
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
@Deprecated
|
||||
public QInstance getInstance()
|
||||
{
|
||||
return (QContext.getQInstance());
|
||||
@ -119,8 +121,10 @@ public class AbstractActionInput
|
||||
/*******************************************************************************
|
||||
** Getter for session
|
||||
**
|
||||
** Deprecated. Please use QContext.getSession() instead
|
||||
*******************************************************************************/
|
||||
@JsonIgnore
|
||||
@Deprecated
|
||||
public QSession getSession()
|
||||
{
|
||||
return (QContext.getQSession());
|
||||
|
@ -53,7 +53,7 @@ public class QueryOutput extends AbstractActionOutput implements Serializable
|
||||
}
|
||||
else
|
||||
{
|
||||
storage = new QueryOutputList();
|
||||
storage = new QueryOutputList(queryInput);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -24,7 +24,10 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -33,15 +36,50 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
*******************************************************************************/
|
||||
class QueryOutputList implements QueryOutputStorageInterface
|
||||
{
|
||||
private List<QRecord> records = new ArrayList<>();
|
||||
private static final QLogger LOG = QLogger.getLogger(QueryOutputList.class);
|
||||
|
||||
private final String tableName;
|
||||
private List<QRecord> records = new ArrayList<>();
|
||||
|
||||
private static int LOG_SIZE_INFO_OVER = 50_000;
|
||||
private static int LOG_SIZE_WARN_OVER = 100_000;
|
||||
private static int LOG_SIZE_ERROR_OVER = 250_000;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public QueryOutputList()
|
||||
public QueryOutputList(QueryInput queryInput)
|
||||
{
|
||||
tableName = queryInput.getTableName();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void logSize(int sizeBefore, int sizeAfter)
|
||||
{
|
||||
Level level = null;
|
||||
if(sizeBefore < LOG_SIZE_ERROR_OVER && sizeAfter >= LOG_SIZE_ERROR_OVER)
|
||||
{
|
||||
level = Level.ERROR;
|
||||
}
|
||||
else if(sizeBefore < LOG_SIZE_WARN_OVER && sizeAfter >= LOG_SIZE_WARN_OVER)
|
||||
{
|
||||
level = Level.WARN;
|
||||
}
|
||||
else if(sizeBefore < LOG_SIZE_INFO_OVER && sizeAfter >= LOG_SIZE_INFO_OVER)
|
||||
{
|
||||
level = Level.INFO;
|
||||
}
|
||||
|
||||
if(level != null)
|
||||
{
|
||||
LOG.log(level, "Large number of records in QueryOutputList", new Throwable(), logPair("noRecords", sizeAfter), logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -52,7 +90,9 @@ class QueryOutputList implements QueryOutputStorageInterface
|
||||
@Override
|
||||
public void addRecord(QRecord record)
|
||||
{
|
||||
int sizeBefore = this.records.size();
|
||||
records.add(record);
|
||||
logSize(sizeBefore, this.records.size());
|
||||
}
|
||||
|
||||
|
||||
@ -63,7 +103,9 @@ class QueryOutputList implements QueryOutputStorageInterface
|
||||
@Override
|
||||
public void addRecords(List<QRecord> records)
|
||||
{
|
||||
int sizeBefore = this.records.size();
|
||||
this.records.addAll(records);
|
||||
logSize(sizeBefore, this.records.size());
|
||||
}
|
||||
|
||||
|
||||
@ -77,4 +119,36 @@ class QueryOutputList implements QueryOutputStorageInterface
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for LOG_SIZE_INFO_OVER
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static void setLogSizeInfoOver(int logSizeInfoOver)
|
||||
{
|
||||
QueryOutputList.LOG_SIZE_INFO_OVER = logSizeInfoOver;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for LOG_SIZE_WARN_OVER
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static void setLogSizeWarnOver(int logSizeWarnOver)
|
||||
{
|
||||
QueryOutputList.LOG_SIZE_WARN_OVER = logSizeWarnOver;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for LOG_SIZE_ERROR_OVER
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static void setLogSizeErrorOver(int logSizeErrorOver)
|
||||
{
|
||||
QueryOutputList.LOG_SIZE_ERROR_OVER = logSizeErrorOver;
|
||||
}
|
||||
}
|
||||
|
@ -57,7 +57,7 @@ public class QFrontendFieldMetaData
|
||||
private List<FieldAdornment> adornments;
|
||||
private List<QHelpContent> helpContents;
|
||||
|
||||
private List<FieldBehaviorForFrontend> fieldBehaviors;
|
||||
private List<FieldBehaviorForFrontend> behaviors;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
// do not add setters. take values from the source-object in the constructor!! //
|
||||
@ -86,11 +86,11 @@ public class QFrontendFieldMetaData
|
||||
{
|
||||
if(behavior instanceof FieldBehaviorForFrontend fbff)
|
||||
{
|
||||
if(fieldBehaviors == null)
|
||||
if(behaviors == null)
|
||||
{
|
||||
fieldBehaviors = new ArrayList<>();
|
||||
behaviors = new ArrayList<>();
|
||||
}
|
||||
fieldBehaviors.add(fbff);
|
||||
behaviors.add(fbff);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -222,8 +222,8 @@ public class QFrontendFieldMetaData
|
||||
** Getter for fieldBehaviors
|
||||
**
|
||||
*******************************************************************************/
|
||||
public List<FieldBehaviorForFrontend> getFieldBehaviors()
|
||||
public List<FieldBehaviorForFrontend> getBehaviors()
|
||||
{
|
||||
return fieldBehaviors;
|
||||
return behaviors;
|
||||
}
|
||||
}
|
||||
|
@ -245,9 +245,9 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
// extract keys from source records //
|
||||
//////////////////////////////////////
|
||||
List<Serializable> sourceKeyList = runBackendStepInput.getRecords().stream()
|
||||
.map(r -> r.getValueString(sourceTableKeyField))
|
||||
.map(r -> extractSourceKeyValueFromRecord(r, sourceTableKeyField))
|
||||
.filter(Objects::nonNull)
|
||||
.filter(v -> !"".equals(v))
|
||||
.filter(v -> !"".equals(String.valueOf(v)))
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if(this.recordLookupHelper == null)
|
||||
@ -267,12 +267,12 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// foreach source record, build the record we'll insert/update //
|
||||
/////////////////////////////////////////////////////////////////
|
||||
QFieldMetaData destinationForeignKeyField = runBackendStepInput.getInstance().getTable(destinationTableName).getField(destinationTableForeignKeyField);
|
||||
QFieldMetaData destinationForeignKeyField = QContext.getQInstance().getTable(destinationTableName).getField(destinationTableForeignKeyField);
|
||||
Set<Serializable> processedSourceKeys = new HashSet<>();
|
||||
for(QRecord sourceRecord : runBackendStepInput.getRecords())
|
||||
{
|
||||
Serializable sourcePrimaryKey = sourceRecord.getValue(QContext.getQInstance().getTable(config.sourceTable).getPrimaryKeyField());
|
||||
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
|
||||
Serializable sourceKeyValue = extractSourceKeyValueFromRecord(sourceRecord, sourceTableKeyField);
|
||||
if(processedSourceKeys.contains(sourceKeyValue))
|
||||
{
|
||||
LOG.info("Skipping duplicated source-key within page", logPair("key", sourceKeyValue));
|
||||
@ -373,6 +373,18 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Given a source record, extract what we'll use as its key from it.
|
||||
**
|
||||
** Normally this is just its sourceTableKeyField value - but - a subclass may
|
||||
** do something more interesting, including, returning a java-record.
|
||||
*******************************************************************************/
|
||||
protected Serializable extractSourceKeyValueFromRecord(QRecord sourceRecord, String sourceTableKeyField)
|
||||
{
|
||||
return sourceRecord.getValue(sourceTableKeyField);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
|
@ -22,6 +22,7 @@
|
||||
package com.kingsrook.qqq.backend.core.actions.values;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.CaseChangeBehavior;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldBehavior;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldFilterBehavior;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
@ -164,7 +166,7 @@ class ValueBehaviorApplierTest extends BaseTest
|
||||
|
||||
QRecord record = new QRecord().withValue("firstName", "Homer").withValue("lastName", "Simpson").withValue("ssn", "0123456789");
|
||||
ValueBehaviorApplier.applyFieldBehaviors(ValueBehaviorApplier.Action.FORMATTING, qInstance, table, List.of(record), null);
|
||||
|
||||
|
||||
assertEquals("HOMER", record.getDisplayValue("firstName"));
|
||||
assertNull(record.getDisplayValue("lastName")); // noop will literally do nothing, not even pass value through.
|
||||
assertEquals("0123456789", record.getValueString("ssn")); // formatting action should not run the too-long truncate behavior
|
||||
@ -314,11 +316,87 @@ class ValueBehaviorApplierTest extends BaseTest
|
||||
assertEquals("square", hasCriteriaAndSubFilterUpdated.getCriteria().get(0).getValues().get(0));
|
||||
assertEquals("circle", hasCriteriaAndSubFilterUpdated.getSubFilters().get(0).getCriteria().get(0).getValues().get(0));
|
||||
|
||||
QQueryFilter hasMultiValueCriteriaToUpdate = new QQueryFilter().withCriteria(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle", "Square"));
|
||||
QQueryFilter hasMultiValueCriteriaToUpdate = new QQueryFilter().withCriteria(new QFilterCriteria("name", QCriteriaOperator.IN, "Triangle", "Square"));
|
||||
QQueryFilter hasMultiValueCriteriaUpdated = ValueBehaviorApplier.applyFieldBehaviorsToFilter(qInstance, table, hasMultiValueCriteriaToUpdate, null);
|
||||
assertNotSame(hasMultiValueCriteriaToUpdate, hasMultiValueCriteriaUpdated);
|
||||
assertEquals(List.of("triangle", "square"), hasMultiValueCriteriaUpdated.getCriteria().get(0).getValues());
|
||||
assertEquals(hasMultiValueCriteriaToUpdate.getSubFilters(), hasMultiValueCriteriaUpdated.getSubFilters());
|
||||
|
||||
QQueryFilter hasMultipleCriteriaOnlyToUpdate = new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Square"))
|
||||
.withCriteria(new QFilterCriteria("id", QCriteriaOperator.IS_NOT_BLANK));
|
||||
|
||||
QQueryFilter hasMultipleCriteriaOnlyOneUpdated = ValueBehaviorApplier.applyFieldBehaviorsToFilter(qInstance, table, hasMultipleCriteriaOnlyToUpdate, null);
|
||||
assertNotSame(hasMultipleCriteriaOnlyToUpdate, hasMultipleCriteriaOnlyOneUpdated);
|
||||
assertEquals(2, hasMultipleCriteriaOnlyOneUpdated.getCriteria().size());
|
||||
assertEquals(List.of("square"), hasMultipleCriteriaOnlyOneUpdated.getCriteria().get(0).getValues());
|
||||
assertEquals(hasMultipleCriteriaOnlyToUpdate.getSubFilters(), hasMultipleCriteriaOnlyOneUpdated.getSubFilters());
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// set 2 behaviors on the field - make sure both happen //
|
||||
//////////////////////////////////////////////////////////
|
||||
field.setBehaviors(Set.of(CaseChangeBehavior.TO_LOWER_CASE, new AppendSomethingBehavior("-x")));
|
||||
QQueryFilter criteriaValueToUpdateTwice = new QQueryFilter().withCriteria(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Triangle"));
|
||||
QQueryFilter criteriaValueUpdatedTwice = ValueBehaviorApplier.applyFieldBehaviorsToFilter(qInstance, table, criteriaValueToUpdateTwice, null);
|
||||
assertNotSame(criteriaValueToUpdateTwice, criteriaValueUpdatedTwice);
|
||||
assertEquals("triangle-x", criteriaValueUpdatedTwice.getCriteria().get(0).getValues().get(0));
|
||||
assertEquals(criteriaValueToUpdateTwice.getSubFilters(), criteriaValueUpdatedTwice.getSubFilters());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
***************************************************************************/
|
||||
public static class AppendSomethingBehavior implements FieldBehavior<AppendSomethingBehavior>, FieldFilterBehavior<AppendSomethingBehavior>
|
||||
{
|
||||
private String something;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public AppendSomethingBehavior(String something)
|
||||
{
|
||||
this.something = something;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public Serializable applyToFilterCriteriaValue(Serializable value, QInstance instance, QTableMetaData table, QFieldMetaData field)
|
||||
{
|
||||
return value + something;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public AppendSomethingBehavior getDefault()
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
*
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void apply(ValueBehaviorApplier.Action action, List<QRecord> recordList, QInstance instance, QTableMetaData table, QFieldMetaData field)
|
||||
{
|
||||
//////////
|
||||
// noop //
|
||||
//////////
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2024. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.actions.tables.query;
|
||||
|
||||
|
||||
import java.util.Collections;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for QueryOutputList
|
||||
*******************************************************************************/
|
||||
class QueryOutputListTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testLogSize() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON);
|
||||
QueryOutput queryOutput = new QueryOutput(queryInput);
|
||||
|
||||
QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(QueryOutputList.class);
|
||||
|
||||
///////////////////////
|
||||
// set up our limits //
|
||||
///////////////////////
|
||||
int infoLimit = 10;
|
||||
int warnLimit = 20;
|
||||
int errorLimit = 30;
|
||||
|
||||
QueryOutputList.setLogSizeInfoOver(infoLimit);
|
||||
QueryOutputList.setLogSizeWarnOver(warnLimit);
|
||||
QueryOutputList.setLogSizeErrorOver(errorLimit);
|
||||
|
||||
////////////////////////////
|
||||
// add records one-by-one //
|
||||
////////////////////////////
|
||||
for(int i = 0; i < errorLimit; i++)
|
||||
{
|
||||
queryOutput.addRecord(new QRecord());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////
|
||||
// assert we got the expected logs as each level was crossed //
|
||||
///////////////////////////////////////////////////////////////
|
||||
assertEquals(3, collectingLogger.getCollectedMessages().size());
|
||||
|
||||
assertEquals(Level.INFO, collectingLogger.getCollectedMessages().get(0).getLevel());
|
||||
assertThat(collectingLogger.getCollectedMessages().get(0).getMessage())
|
||||
.contains("\"noRecords\":" + infoLimit)
|
||||
.contains("\"tableName\":\"" + TestUtils.TABLE_NAME_PERSON + "\"");
|
||||
|
||||
assertEquals(Level.WARN, collectingLogger.getCollectedMessages().get(1).getLevel());
|
||||
assertThat(collectingLogger.getCollectedMessages().get(1).getMessage())
|
||||
.contains("\"noRecords\":" + warnLimit)
|
||||
.contains("\"tableName\":\"" + TestUtils.TABLE_NAME_PERSON + "\"");
|
||||
|
||||
assertEquals(Level.ERROR, collectingLogger.getCollectedMessages().get(2).getLevel());
|
||||
assertThat(collectingLogger.getCollectedMessages().get(2).getMessage())
|
||||
.contains("\"noRecords\":" + errorLimit)
|
||||
.contains("\"tableName\":\"" + TestUtils.TABLE_NAME_PERSON + "\"");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// reset the logger - then run again, doing a bulk add that goes straight to error size //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
collectingLogger.clear();
|
||||
queryOutput = new QueryOutput(queryInput);
|
||||
int bulkSize = errorLimit + 1;
|
||||
queryOutput.addRecords(Collections.nCopies(bulkSize, new QRecord()));
|
||||
|
||||
assertEquals(1, collectingLogger.getCollectedMessages().size());
|
||||
assertEquals(Level.ERROR, collectingLogger.getCollectedMessages().get(0).getLevel());
|
||||
assertThat(collectingLogger.getCollectedMessages().get(0).getMessage())
|
||||
.contains("\"noRecords\":" + bulkSize)
|
||||
.contains("\"tableName\":\"" + TestUtils.TABLE_NAME_PERSON + "\"");
|
||||
|
||||
}
|
||||
|
||||
}
|
@ -55,6 +55,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.Pair;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
@ -356,18 +357,29 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
|
||||
*******************************************************************************/
|
||||
private PreparedStatement createStatement(Connection connection, String sql, QueryInput queryInput) throws SQLException
|
||||
{
|
||||
if(connection.getClass().getName().startsWith("com.mysql"))
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// if we're allowed to use the mysqlResultSetOptimization, and we have //
|
||||
// the query hint of "potentially large no of results", then check if //
|
||||
// our backend is indeed mysql, and if so, then apply those settings. //
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
if(mysqlResultSetOptimizationEnabled && queryInput.hasQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS))
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if we're allowed to use the mysqlResultSetOptimization, and we have the query hint of "expected large result set", then do it. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(mysqlResultSetOptimizationEnabled && queryInput.getQueryHints() != null && queryInput.getQueryHints().contains(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS))
|
||||
RDBMSBackendMetaData rdbmsBackendMetaData = (RDBMSBackendMetaData) queryInput.getBackend();
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// todo - remove "aurora" - it's a legacy value here for a staged rollout //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || "aurora".equals(rdbmsBackendMetaData.getVendor()))
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// mysql "optimization", presumably here - from Result Set section of https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html //
|
||||
// without this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a large query (e.g., > 1,000,000 rows). //
|
||||
// with this change, we start to get results immediately, and the total runtime also seems lower... //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// mysql "optimization", presumably here - from Result Set section of //
|
||||
// https://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html without //
|
||||
// this change, we saw ~10 seconds of "wait" time, before results would start to stream out of a //
|
||||
// large query (e.g., > 1,000,000 rows). //
|
||||
// with this change, we start to get results immediately, and the total runtime also seems lower... //
|
||||
// perhaps more importantly, without this change, the whole result set goes into memory - but with //
|
||||
// this change, it is streamed. //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
PreparedStatement statement = connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
|
||||
statement.setFetchSize(Integer.MIN_VALUE);
|
||||
return (statement);
|
||||
|
@ -150,8 +150,11 @@ public class ConnectionManager
|
||||
|
||||
return switch(backend.getVendor())
|
||||
{
|
||||
case "mysql", "aurora" -> "com.mysql.cj.jdbc.Driver";
|
||||
case "h2" -> "org.h2.Driver";
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// todo - remove "aurora" - it's a legacy value here for a staged rollout //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "com.mysql.cj.jdbc.Driver";
|
||||
case RDBMSBackendMetaData.VENDOR_H2 -> "org.h2.Driver";
|
||||
default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data."));
|
||||
};
|
||||
}
|
||||
@ -170,11 +173,17 @@ public class ConnectionManager
|
||||
|
||||
return switch(backend.getVendor())
|
||||
{
|
||||
// TODO aws-mysql-jdbc driver not working when running on AWS
|
||||
////////////////////////////////////////////////////////////////
|
||||
// TODO aws-mysql-jdbc driver not working when running on AWS //
|
||||
////////////////////////////////////////////////////////////////
|
||||
// jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL";
|
||||
case "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
|
||||
case "mysql" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
|
||||
case "h2" -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
// todo - remove "aurora" - it's a legacy value here for a staged rollout //
|
||||
////////////////////////////////////////////////////////////////////////////
|
||||
case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
|
||||
case RDBMSBackendMetaData.VENDOR_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
|
||||
case RDBMSBackendMetaData.VENDOR_H2 -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
|
||||
default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor());
|
||||
};
|
||||
}
|
||||
|
@ -49,6 +49,13 @@ public class RDBMSBackendMetaData extends QBackendMetaData
|
||||
|
||||
private RDBMSBackendMetaData readOnlyBackendMetaData;
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// define well-known (and fully supported) vendor values //
|
||||
///////////////////////////////////////////////////////////
|
||||
public static final String VENDOR_MYSQL = "mysql";
|
||||
public static final String VENDOR_H2 = "h2";
|
||||
public static final String VENDOR_AURORA_MYSQL = "aurora-mysql";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
|
@ -70,6 +70,7 @@ class QPicoCliImplementationTest
|
||||
@BeforeEach
|
||||
public void beforeEach() throws Exception
|
||||
{
|
||||
System.setProperty("picocli.ansi", "false");
|
||||
TestUtils.primeTestDatabase();
|
||||
QContext.init(TestUtils.defineInstance(), new QSession());
|
||||
}
|
||||
|
@ -68,7 +68,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-material-dashboard</artifactId>
|
||||
<version>0.20.0-20240418.180316-42</version>
|
||||
<version>0.20.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.h2database</groupId>
|
||||
|
Reference in New Issue
Block a user