From c75d19d72a5fcaa6b104a1adf3d063a618c48e18 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 May 2023 15:27:40 -0500 Subject: [PATCH 1/4] Add possible value 422... would be nice to get these all from a lib... --- .../module/api/model/OutboundAPILogMetaDataProvider.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java index ce1d5007..060fb1e7 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/model/OutboundAPILogMetaDataProvider.java @@ -84,6 +84,7 @@ public class OutboundAPILogMetaDataProvider new QPossibleValue<>(401, "401 (Not Authorized)"), new QPossibleValue<>(403, "403 (Forbidden)"), new QPossibleValue<>(404, "404 (Not Found)"), + new QPossibleValue<>(422, "422 (Unprocessable Entity)"), new QPossibleValue<>(429, "429 (Too Many Requests)"), new QPossibleValue<>(500, "500 (Internal Server Error)") ))); @@ -131,6 +132,7 @@ public class OutboundAPILogMetaDataProvider .withValue(AdornmentType.ChipValues.colorValue(401, AdornmentType.ChipValues.COLOR_ERROR)) .withValue(AdornmentType.ChipValues.colorValue(403, AdornmentType.ChipValues.COLOR_ERROR)) .withValue(AdornmentType.ChipValues.colorValue(404, AdornmentType.ChipValues.COLOR_ERROR)) + .withValue(AdornmentType.ChipValues.colorValue(422, AdornmentType.ChipValues.COLOR_ERROR)) .withValue(AdornmentType.ChipValues.colorValue(429, AdornmentType.ChipValues.COLOR_ERROR)) .withValue(AdornmentType.ChipValues.colorValue(500, AdornmentType.ChipValues.COLOR_ERROR))); From cedc1edfac46e4f91c21245c4a5d10a7105e8853 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 8 May 2023 15:27:57 -0500 Subject: [PATCH 2/4] Add queryJoins to access log if-slow --- .../qqq/backend/javalin/QJavalinImplementation.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 30a43f0b..be89714c 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -882,12 +882,13 @@ public class QJavalinImplementation queryInput.getFilter().setLimit(limit); } - queryInput.setQueryJoins(processQueryJoinsParam(context)); + List queryJoins = processQueryJoinsParam(context); + queryInput.setQueryJoins(queryJoins); QueryAction queryAction = new QueryAction(); QueryOutput queryOutput = queryAction.execute(queryInput); - QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS)); + QJavalinAccessLogger.logEndSuccess(logPair("recordCount", queryOutput.getRecords().size()), logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS), logPairIfSlow("joins", queryJoins, SLOW_LOG_THRESHOLD_MS)); context.result(JsonUtils.toJson(queryOutput)); } catch(Exception e) From 815bd8b0ce6fe4fb42e9639922775a0b433aa550 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 9 May 2023 10:43:23 -0500 Subject: [PATCH 3/4] Updates to work with branch-specific maven deployments in/with circleci --- .circleci/adjust-pom-version.sh | 23 +++++++++++++ .circleci/config.yml | 4 +++ qqq-dev-tools/bin/get-latest-snapshot.sh | 43 +++++++++++++++++++++--- qqq-dev-tools/bin/git-tag | 11 ++++++ qqq-dev-tools/bin/update-all-qqq-deps.sh | 2 +- 5 files changed, 78 insertions(+), 5 deletions(-) create mode 100755 .circleci/adjust-pom-version.sh diff --git a/.circleci/adjust-pom-version.sh b/.circleci/adjust-pom-version.sh new file mode 100755 index 00000000..054fbdf2 --- /dev/null +++ b/.circleci/adjust-pom-version.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +if [ -z "$CIRCLE_BRANCH" ] && [ -z "$CIRCLE_TAG" ]; then + echo "Error: env vars CIRCLE_BRANCH and CIRCLE_TAG were not set." + exit 1; +fi + +if [ "$CIRCLE_BRANCH" == "dev" ] || [ "$CIRCLE_BRANCH" == "staging" ] || [ "$CIRCLE_BRANCH" == "main" ]; then + echo "On a primary branch [$CIRCLE_BRANCH] - will not edit the pom version."; + exit 0; +fi + +if [ -n "$CIRCLE_BRANCH" ]; then + SLUG=$(echo $CIRCLE_BRANCH | sed 's/[^a-zA-Z0-9]/-/g') +else + SLUG=$(echo $CIRCLE_TAG | sed 's/^snapshot-//g') +fi + +POM=$(dirname $0)/../pom.xml + +echo "Updating $POM to: $SLUG-SNAPSHOT" +sed -i "s/.*/$SLUG-SNAPSHOT<\/revision>/" $POM +git diff $POM diff --git a/.circleci/config.yml b/.circleci/config.yml index 5981d2fa..42bdf306 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,6 +82,10 @@ commands: mvn_jar_deploy: steps: - checkout + - run: + name: Adjust pom version + command: | + .circleci/adjust-pom-version.sh - restore_cache: keys: - v1-dependencies-{{ checksum "pom.xml" }} diff --git a/qqq-dev-tools/bin/get-latest-snapshot.sh b/qqq-dev-tools/bin/get-latest-snapshot.sh index 11025009..232842d5 100755 --- a/qqq-dev-tools/bin/get-latest-snapshot.sh +++ b/qqq-dev-tools/bin/get-latest-snapshot.sh @@ -9,6 +9,34 @@ CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)" MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST . $QQQ_DEV_TOOLS_DIR/.env +BRANCH=$(git rev-parse --abbrev-ref HEAD) +SLUG=$(echo $BRANCH | sed 's/[^a-zA-Z0-9]/-/g') + +function checkForBranchBuild +{ + artifact=$1 + + ############################################################# + ## on standard branches, don't look for branch deployments ## + ############################################################# + if [ "$BRANCH" == "dev" ] || [ "$BRANCH" == "staging" ] || [ "$BRANCH" == "main" ]; then + echo 0; + return; + fi + + ################################################################### + ## else, do look for a branch deployment, and return accordingly ## + ################################################################### + curl -s --user ${GITHUB_USER}:${GITHUB_TOKEN} https://maven.pkg.github.com/Kingsrook/qqq-maven-registry/com/kingsrook/qqq/${artifact}/${SLUG}-SNAPSHOT/maven-metadata.xml | grep unable.to.fetch + if [ "$?" == "1" ]; then + echo 1; + return; + fi + + echo 0; + return; +} + function getLatestVersion { artifact=$1 @@ -23,10 +51,10 @@ function getLatestVersion return fi - timetsamp=$(xpath -q -e '/metadata/versioning/snapshot/timestamp/text()' /tmp/metadata.xml) + timestamp=$(xpath -q -e '/metadata/versioning/snapshot/timestamp/text()' /tmp/metadata.xml) buildNumber=$(xpath -q -e '/metadata/versioning/snapshot/buildNumber/text()' /tmp/metadata.xml) - echo "$version-$timetsamp-$buildNumber" + echo "$version-$timestamp-$buildNumber" } function promptForVersion @@ -70,11 +98,18 @@ else artifact=$1 version=$2 if [ "$version" == "-l" ]; then - version=$CURRENT_VERSION + useSlug=$(checkForBranchBuild $artifact) + if [ "$useSlug" == "1" ]; then + version=$SLUG + else + version=$CURRENT_VERSION + fi + echo "Using $version for $artifact" >&2 fi - if [ -z "$artifact" -o -z "$version" ]; then + if [ -z "$ar^tifact" -o -z "$version" ]; then echo "Usage: $0 artifact snapshot-version-prefix" + echo " or: $0 artifact -l (latest of CURRENT_VERSION, or branch-slug, if it has been deployed)" echo " or: $0 -i (interactive mode)" echo " or: $0 -a [snapshot-version-prefix] (all mode)" echo "Ex: $0 qqq-backend-core $CURRENT_VERSION" diff --git a/qqq-dev-tools/bin/git-tag b/qqq-dev-tools/bin/git-tag index 923f9f63..ddb02dd8 100755 --- a/qqq-dev-tools/bin/git-tag +++ b/qqq-dev-tools/bin/git-tag @@ -3,6 +3,11 @@ ############################################################################ ## git-tag ## Add (deleting, if it already exists) a git tag to a repo (both local & remote) +## +## Usage: +## git-tag tag-name - make/update/push the specified tag-name. +## git-tag -l - list tags. +## git-tag snapshot-BRANCH_SLUG - make the tag name "snapshot-" + current branch name, in "slug" form ############################################################################ TAG=$1 @@ -16,6 +21,12 @@ if [ "$TAG" == "-l" ]; then exit 0; fi +if [ "$TAG" == "snapshot-BRANCH_SLUG" ]; then + BRANCH=$(git rev-parse --abbrev-ref HEAD) + SLUG=$(echo $BRANCH | sed 's/[^a-zA-Z0-9]/-/g') + TAG="snapshot-$SLUG" +fi + echo echo "== Deleting $TAG on local" git tag -d $TAG diff --git a/qqq-dev-tools/bin/update-all-qqq-deps.sh b/qqq-dev-tools/bin/update-all-qqq-deps.sh index baa73113..846eefeb 100755 --- a/qqq-dev-tools/bin/update-all-qqq-deps.sh +++ b/qqq-dev-tools/bin/update-all-qqq-deps.sh @@ -15,7 +15,7 @@ MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST for module in $(cat $MODULE_LIST_FILE); do echo "Updating $module..." - version=$(get-latest-snapshot.sh $module $CURRENT_VERSION) + version=$(get-latest-snapshot.sh $module -l) update-dep.sh $module $version -q done From 14fc7b0ba8b10b2c05ae37d120d4633fbdccae71 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 12 May 2023 14:50:00 -0500 Subject: [PATCH 4/4] Add criteria operator NOT_EQUALS_OR_IS_NULL --- .../tables/query/QCriteriaOperator.java | 1 + .../utils/BackendQueryFilterUtils.java | 1 + .../utils/BackendQueryFilterUtilsTest.java | 7 ++++ .../rdbms/actions/AbstractRDBMSAction.java | 6 ++++ .../rdbms/actions/RDBMSQueryActionTest.java | 33 +++++++++++++++++++ 5 files changed, 48 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java index 35352192..6dc50b1d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QCriteriaOperator.java @@ -30,6 +30,7 @@ public enum QCriteriaOperator { EQUALS, NOT_EQUALS, + NOT_EQUALS_OR_IS_NULL, IN, NOT_IN, IS_NULL_OR_IN, diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java index e05d0bbd..06d36f64 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtils.java @@ -131,6 +131,7 @@ public class BackendQueryFilterUtils { case EQUALS -> testEquals(criterion, value); case NOT_EQUALS -> !testEquals(criterion, value); + case NOT_EQUALS_OR_IS_NULL -> !testEquals(criterion, value) || testBlank(criterion, value); case IN -> testIn(criterion, value); case NOT_IN -> !testIn(criterion, value); case IS_BLANK -> testBlank(criterion, value); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java index 75c8578c..677fb8d8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/utils/BackendQueryFilterUtilsTest.java @@ -113,6 +113,13 @@ class BackendQueryFilterUtilsTest assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(new QFilterCriteria("f", QCriteriaOperator.NOT_IN, "A", "B"), "f", "B")); assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(new QFilterCriteria("f", QCriteriaOperator.NOT_IN, List.of()), "f", "A")); assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(new QFilterCriteria("f", QCriteriaOperator.NOT_IN, ListBuilder.of(null)), "f", "A")); + + /////////////////////////// + // NOT_EQUALS_OR_IS_NULL // + /////////////////////////// + assertFalse(BackendQueryFilterUtils.doesCriteriaMatch(new QFilterCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", "A")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(new QFilterCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", "B")); + assertTrue(BackendQueryFilterUtils.doesCriteriaMatch(new QFilterCriteria("f", QCriteriaOperator.NOT_EQUALS_OR_IS_NULL, "A"), "f", null)); } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index ec7e7488..6145e8ef 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -534,6 +534,12 @@ public abstract class AbstractRDBMSAction implements QActionInterface expectedNoOfParams = 1; break; } + case NOT_EQUALS_OR_IS_NULL: + { + clause += " != ? OR " + column + " IS NULL "; + expectedNoOfParams = 1; + break; + } case IN: { if(values.isEmpty()) diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 637601fa..692ea261 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -148,6 +148,39 @@ public class RDBMSQueryActionTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsOrIsNullQuery() throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // 5 rows, 1 has a null salary, 1 has 1,000,000. // + // first confirm that query for != returns 3 (the null does NOT come back) // + // then, confirm that != or is null gives the (more humanly expected) 4. // + ///////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(1_000_000)))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + + queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS_OR_IS_NULL) + .withValues(List.of(1_000_000)))); + queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + Assertions.assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary"); + } + + + /******************************************************************************* ** *******************************************************************************/