From 00a5b72bf32cb7aff9db97fdeb23dc6e223c911e Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 16 Jan 2024 14:32:40 -0600 Subject: [PATCH 1/9] added string util method for appending strings --- .../qqq/backend/core/utils/StringUtils.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java index 5f323ddf..a8348756 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java @@ -176,6 +176,19 @@ public class StringUtils + /******************************************************************************* + ** safely appends a string to another, changing empty string if either value is null + ** + *******************************************************************************/ + public static String safeAppend(String input, String contentToAppend) + { + input = input != null ? input : ""; + contentToAppend = contentToAppend != null ? contentToAppend : ""; + return input + contentToAppend; + } + + + /******************************************************************************* ** returns input if not null, or nullOutput if input == null (as in SQL NVL) ** From 74d66d0fa540eb967551157c7a2ee55a205ad859 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 31 Jan 2024 11:14:50 -0600 Subject: [PATCH 2/9] Add Log for missing value in sourceKeyField --- .../tablesync/AbstractTableSyncTransformStep.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index ed754f03..090cdb8f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -276,6 +276,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt if(sourceKeyValue == null || "".equals(sourceKeyValue)) { + LOG.debug("Skipping record without a value in the sourceKeyField", logPair("keyField", sourceTableKeyField)); errorMissingKeyField.incrementCountAndAddPrimaryKey(sourcePrimaryKey); try From 612370fc133fbd738e222f54aeb2ac41a0e054c6 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 31 Jan 2024 16:16:14 -0600 Subject: [PATCH 3/9] Fix the check for primary key of integer (to work for null primary keys, and to be inside the try-catch); tests on that --- .../core/actions/audits/DMLAuditAction.java | 29 ++++++------ .../actions/audits/DMLAuditActionTest.java | 45 +++++++++++++++++++ 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index e7ec6a73..d5deff86 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -91,20 +91,6 @@ public class DMLAuditAction extends AbstractQActionFunction recordList = CollectionUtils.nonNullList(input.getRecordList()).stream() @@ -119,6 +105,21 @@ public class DMLAuditAction extends AbstractQActionFunction auditList = TestUtils.queryTable("audit"); + assertTrue(auditList.isEmpty()); + } + } From 2681d66b3234b6b88bc57884cd56b55f3901d351 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 1 Feb 2024 11:13:30 -0600 Subject: [PATCH 4/9] CE-779: updated all addHeader calls to setHeader to avoid duplicate entries --- .../module/api/actions/BaseAPIActionUtil.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index a14b23b1..84e79e24 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -715,11 +715,11 @@ public class BaseAPIActionUtil if(backendMetaData.getAuthorizationType().equals(AuthorizationType.BASIC_AUTH_USERNAME_PASSWORD)) { - request.addHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField()))); + request.setHeader("Authorization", getBasicAuthenticationHeader(record.getValueString(backendMetaData.getVariantOptionsTableUsernameField()), record.getValueString(backendMetaData.getVariantOptionsTablePasswordField()))); } else if(backendMetaData.getAuthorizationType().equals(AuthorizationType.API_KEY_HEADER)) { - request.addHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField())); + request.setHeader("API-Key", record.getValueString(backendMetaData.getVariantOptionsTableApiKeyField())); } else { @@ -733,10 +733,10 @@ public class BaseAPIActionUtil /////////////////////////////////////////////////////////////////////////////////////////// switch(backendMetaData.getAuthorizationType()) { - case BASIC_AUTH_API_KEY -> request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey())); - case BASIC_AUTH_USERNAME_PASSWORD -> request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword())); - case API_KEY_HEADER -> request.addHeader("API-Key", backendMetaData.getApiKey()); - case API_TOKEN -> request.addHeader("Authorization", "Token " + backendMetaData.getApiKey()); + case BASIC_AUTH_API_KEY -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getApiKey())); + case BASIC_AUTH_USERNAME_PASSWORD -> request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getUsername(), backendMetaData.getPassword())); + case API_KEY_HEADER -> request.setHeader("API-Key", backendMetaData.getApiKey()); + case API_TOKEN -> request.setHeader("Authorization", "Token " + backendMetaData.getApiKey()); case OAUTH2 -> request.setHeader("Authorization", "Bearer " + getOAuth2Token()); case API_KEY_QUERY_PARAM -> { @@ -792,9 +792,9 @@ public class BaseAPIActionUtil if(setCredentialsInHeader) { - request.addHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret())); + request.setHeader("Authorization", getBasicAuthenticationHeader(backendMetaData.getClientId(), backendMetaData.getClientSecret())); } - request.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8"); + request.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=utf-8"); HttpResponse response = executeOAuthTokenRequest(client, request); int statusCode = response.getStatusLine().getStatusCode(); @@ -856,7 +856,7 @@ public class BaseAPIActionUtil *******************************************************************************/ protected void setupContentTypeInRequest(HttpRequestBase request) { - request.addHeader("Content-Type", backendMetaData.getContentType()); + request.setHeader("Content-Type", backendMetaData.getContentType()); } @@ -878,7 +878,7 @@ public class BaseAPIActionUtil *******************************************************************************/ public void setupAdditionalHeaders(HttpRequestBase request) { - request.addHeader("Accept", "application/json"); + request.setHeader("Accept", "application/json"); } From c6a58ac68f01421aca8e52022f27675320889e76 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 1 Feb 2024 16:00:45 -0600 Subject: [PATCH 5/9] added tests to StringUtils.safeAppend() --- .../qqq/backend/core/utils/StringUtilsTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java index 26999d62..b2cad605 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java @@ -78,6 +78,20 @@ class StringUtilsTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test_safeAppend() + { + assertEquals("Foo", StringUtils.safeAppend("Foo", null)); + assertEquals("Foo", StringUtils.safeAppend(null, "Foo")); + assertEquals("FooBar", StringUtils.safeAppend("Foo", "Bar")); + assertEquals("", StringUtils.safeAppend(null, null)); + } + + + /******************************************************************************* ** *******************************************************************************/ From 8e8d3b5d2bf0c4161d4c57c18cbd250614de77c6 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 6 Feb 2024 14:55:38 -0600 Subject: [PATCH 6/9] downgraded some loggly infos to debugs to stop filling up --- .../java/com/kingsrook/qqq/backend/core/model/data/QRecord.java | 2 +- .../implementations/Auth0AuthenticationModule.java | 2 +- .../qqq/backend/module/api/actions/BaseAPIActionUtil.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 32ab9000..ce37a9a2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -186,7 +186,7 @@ public class QRecord implements Serializable ////////////////////////////////////////////////////////////////////////////// // we know entry is serializable at this point, based on type param's bound // ////////////////////////////////////////////////////////////////////////////// - LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass())); + LOG.debug("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass())); clone.put(entry.getKey(), (V) SerializationUtils.clone(entry.getValue())); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java index 70fe607a..82cf2ed8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/authentication/implementations/Auth0AuthenticationModule.java @@ -176,7 +176,7 @@ public class Auth0AuthenticationModule implements QAuthenticationModuleInterface // process a sessionUUID - looks up userSession record - cannot create token this way. // ///////////////////////////////////////////////////////////////////////////////////////// String sessionUUID = context.get(SESSION_UUID_KEY); - LOG.info("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID))); + LOG.debug("Creating session from sessionUUID (userSession)", logPair("sessionUUID", maskForLog(sessionUUID))); if(sessionUUID != null) { accessToken = getAccessTokenFromSessionUUID(metaData, sessionUUID); diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 5347b9d0..98393b89 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -1081,7 +1081,7 @@ public class BaseAPIActionUtil ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // trim response body (just to keep logs smaller, or, in case someone consuming logs doesn't want such long lines) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - LOG.info("Received successful response with code [" + qResponse.getStatusCode() + "] and content [" + StringUtils.safeTruncate(qResponse.getContent(), getMaxResponseMessageLengthForLog(), "...") + "]."); + LOG.debug("Received successful response with code [" + qResponse.getStatusCode() + "] and content [" + StringUtils.safeTruncate(qResponse.getContent(), getMaxResponseMessageLengthForLog(), "...") + "]."); return (qResponse); } } From c0b5d11a090959441d111d2f5dff27ea93b5dbbb Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 7 Feb 2024 09:18:57 -0600 Subject: [PATCH 7/9] added getAPIResponseLogLevel as base method that can be overridden in subsclasses --- .../module/api/actions/BaseAPIActionUtil.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 98393b89..5be4635d 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -96,6 +96,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.Level; import org.json.JSONArray; import org.json.JSONObject; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -1081,7 +1082,7 @@ public class BaseAPIActionUtil ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // trim response body (just to keep logs smaller, or, in case someone consuming logs doesn't want such long lines) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - LOG.debug("Received successful response with code [" + qResponse.getStatusCode() + "] and content [" + StringUtils.safeTruncate(qResponse.getContent(), getMaxResponseMessageLengthForLog(), "...") + "]."); + LOG.log(getAPIResponseLogLevel(), "Received successful response with code [" + qResponse.getStatusCode() + "] and content [" + StringUtils.safeTruncate(qResponse.getContent(), getMaxResponseMessageLengthForLog(), "...") + "]."); return (qResponse); } } @@ -1507,4 +1508,14 @@ public class BaseAPIActionUtil // nothing to do at this layer, meant to be overridden by subclasses // /////////////////////////////////////////////////////////////////////// } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected Level getAPIResponseLogLevel() throws QException + { + return (Level.DEBUG); + } } From aef42a4a5e5aaab6324497df33c2f14a0a607d45 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 8 Feb 2024 13:57:17 -0600 Subject: [PATCH 8/9] Updated 1Password vault --- qqq-dev-tools/bin/setup-environments.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-dev-tools/bin/setup-environments.sh b/qqq-dev-tools/bin/setup-environments.sh index 617a06a1..6594e3ce 100755 --- a/qqq-dev-tools/bin/setup-environments.sh +++ b/qqq-dev-tools/bin/setup-environments.sh @@ -52,7 +52,7 @@ fi ## locations of env files in 1password ## ######################################### QQQ_OP_LOCATION="op://Development Environments/" -CTL_OP_LOCATION="op://NF-One-Development/" +CTL_OP_LOCATION="op://Engineering - CTL-Development/" ################################################## From c77e37d6dc17ee18768753eb3154cdfc860f38c4 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 8 Feb 2024 16:12:59 -0600 Subject: [PATCH 9/9] Revert "Updated 1Password vault" This reverts commit aef42a4a5e5aaab6324497df33c2f14a0a607d45. --- qqq-dev-tools/bin/setup-environments.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-dev-tools/bin/setup-environments.sh b/qqq-dev-tools/bin/setup-environments.sh index 6594e3ce..617a06a1 100755 --- a/qqq-dev-tools/bin/setup-environments.sh +++ b/qqq-dev-tools/bin/setup-environments.sh @@ -52,7 +52,7 @@ fi ## locations of env files in 1password ## ######################################### QQQ_OP_LOCATION="op://Development Environments/" -CTL_OP_LOCATION="op://Engineering - CTL-Development/" +CTL_OP_LOCATION="op://NF-One-Development/" ##################################################