From d2fd0d13b5752eade916f88e0d463e840faed0d2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 15 Nov 2023 08:39:21 -0600 Subject: [PATCH 1/4] Add method getProcessMetaData --- .../columnstats/ColumnStatsStep.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index 53816fae..bfeb47f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -57,9 +57,14 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -80,6 +85,21 @@ public class ColumnStatsStep implements BackendStep + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("columnStats") + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)) + .withStepList(List.of(new QBackendStepMetaData() + .withName("step") + .withCode(new QCodeReference(ColumnStatsStep.class))))); + } + + + /******************************************************************************* ** *******************************************************************************/ From 1b58cdeb3cfc259d3a47dc03cc51acef3a43c4f7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 15 Nov 2023 08:49:04 -0600 Subject: [PATCH 2/4] Move packagesToKeep to a system-property/env-var --- .../qqq/backend/core/logging/LogUtils.java | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogUtils.java index 6fe84825..57c0331f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/LogUtils.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; @@ -34,6 +35,17 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; *******************************************************************************/ public class LogUtils { + /////////////////////////////////////////////////////////////////////////////////////////////// + // This string will be used in regex, inside ()'s, so you can supply pipe-delimited packages // + // as in, com.kingsrook|com.yourdomain|org.some.other.package // + /////////////////////////////////////////////////////////////////////////////////////////////// + private static String packagesToKeep = "."; + + static + { + packagesToKeep = new QMetaDataVariableInterpreter().getStringFromPropertyOrEnvironment("qqq.logger.packagesToKeep", "QQQ_LOGGER_PACKAGES_TO_KEEP", "."); + } + /******************************************************************************* ** @@ -118,9 +130,8 @@ public class LogUtils { try { - String packagesToKeep = "com.kingsrook|com.coldtrack"; // todo - parameterize!! - StringBuilder rs = new StringBuilder(); - String[] lines = stackTrace.split("\n"); + StringBuilder rs = new StringBuilder(); + String[] lines = stackTrace.split("\n"); int indexWithinSubStack = 0; int skipsInThisPackage = 0; @@ -134,7 +145,13 @@ public class LogUtils { keepLine = false; indexWithinSubStack++; - if(line.matches("^\\s+at (" + packagesToKeep + ").*")) + + ///////////////////////////////////////////////////////////////////////////// + // avoid NPE on packages to keep (and keep all packages) // + // also, avoid the regex call if it's the default of "." (e.g., match all) // + // otherwise, check if the line matches "at (packagesToKeep).*" // + ///////////////////////////////////////////////////////////////////////////// + if(packagesToKeep == null || ".".equals(packagesToKeep) || line.matches("^\\s+at (" + packagesToKeep + ").*")) { keepLine = true; } From 6aa4867bba0ede596967c40e70a6f1299783eebf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 15 Nov 2023 08:52:32 -0600 Subject: [PATCH 3/4] Add getStringFromPropertyOrEnvironment --- .../QMetaDataVariableInterpreter.java | 26 +++++++++++++ .../QMetaDataVariableInterpreterTest.java | 37 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java index 339dea31..f4df44a1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java @@ -270,6 +270,32 @@ public class QMetaDataVariableInterpreter + /******************************************************************************* + ** First look for a string in the specified system property - + ** Next look for a string in the specified env var name - + ** Finally return the default. + *******************************************************************************/ + public String getStringFromPropertyOrEnvironment(String systemPropertyName, String environmentVariableName, String defaultIfNotSet) + { + String propertyValue = System.getProperty(systemPropertyName); + if(StringUtils.hasContent(propertyValue)) + { + LOG.info("Read system property [" + systemPropertyName + "] as [" + propertyValue + "]."); + return (propertyValue); + } + + String envValue = interpret("${env." + environmentVariableName + "}"); + if(StringUtils.hasContent(envValue)) + { + LOG.info("Read env var [" + environmentVariableName + "] as [" + envValue + "]."); + return (envValue); + } + + return defaultIfNotSet; + } + + + /******************************************************************************* ** First look for a boolean ("true" or "false") in the specified system property - ** Next look for a boolean in the specified env var name - diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java index 3e3ea57b..474fd084 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java @@ -226,6 +226,43 @@ class QMetaDataVariableInterpreterTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetStringFromPropertyOrEnvironment() + { + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + + ////////////////////////////////////////////////////////// + // if neither prop nor env is set, get back the default // + ////////////////////////////////////////////////////////// + assertEquals("default", interpreter.getStringFromPropertyOrEnvironment("notSet", "NOT_SET", "default")); + + ///////////////////////////////// + // if only prop is set, get it // + ///////////////////////////////// + assertEquals("default", interpreter.getStringFromPropertyOrEnvironment("foo.value", "FOO_VALUE", "default")); + System.setProperty("foo.value", "fooPropertyValue"); + assertEquals("fooPropertyValue", interpreter.getStringFromPropertyOrEnvironment("foo.value", "FOO_VALUE", "default")); + + //////////////////////////////// + // if only env is set, get it // + //////////////////////////////// + assertEquals("default", interpreter.getStringFromPropertyOrEnvironment("bar.value", "BAR_VALUE", "default")); + interpreter.setEnvironmentOverrides(Map.of("BAR_VALUE", "barEnvValue")); + assertEquals("barEnvValue", interpreter.getStringFromPropertyOrEnvironment("bar.value", "BAR_VALUE", "default")); + + /////////////////////////////////// + // if both are set, get the prop // + /////////////////////////////////// + System.setProperty("baz.value", "bazPropertyValue"); + interpreter.setEnvironmentOverrides(Map.of("BAZ_VALUE", "bazEnvValue")); + assertEquals("bazPropertyValue", interpreter.getStringFromPropertyOrEnvironment("baz.value", "BAZ_VALUE", "default")); + } + + + /******************************************************************************* ** *******************************************************************************/ From 74464809c4c629def1dab649c1d33a71a574d547 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 15 Nov 2023 08:53:25 -0600 Subject: [PATCH 4/4] Fix method updateRecordsWithDisplayValuesAndPossibleValues - was acting like it used DESTINATION_TABLE, but then changing to input.getTable --- .../etl/streamedwithfrontend/BaseStreamedETLStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java index 27dab396..74cab0b6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/BaseStreamedETLStep.java @@ -93,7 +93,7 @@ public class BaseStreamedETLStep qValueFormatter.setDisplayValuesInRecords(table, list); QPossibleValueTranslator qPossibleValueTranslator = new QPossibleValueTranslator(input.getInstance(), input.getSession()); - qPossibleValueTranslator.translatePossibleValuesInRecords(input.getTable(), list); + qPossibleValueTranslator.translatePossibleValuesInRecords(table, list); } }