diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java index 2ce856b7..4a1e3d37 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java @@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu ** Interface for the Aggregate action. ** *******************************************************************************/ -public interface AggregateInterface +public interface AggregateInterface extends BaseQueryInterface { /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java new file mode 100644 index 00000000..f02beada --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.interfaces; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; + + +/******************************************************************************* + ** Base class for "query" (e.g., read-operations) action interfaces (query, count, aggregate). + ** Initially just here for the QueryStat methods - if we expand those to apply + ** to insert/update/delete, well, then rename this maybe to BaseActionInterface? + *******************************************************************************/ +public interface BaseQueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + default void setQueryStat(QueryStat queryStat) + { + ////////// + // noop // + ////////// + } + + /******************************************************************************* + ** + *******************************************************************************/ + default QueryStat getQueryStat() + { + return (null); + } + + /******************************************************************************* + ** + *******************************************************************************/ + default void setQueryStatFirstResultTime() + { + QueryStat queryStat = getQueryStat(); + if(queryStat != null) + { + if(queryStat.getFirstResultTimestamp() == null) + { + queryStat.setFirstResultTimestamp(Instant.now()); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java index 3ef3cd07..87ac0161 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java @@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; ** Interface for the Count action. ** *******************************************************************************/ -public interface CountInterface +public interface CountInterface extends BaseQueryInterface { /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java index d700a2e3..ad029050 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java @@ -22,67 +22,20 @@ package com.kingsrook.qqq.backend.core.actions.interfaces; -import java.time.Instant; -import java.util.Set; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; -import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; /******************************************************************************* ** Interface for the Query action. ** *******************************************************************************/ -public interface QueryInterface +public interface QueryInterface extends BaseQueryInterface { /******************************************************************************* ** *******************************************************************************/ QueryOutput execute(QueryInput queryInput) throws QException; - /******************************************************************************* - ** - *******************************************************************************/ - default void setQueryStat(QueryStat queryStat) - { - ////////// - // noop // - ////////// - } - - /******************************************************************************* - ** - *******************************************************************************/ - default QueryStat getQueryStat() - { - return (null); - } - - /******************************************************************************* - ** - *******************************************************************************/ - default void setQueryStatJoinTables(Set joinTableNames) - { - QueryStat queryStat = getQueryStat(); - if(queryStat != null) - { - queryStat.setJoinTableNames(joinTableNames); - } - } - - /******************************************************************************* - ** - *******************************************************************************/ - default void setQueryStatFirstResultTime() - { - QueryStat queryStat = getQueryStat(); - if(queryStat != null) - { - if(queryStat.getFirstResultTimestamp() == null) - { - queryStat.setFirstResultTimestamp(Instant.now()); - } - } - } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java index de4bdb93..56ce9555 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java @@ -23,9 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -43,11 +48,20 @@ public class AggregateAction { ActionHelper.validateSession(aggregateInput); + QTableMetaData table = aggregateInput.getTable(); + QBackendMetaData backend = aggregateInput.getBackend(); + + QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, aggregateInput.getFilter()); + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(aggregateInput.getBackend()); - // todo pre-customization - just get to modify the request? - AggregateOutput aggregateOutput = qModule.getAggregateInterface().execute(aggregateInput); - // todo post-customization - can do whatever w/ the result if you want + + AggregateInterface aggregateInterface = qModule.getAggregateInterface(); + aggregateInterface.setQueryStat(queryStat); + AggregateOutput aggregateOutput = aggregateInterface.execute(aggregateInput); + + QueryStatManager.getInstance().add(queryStat); + return aggregateOutput; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java index a4a0beb7..92337d6f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java @@ -23,9 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -43,11 +48,20 @@ public class CountAction { ActionHelper.validateSession(countInput); + QTableMetaData table = countInput.getTable(); + QBackendMetaData backend = countInput.getBackend(); + + QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, countInput.getFilter()); + QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); - QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend()); - // todo pre-customization - just get to modify the request? - CountOutput countOutput = qModule.getCountInterface().execute(countInput); - // todo post-customization - can do whatever w/ the result if you want + QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend()); + + CountInterface countInterface = qModule.getCountInterface(); + countInterface.setQueryStat(queryStat); + CountOutput countOutput = countInterface.execute(countInput); + + QueryStatManager.getInstance().add(queryStat); + return countOutput; } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index 01e3d4ae..c89c128a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -23,13 +23,11 @@ package com.kingsrook.qqq.backend.core.actions.tables; import java.io.Serializable; -import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.ActionHelper; @@ -57,7 +55,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; @@ -118,29 +115,16 @@ public class QueryAction } } - QueryStat queryStat = null; - if(table.isCapabilityEnabled(backend, Capability.QUERY_STATS)) - { - queryStat = new QueryStat(); - queryStat.setTableName(queryInput.getTableName()); - queryStat.setQueryFilter(Objects.requireNonNullElse(queryInput.getFilter(), new QQueryFilter())); - queryStat.setStartTimestamp(Instant.now()); - } + QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, queryInput.getFilter()); QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher(); QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend); - // todo pre-customization - just get to modify the request? QueryInterface queryInterface = qModule.getQueryInterface(); queryInterface.setQueryStat(queryStat); QueryOutput queryOutput = queryInterface.execute(queryInput); - // todo post-customization - can do whatever w/ the result if you want? - - if(queryStat != null) - { - QueryStatManager.getInstance().add(queryStat); - } + QueryStatManager.getInstance().add(queryStat); if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java index 2be92c6c..e32caf6e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java @@ -26,6 +26,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; @@ -43,7 +45,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.model.querystats.QueryStatCriteriaField; @@ -59,10 +63,18 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* + ** Singleton, which starts a thread, to store query stats into a table. ** + ** Supports these systemProperties or ENV_VARS: + ** qqq.queryStatManager.enabled / QQQ_QUERY_STAT_MANAGER_ENABLED + ** qqq.queryStatManager.minMillisToStore / QQQ_QUERY_STAT_MANAGER_MIN_MILLIS_TO_STORE + ** qqq.queryStatManager.jobPeriodSeconds / QQQ_QUERY_STAT_MANAGER_JOB_PERIOD_SECONDS + ** qqq.queryStatManager.jobInitialDelay / QQQ_QUERY_STAT_MANAGER_JOB_INITIAL_DELAY *******************************************************************************/ public class QueryStatManager { + private static final QLogger LOG = QLogger.getLogger(QueryStatManager.class); + private static QueryStatManager queryStatManager = null; // todo - support multiple qInstances? @@ -74,6 +86,10 @@ public class QueryStatManager private ScheduledExecutorService executorService; + private int jobPeriodSeconds = 60; + private int jobInitialDelay = 60; + private int minMillisToStore = 0; + /******************************************************************************* @@ -94,17 +110,66 @@ public class QueryStatManager if(queryStatManager == null) { queryStatManager = new QueryStatManager(); + + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + + Integer propertyMinMillisToStore = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.minMillisToStore", "QQQ_QUERY_STAT_MANAGER_MIN_MILLIS_TO_STORE", null); + if(propertyMinMillisToStore != null) + { + queryStatManager.setMinMillisToStore(propertyMinMillisToStore); + } + + Integer propertyJobPeriodSeconds = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.jobPeriodSeconds", "QQQ_QUERY_STAT_MANAGER_JOB_PERIOD_SECONDS", null); + if(propertyJobPeriodSeconds != null) + { + queryStatManager.setJobPeriodSeconds(propertyJobPeriodSeconds); + } + + Integer propertyJobInitialDelay = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.jobInitialDelay", "QQQ_QUERY_STAT_MANAGER_JOB_INITIAL_DELAY", null); + if(propertyJobInitialDelay != null) + { + queryStatManager.setJobInitialDelay(propertyJobInitialDelay); + } + } return (queryStatManager); } + /******************************************************************************* + ** + *******************************************************************************/ + public static QueryStat newQueryStat(QBackendMetaData backend, QTableMetaData table, QQueryFilter filter) + { + QueryStat queryStat = null; + + if(table.isCapabilityEnabled(backend, Capability.QUERY_STATS)) + { + queryStat = new QueryStat(); + queryStat.setTableName(table.getName()); + queryStat.setQueryFilter(Objects.requireNonNullElse(filter, new QQueryFilter())); + queryStat.setStartTimestamp(Instant.now()); + } + + return (queryStat); + } + + + /******************************************************************************* ** *******************************************************************************/ public void start(QInstance qInstance, Supplier sessionSupplier) { + if(!isEnabled()) + { + LOG.info("Not starting QueryStatManager per settings."); + return; + } + + LOG.info("Starting QueryStatManager"); + this.qInstance = qInstance; this.sessionSupplier = sessionSupplier; @@ -112,7 +177,17 @@ public class QueryStatManager queryStats = new ArrayList<>(); executorService = Executors.newSingleThreadScheduledExecutor(); - executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), 60, 60, TimeUnit.SECONDS); + executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), jobInitialDelay, jobPeriodSeconds, TimeUnit.SECONDS); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isEnabled() + { + return new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.queryStatManager.enabled", "QQQ_QUERY_STAT_MANAGER_ENABLED", true); } @@ -139,6 +214,11 @@ public class QueryStatManager *******************************************************************************/ public void add(QueryStat queryStat) { + if(queryStat == null) + { + return; + } + if(active) { //////////////////////////////////////////////////////////////////////////////////////// @@ -149,6 +229,20 @@ public class QueryStatManager queryStat.setFirstResultTimestamp(Instant.now()); } + if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null) + { + long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli(); + queryStat.setFirstResultMillis((int) millis); + } + + if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore) + { + ////////////////////////////////////////////////////////////// + // discard this record if it's under the min millis setting // + ////////////////////////////////////////////////////////////// + return; + } + if(queryStat.getSessionId() == null && QContext.getQSession() != null) { queryStat.setSessionId(QContext.getQSession().getUuid()); @@ -170,12 +264,13 @@ public class QueryStatManager if(className.contains(QueryStatManagerInsertJob.class.getName())) { expected = true; + break; } } if(!expected) { - e.printStackTrace(); + LOG.debug(e); } } } @@ -210,7 +305,7 @@ public class QueryStatManager /******************************************************************************* - ** + ** force stats to be stored right now (rather than letting the scheduled job do it) *******************************************************************************/ public void storeStatsNow() { @@ -220,7 +315,7 @@ public class QueryStatManager /******************************************************************************* - ** + ** Runnable that gets scheduled to periodically reset and store the list of collected stats *******************************************************************************/ private static class QueryStatManagerInsertJob implements Runnable { @@ -238,7 +333,18 @@ public class QueryStatManager { QContext.init(getInstance().qInstance, getInstance().sessionSupplier.get()); + ///////////////////////////////////////////////////////////////////////////////////// + // every time we re-run, check if we've been turned off - if so, stop the service. // + ///////////////////////////////////////////////////////////////////////////////////// + if(!isEnabled()) + { + LOG.info("Stopping QueryStatManager."); + getInstance().stop(); + return; + } + List list = getInstance().getListAndReset(); + LOG.info(logPair("queryStatListSize", list.size())); if(list.isEmpty()) @@ -254,15 +360,6 @@ public class QueryStatManager { try { - /////////////////////////////////////////////// - // compute the millis (so you don't have to) // - /////////////////////////////////////////////// - if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null) - { - long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli(); - queryStat.setFirstResultMillis((int) millis); - } - ////////////////////// // set the table id // ////////////////////// @@ -386,22 +483,25 @@ public class QueryStatManager String fieldName = orderBy.getFieldName(); QueryStatOrderByField queryStatOrderByField = new QueryStatOrderByField(); - if(fieldName.contains(".")) + if(fieldName != null) { - String[] parts = fieldName.split("\\."); - if(parts.length > 1) + if(fieldName.contains(".")) { - queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0])); - queryStatOrderByField.setName(parts[1]); + String[] parts = fieldName.split("\\."); + if(parts.length > 1) + { + queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0])); + queryStatOrderByField.setName(parts[1]); + } + } + else + { + queryStatOrderByField.setQqqTableId(qqqTableId); + queryStatOrderByField.setName(fieldName); } - } - else - { - queryStatOrderByField.setQqqTableId(qqqTableId); - queryStatOrderByField.setName(fieldName); - } - queryStatOrderByFieldList.add(queryStatOrderByField); + queryStatOrderByFieldList.add(queryStatOrderByField); + } } } @@ -444,4 +544,97 @@ public class QueryStatManager } } + + + /******************************************************************************* + ** Getter for jobPeriodSeconds + *******************************************************************************/ + public int getJobPeriodSeconds() + { + return (this.jobPeriodSeconds); + } + + + + /******************************************************************************* + ** Setter for jobPeriodSeconds + *******************************************************************************/ + public void setJobPeriodSeconds(int jobPeriodSeconds) + { + this.jobPeriodSeconds = jobPeriodSeconds; + } + + + + /******************************************************************************* + ** Fluent setter for jobPeriodSeconds + *******************************************************************************/ + public QueryStatManager withJobPeriodSeconds(int jobPeriodSeconds) + { + this.jobPeriodSeconds = jobPeriodSeconds; + return (this); + } + + + + /******************************************************************************* + ** Getter for jobInitialDelay + *******************************************************************************/ + public int getJobInitialDelay() + { + return (this.jobInitialDelay); + } + + + + /******************************************************************************* + ** Setter for jobInitialDelay + *******************************************************************************/ + public void setJobInitialDelay(int jobInitialDelay) + { + this.jobInitialDelay = jobInitialDelay; + } + + + + /******************************************************************************* + ** Fluent setter for jobInitialDelay + *******************************************************************************/ + public QueryStatManager withJobInitialDelay(int jobInitialDelay) + { + this.jobInitialDelay = jobInitialDelay; + return (this); + } + + + + /******************************************************************************* + ** Getter for minMillisToStore + *******************************************************************************/ + public int getMinMillisToStore() + { + return (this.minMillisToStore); + } + + + + /******************************************************************************* + ** Setter for minMillisToStore + *******************************************************************************/ + public void setMinMillisToStore(int minMillisToStore) + { + this.minMillisToStore = minMillisToStore; + } + + + + /******************************************************************************* + ** Fluent setter for minMillisToStore + *******************************************************************************/ + public QueryStatManager withMinMillisToStore(int minMillisToStore) + { + this.minMillisToStore = minMillisToStore; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java index 8f01d2e3..7df652e8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java @@ -30,6 +30,7 @@ import java.util.Locale; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import io.github.cdimascio.dotenv.Dotenv; import io.github.cdimascio.dotenv.DotenvEntry; @@ -266,4 +267,111 @@ public class QMetaDataVariableInterpreter valueMaps.put(name, values); } + + + + /******************************************************************************* + ** First look for a boolean ("true" or "false") in the specified system property - + ** Next look for a boolean in the specified env var name - + ** Finally return the default. + *******************************************************************************/ + public boolean getBooleanFromPropertyOrEnvironment(String systemPropertyName, String environmentVariableName, boolean defaultIfNotSet) + { + String propertyValue = System.getProperty(systemPropertyName); + if(StringUtils.hasContent(propertyValue)) + { + if("false".equalsIgnoreCase(propertyValue)) + { + LOG.info("Read system property [" + systemPropertyName + "] as boolean false."); + return (false); + } + else if("true".equalsIgnoreCase(propertyValue)) + { + LOG.info("Read system property [" + systemPropertyName + "] as boolean true."); + return (true); + } + else + { + LOG.warn("Unrecognized boolean value [" + propertyValue + "] for system property [" + systemPropertyName + "]."); + } + } + + String envValue = interpret("${env." + environmentVariableName + "}"); + if(StringUtils.hasContent(envValue)) + { + if("false".equalsIgnoreCase(envValue)) + { + LOG.info("Read env var [" + environmentVariableName + "] as boolean false."); + return (false); + } + else if("true".equalsIgnoreCase(envValue)) + { + LOG.info("Read env var [" + environmentVariableName + "] as boolean true."); + return (true); + } + else + { + LOG.warn("Unrecognized boolean value [" + envValue + "] for env var [" + environmentVariableName + "]."); + } + } + + return defaultIfNotSet; + } + + + + /******************************************************************************* + ** First look for an Integer in the specified system property - + ** Next look for an Integer in the specified env var name - + ** Finally return the default (null allowed as default!) + *******************************************************************************/ + public Integer getIntegerFromPropertyOrEnvironment(String systemPropertyName, String environmentVariableName, Integer defaultIfNotSet) + { + String propertyValue = System.getProperty(systemPropertyName); + if(StringUtils.hasContent(propertyValue)) + { + if(canParseAsInteger(propertyValue)) + { + LOG.info("Read system property [" + systemPropertyName + "] as integer " + propertyValue); + return (Integer.parseInt(propertyValue)); + } + else + { + LOG.warn("Unrecognized integer value [" + propertyValue + "] for system property [" + systemPropertyName + "]."); + } + } + + String envValue = interpret("${env." + environmentVariableName + "}"); + if(StringUtils.hasContent(envValue)) + { + if(canParseAsInteger(envValue)) + { + LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName); + return (Integer.parseInt(propertyValue)); + } + else + { + LOG.warn("Unrecognized integer value [" + envValue + "] for env var [" + environmentVariableName + "]."); + } + } + + return defaultIfNotSet; + } + + + + /******************************************************************************* + ** we'd use NumberUtils.isDigits, but that doesn't allow negatives, or + ** numberUtils.isParseable, but that allows decimals, so... + *******************************************************************************/ + private boolean canParseAsInteger(String value) + { + if(value == null) + { + return (false); + } + + return (value.matches("^-?[0-9]+$")); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java index b8e91b99..88e0ddb2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java @@ -110,20 +110,9 @@ public class ScheduleManager *******************************************************************************/ public void start() { - String propertyName = "qqq.scheduleManager.enabled"; - String propertyValue = System.getProperty(propertyName); - if("false".equals(propertyValue)) + if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true)) { - LOG.info("Not starting ScheduleManager (per system property] [" + propertyName + "=" + propertyValue + "])."); - return; - } - - QMetaDataVariableInterpreter qMetaDataVariableInterpreter = new QMetaDataVariableInterpreter(); - String envName = "QQQ_SCHEDULE_MANAGER_ENABLED"; - String envValue = qMetaDataVariableInterpreter.interpret("${env." + envName + "}"); - if("false".equals(envValue)) - { - LOG.info("Not starting ScheduleManager (per environment variable] [" + envName + "=" + envValue + "])."); + LOG.info("Not starting ScheduleManager per settings."); return; } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java index 024ba855..9633dec9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java @@ -30,8 +30,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -224,6 +226,53 @@ class QMetaDataVariableInterpreterTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetBooleanFromPropertyOrEnvironment() + { + QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter(); + + ////////////////////////////////////////////////////////// + // if neither prop nor env is set, get back the default // + ////////////////////////////////////////////////////////// + assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("notSet", "NOT_SET", false)); + assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("notSet", "NOT_SET", true)); + + ///////////////////////////////////////////// + // unrecognized values are same as not set // + ///////////////////////////////////////////// + System.setProperty("unrecognized", "asdf"); + interpreter.setEnvironmentOverrides(Map.of("UNRECOGNIZED", "1234")); + assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", false)); + assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", true)); + + ///////////////////////////////// + // if only prop is set, get it // + ///////////////////////////////// + assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("foo.enabled", "FOO_ENABLED", false)); + System.setProperty("foo.enabled", "true"); + assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("foo.enabled", "FOO_ENABLED", false)); + + //////////////////////////////// + // if only env is set, get it // + //////////////////////////////// + assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("bar.enabled", "BAR_ENABLED", false)); + interpreter.setEnvironmentOverrides(Map.of("BAR_ENABLED", "true")); + assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("bar.enabled", "BAR_ENABLED", false)); + + /////////////////////////////////// + // if both are set, get the prop // + /////////////////////////////////// + System.setProperty("baz.enabled", "true"); + interpreter.setEnvironmentOverrides(Map.of("BAZ_ENABLED", "false")); + assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("baz.enabled", "BAZ_ENABLED", true)); + assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("baz.enabled", "BAZ_ENABLED", false)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index f6386885..54d8ce3c 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -68,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -86,6 +87,8 @@ public abstract class AbstractRDBMSAction implements QActionInterface { private static final QLogger LOG = QLogger.getLogger(AbstractRDBMSAction.class); + protected QueryStat queryStat; + /******************************************************************************* @@ -1037,4 +1040,47 @@ public abstract class AbstractRDBMSAction implements QActionInterface return (false); } + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void setSqlAndJoinsInQueryStat(CharSequence sql, JoinsContext joinsContext) + { + if(queryStat != null) + { + queryStat.setQueryText(sql.toString()); + + if(CollectionUtils.nullSafeHasContents(joinsContext.getQueryJoins())) + { + Set joinTableNames = new HashSet<>(); + for(QueryJoin queryJoin : joinsContext.getQueryJoins()) + { + joinTableNames.add(queryJoin.getJoinTable()); + } + queryStat.setJoinTableNames(joinTableNames); + } + } + } + + + + /******************************************************************************* + ** Getter for queryStat + *******************************************************************************/ + public QueryStat getQueryStat() + { + return (this.queryStat); + } + + + + /******************************************************************************* + ** Setter for queryStat + *******************************************************************************/ + public void setQueryStat(QueryStat queryStat) + { + this.queryStat = queryStat; + } + } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index b7ba77ea..8e786242 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -92,6 +92,8 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega // todo sql customization - can edit sql and/or param list + setSqlAndJoinsInQueryStat(sql, joinsContext); + AggregateOutput rs = new AggregateOutput(); List results = new ArrayList<>(); rs.setResults(results); @@ -104,6 +106,8 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega { while(resultSet.next()) { + setQueryStatFirstResultTime(); + AggregateResult result = new AggregateResult(); results.add(result); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index e713c1b5..64676fc9 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -77,6 +77,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf sql += " WHERE " + makeWhereClause(countInput.getInstance(), countInput.getSession(), table, joinsContext, filter, params); // todo sql customization - can edit sql and/or param list + setSqlAndJoinsInQueryStat(sql, joinsContext); + CountOutput rs = new CountOutput(); try(Connection connection = getConnection(countInput)) { @@ -86,6 +88,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf { if(resultSet.next()) { + setQueryStatFirstResultTime(); + rs.setCount(resultSet.getInt("record_count")); if(BooleanUtils.isTrue(countInput.getIncludeDistinctCount())) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 21c09d8e..dd674763 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -30,11 +30,9 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -49,7 +47,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; @@ -63,8 +60,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { private static final QLogger LOG = QLogger.getLogger(RDBMSQueryAction.class); - private QueryStat queryStat; - /******************************************************************************* @@ -105,6 +100,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf // todo sql customization - can edit sql and/or param list + setSqlAndJoinsInQueryStat(sql, joinsContext); + Connection connection; boolean needToCloseConnection = false; if(queryInput.getTransaction() != null && queryInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction) @@ -144,21 +141,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf ////////////////////////////////////////////// QueryOutput queryOutput = new QueryOutput(queryInput); - if(queryStat != null) - { - queryStat.setQueryText(sql.toString()); - - if(CollectionUtils.nullSafeHasContents(joinsContext.getQueryJoins())) - { - Set joinTableNames = new HashSet<>(); - for(QueryJoin queryJoin : joinsContext.getQueryJoins()) - { - joinTableNames.add(queryJoin.getJoinTable()); - } - setQueryStatJoinTables(joinTableNames); - } - } - PreparedStatement statement = createStatement(connection, sql.toString(), queryInput); QueryManager.executeStatement(statement, ((ResultSet resultSet) -> { @@ -352,26 +334,4 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf return (statement); } - - - /******************************************************************************* - ** Getter for queryStat - *******************************************************************************/ - @Override - public QueryStat getQueryStat() - { - return (this.queryStat); - } - - - - /******************************************************************************* - ** Setter for queryStat - *******************************************************************************/ - @Override - public void setQueryStat(QueryStat queryStat) - { - this.queryStat = queryStat; - } - }