QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

add query stats to count, aggregate actions; add system/env prop checks; ready for initial dev deployment

Browse Source
This commit is contained in:
darin.kelkhoff
2023-06-22 11:07:24 -05:00
parent f30b2a9ef8
commit 1822dd8189
15 changed files with 543 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/AggregateInterface.java
View File

@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOu
** Interface for the Aggregate action.
**
*******************************************************************************/
public interface AggregateInterface
public interface AggregateInterface extends BaseQueryInterface
{
/*******************************************************************************
**

70
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/BaseQueryInterface.java Normal file
View File

@ -0,0 +1,70 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.interfaces;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
/*******************************************************************************
** Base class for "query" (e.g., read-operations) action interfaces (query, count, aggregate).
** Initially just here for the QueryStat methods - if we expand those to apply
** to insert/update/delete, well, then rename this maybe to BaseActionInterface?
*******************************************************************************/
public interface BaseQueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
default void setQueryStat(QueryStat queryStat)
{
//////////
// noop //
//////////
}
/*******************************************************************************
**
*******************************************************************************/
default QueryStat getQueryStat()
{
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
default void setQueryStatFirstResultTime()
{
QueryStat queryStat = getQueryStat();
if(queryStat != null)
{
if(queryStat.getFirstResultTimestamp() == null)
{
queryStat.setFirstResultTimestamp(Instant.now());
}
}
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/CountInterface.java
View File

@ -31,7 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
** Interface for the Count action.
**
*******************************************************************************/
public interface CountInterface
public interface CountInterface extends BaseQueryInterface
{
/*******************************************************************************
**

49
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java
View File

@ -22,67 +22,20 @@
package com.kingsrook.qqq.backend.core.actions.interfaces;
import java.time.Instant;
import java.util.Set;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
/*******************************************************************************
** Interface for the Query action.
**
*******************************************************************************/
public interface QueryInterface
public interface QueryInterface extends BaseQueryInterface
{
/*******************************************************************************
**
*******************************************************************************/
QueryOutput execute(QueryInput queryInput) throws QException;
/*******************************************************************************
**
*******************************************************************************/
default void setQueryStat(QueryStat queryStat)
{
//////////
// noop //
//////////
}
/*******************************************************************************
**
*******************************************************************************/
default QueryStat getQueryStat()
{
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
default void setQueryStatJoinTables(Set<String> joinTableNames)
{
QueryStat queryStat = getQueryStat();
if(queryStat != null)
{
queryStat.setJoinTableNames(joinTableNames);
}
}
/*******************************************************************************
**
*******************************************************************************/
default void setQueryStatFirstResultTime()
{
QueryStat queryStat = getQueryStat();
if(queryStat != null)
{
if(queryStat.getFirstResultTimestamp() == null)
{
queryStat.setFirstResultTimestamp(Instant.now());
}
}
}
}

20
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/AggregateAction.java
View File

@ -23,9 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -43,11 +48,20 @@ public class AggregateAction
{
ActionHelper.validateSession(aggregateInput);
QTableMetaData table = aggregateInput.getTable();
QBackendMetaData backend = aggregateInput.getBackend();
QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, aggregateInput.getFilter());
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(aggregateInput.getBackend());
// todo pre-customization - just get to modify the request?
AggregateOutput aggregateOutput = qModule.getAggregateInterface().execute(aggregateInput);
// todo post-customization - can do whatever w/ the result if you want
AggregateInterface aggregateInterface = qModule.getAggregateInterface();
aggregateInterface.setQueryStat(queryStat);
AggregateOutput aggregateOutput = aggregateInterface.execute(aggregateInput);
QueryStatManager.getInstance().add(queryStat);
return aggregateOutput;
}
}

22
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java
View File

@ -23,9 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -43,11 +48,20 @@ public class CountAction
{
ActionHelper.validateSession(countInput);
QTableMetaData table = countInput.getTable();
QBackendMetaData backend = countInput.getBackend();
QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, countInput.getFilter());
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend());
// todo pre-customization - just get to modify the request?
CountOutput countOutput = qModule.getCountInterface().execute(countInput);
// todo post-customization - can do whatever w/ the result if you want
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(countInput.getBackend());
CountInterface countInterface = qModule.getCountInterface();
countInterface.setQueryStat(queryStat);
CountOutput countOutput = countInterface.execute(countInput);
QueryStatManager.getInstance().add(queryStat);
return countOutput;
}
}

20
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
View File

@ -23,13 +23,11 @@ package com.kingsrook.qqq.backend.core.actions.tables;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
@ -57,7 +55,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
@ -118,29 +115,16 @@ public class QueryAction
}
}
QueryStat queryStat = null;
if(table.isCapabilityEnabled(backend, Capability.QUERY_STATS))
{
queryStat = new QueryStat();
queryStat.setTableName(queryInput.getTableName());
queryStat.setQueryFilter(Objects.requireNonNullElse(queryInput.getFilter(), new QQueryFilter()));
queryStat.setStartTimestamp(Instant.now());
}
QueryStat queryStat = QueryStatManager.newQueryStat(backend, table, queryInput.getFilter());
QBackendModuleDispatcher qBackendModuleDispatcher = new QBackendModuleDispatcher();
QBackendModuleInterface qModule = qBackendModuleDispatcher.getQBackendModule(backend);
// todo pre-customization - just get to modify the request?
QueryInterface queryInterface = qModule.getQueryInterface();
queryInterface.setQueryStat(queryStat);
QueryOutput queryOutput = queryInterface.execute(queryInput);
// todo post-customization - can do whatever w/ the result if you want?
if(queryStat != null)
{
QueryStatManager.getInstance().add(queryStat);
}
QueryStatManager.getInstance().add(queryStat);
if(queryInput.getRecordPipe() instanceof BufferedRecordPipe bufferedRecordPipe)
{

243
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java
View File

@ -26,6 +26,7 @@ import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
@ -43,7 +45,9 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStatCriteriaField;
@ -59,10 +63,18 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Singleton, which starts a thread, to store query stats into a table.
**
** Supports these systemProperties or ENV_VARS:
** qqq.queryStatManager.enabled / QQQ_QUERY_STAT_MANAGER_ENABLED
** qqq.queryStatManager.minMillisToStore / QQQ_QUERY_STAT_MANAGER_MIN_MILLIS_TO_STORE
** qqq.queryStatManager.jobPeriodSeconds / QQQ_QUERY_STAT_MANAGER_JOB_PERIOD_SECONDS
** qqq.queryStatManager.jobInitialDelay / QQQ_QUERY_STAT_MANAGER_JOB_INITIAL_DELAY
*******************************************************************************/
public class QueryStatManager
{
private static final QLogger LOG = QLogger.getLogger(QueryStatManager.class);
private static QueryStatManager queryStatManager = null;
// todo - support multiple qInstances?
@ -74,6 +86,10 @@ public class QueryStatManager
private ScheduledExecutorService executorService;
private int jobPeriodSeconds = 60;
private int jobInitialDelay = 60;
private int minMillisToStore = 0;
/*******************************************************************************
@ -94,17 +110,66 @@ public class QueryStatManager
if(queryStatManager == null)
{
queryStatManager = new QueryStatManager();
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
Integer propertyMinMillisToStore = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.minMillisToStore", "QQQ_QUERY_STAT_MANAGER_MIN_MILLIS_TO_STORE", null);
if(propertyMinMillisToStore != null)
{
queryStatManager.setMinMillisToStore(propertyMinMillisToStore);
}
Integer propertyJobPeriodSeconds = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.jobPeriodSeconds", "QQQ_QUERY_STAT_MANAGER_JOB_PERIOD_SECONDS", null);
if(propertyJobPeriodSeconds != null)
{
queryStatManager.setJobPeriodSeconds(propertyJobPeriodSeconds);
}
Integer propertyJobInitialDelay = interpreter.getIntegerFromPropertyOrEnvironment("qqq.queryStatManager.jobInitialDelay", "QQQ_QUERY_STAT_MANAGER_JOB_INITIAL_DELAY", null);
if(propertyJobInitialDelay != null)
{
queryStatManager.setJobInitialDelay(propertyJobInitialDelay);
}
}
return (queryStatManager);
}
/*******************************************************************************
**
*******************************************************************************/
public static QueryStat newQueryStat(QBackendMetaData backend, QTableMetaData table, QQueryFilter filter)
{
QueryStat queryStat = null;
if(table.isCapabilityEnabled(backend, Capability.QUERY_STATS))
{
queryStat = new QueryStat();
queryStat.setTableName(table.getName());
queryStat.setQueryFilter(Objects.requireNonNullElse(filter, new QQueryFilter()));
queryStat.setStartTimestamp(Instant.now());
}
return (queryStat);
}
/*******************************************************************************
**
*******************************************************************************/
public void start(QInstance qInstance, Supplier<QSession> sessionSupplier)
{
if(!isEnabled())
{
LOG.info("Not starting QueryStatManager per settings.");
return;
}
LOG.info("Starting QueryStatManager");
this.qInstance = qInstance;
this.sessionSupplier = sessionSupplier;
@ -112,7 +177,17 @@ public class QueryStatManager
queryStats = new ArrayList<>();
executorService = Executors.newSingleThreadScheduledExecutor();
executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), 60, 60, TimeUnit.SECONDS);
executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), jobInitialDelay, jobPeriodSeconds, TimeUnit.SECONDS);
}
/*******************************************************************************
**
*******************************************************************************/
private static boolean isEnabled()
{
return new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.queryStatManager.enabled", "QQQ_QUERY_STAT_MANAGER_ENABLED", true);
}
@ -139,6 +214,11 @@ public class QueryStatManager
*******************************************************************************/
public void add(QueryStat queryStat)
{
if(queryStat == null)
{
return;
}
if(active)
{
////////////////////////////////////////////////////////////////////////////////////////
@ -149,6 +229,20 @@ public class QueryStatManager
queryStat.setFirstResultTimestamp(Instant.now());
}
if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
{
long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
queryStat.setFirstResultMillis((int) millis);
}
if(queryStat.getFirstResultMillis() != null && queryStat.getFirstResultMillis() < minMillisToStore)
{
//////////////////////////////////////////////////////////////
// discard this record if it's under the min millis setting //
//////////////////////////////////////////////////////////////
return;
}
if(queryStat.getSessionId() == null && QContext.getQSession() != null)
{
queryStat.setSessionId(QContext.getQSession().getUuid());
@ -170,12 +264,13 @@ public class QueryStatManager
if(className.contains(QueryStatManagerInsertJob.class.getName()))
{
expected = true;
break;
}
}
if(!expected)
{
e.printStackTrace();
LOG.debug(e);
}
}
}
@ -210,7 +305,7 @@ public class QueryStatManager
/*******************************************************************************
**
** force stats to be stored right now (rather than letting the scheduled job do it)
*******************************************************************************/
public void storeStatsNow()
{
@ -220,7 +315,7 @@ public class QueryStatManager
/*******************************************************************************
**
** Runnable that gets scheduled to periodically reset and store the list of collected stats
*******************************************************************************/
private static class QueryStatManagerInsertJob implements Runnable
{
@ -238,7 +333,18 @@ public class QueryStatManager
{
QContext.init(getInstance().qInstance, getInstance().sessionSupplier.get());
/////////////////////////////////////////////////////////////////////////////////////
// every time we re-run, check if we've been turned off - if so, stop the service. //
/////////////////////////////////////////////////////////////////////////////////////
if(!isEnabled())
{
LOG.info("Stopping QueryStatManager.");
getInstance().stop();
return;
}
List<QueryStat> list = getInstance().getListAndReset();
LOG.info(logPair("queryStatListSize", list.size()));
if(list.isEmpty())
@ -254,15 +360,6 @@ public class QueryStatManager
{
try
{
///////////////////////////////////////////////
// compute the millis (so you don't have to) //
///////////////////////////////////////////////
if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
{
long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
queryStat.setFirstResultMillis((int) millis);
}
//////////////////////
// set the table id //
//////////////////////
@ -386,22 +483,25 @@ public class QueryStatManager
String fieldName = orderBy.getFieldName();
QueryStatOrderByField queryStatOrderByField = new QueryStatOrderByField();
if(fieldName.contains("."))
if(fieldName != null)
{
String[] parts = fieldName.split("\\.");
if(parts.length > 1)
if(fieldName.contains("."))
{
queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0]));
queryStatOrderByField.setName(parts[1]);
String[] parts = fieldName.split("\\.");
if(parts.length > 1)
{
queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0]));
queryStatOrderByField.setName(parts[1]);
}
}
else
{
queryStatOrderByField.setQqqTableId(qqqTableId);
queryStatOrderByField.setName(fieldName);
}
}
else
{
queryStatOrderByField.setQqqTableId(qqqTableId);
queryStatOrderByField.setName(fieldName);
}
queryStatOrderByFieldList.add(queryStatOrderByField);
queryStatOrderByFieldList.add(queryStatOrderByField);
}
}
}
@ -444,4 +544,97 @@ public class QueryStatManager
}
}
/*******************************************************************************
** Getter for jobPeriodSeconds
*******************************************************************************/
public int getJobPeriodSeconds()
{
return (this.jobPeriodSeconds);
}
/*******************************************************************************
** Setter for jobPeriodSeconds
*******************************************************************************/
public void setJobPeriodSeconds(int jobPeriodSeconds)
{
this.jobPeriodSeconds = jobPeriodSeconds;
}
/*******************************************************************************
** Fluent setter for jobPeriodSeconds
*******************************************************************************/
public QueryStatManager withJobPeriodSeconds(int jobPeriodSeconds)
{
this.jobPeriodSeconds = jobPeriodSeconds;
return (this);
}
/*******************************************************************************
** Getter for jobInitialDelay
*******************************************************************************/
public int getJobInitialDelay()
{
return (this.jobInitialDelay);
}
/*******************************************************************************
** Setter for jobInitialDelay
*******************************************************************************/
public void setJobInitialDelay(int jobInitialDelay)
{
this.jobInitialDelay = jobInitialDelay;
}
/*******************************************************************************
** Fluent setter for jobInitialDelay
*******************************************************************************/
public QueryStatManager withJobInitialDelay(int jobInitialDelay)
{
this.jobInitialDelay = jobInitialDelay;
return (this);
}
/*******************************************************************************
** Getter for minMillisToStore
*******************************************************************************/
public int getMinMillisToStore()
{
return (this.minMillisToStore);
}
/*******************************************************************************
** Setter for minMillisToStore
*******************************************************************************/
public void setMinMillisToStore(int minMillisToStore)
{
this.minMillisToStore = minMillisToStore;
}
/*******************************************************************************
** Fluent setter for minMillisToStore
*******************************************************************************/
public QueryStatManager withMinMillisToStore(int minMillisToStore)
{
this.minMillisToStore = minMillisToStore;
return (this);
}
}

108
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java
View File

@ -30,6 +30,7 @@ import java.util.Locale;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import io.github.cdimascio.dotenv.Dotenv;
import io.github.cdimascio.dotenv.DotenvEntry;
@ -266,4 +267,111 @@ public class QMetaDataVariableInterpreter
valueMaps.put(name, values);
}
/*******************************************************************************
** First look for a boolean ("true" or "false") in the specified system property -
** Next look for a boolean in the specified env var name -
** Finally return the default.
*******************************************************************************/
public boolean getBooleanFromPropertyOrEnvironment(String systemPropertyName, String environmentVariableName, boolean defaultIfNotSet)
{
String propertyValue = System.getProperty(systemPropertyName);
if(StringUtils.hasContent(propertyValue))
{
if("false".equalsIgnoreCase(propertyValue))
{
LOG.info("Read system property [" + systemPropertyName + "] as boolean false.");
return (false);
}
else if("true".equalsIgnoreCase(propertyValue))
{
LOG.info("Read system property [" + systemPropertyName + "] as boolean true.");
return (true);
}
else
{
LOG.warn("Unrecognized boolean value [" + propertyValue + "] for system property [" + systemPropertyName + "].");
}
}
String envValue = interpret("${env." + environmentVariableName + "}");
if(StringUtils.hasContent(envValue))
{
if("false".equalsIgnoreCase(envValue))
{
LOG.info("Read env var [" + environmentVariableName + "] as boolean false.");
return (false);
}
else if("true".equalsIgnoreCase(envValue))
{
LOG.info("Read env var [" + environmentVariableName + "] as boolean true.");
return (true);
}
else
{
LOG.warn("Unrecognized boolean value [" + envValue + "] for env var [" + environmentVariableName + "].");
}
}
return defaultIfNotSet;
}
/*******************************************************************************
** First look for an Integer in the specified system property -
** Next look for an Integer in the specified env var name -
** Finally return the default (null allowed as default!)
*******************************************************************************/
public Integer getIntegerFromPropertyOrEnvironment(String systemPropertyName, String environmentVariableName, Integer defaultIfNotSet)
{
String propertyValue = System.getProperty(systemPropertyName);
if(StringUtils.hasContent(propertyValue))
{
if(canParseAsInteger(propertyValue))
{
LOG.info("Read system property [" + systemPropertyName + "] as integer " + propertyValue);
return (Integer.parseInt(propertyValue));
}
else
{
LOG.warn("Unrecognized integer value [" + propertyValue + "] for system property [" + systemPropertyName + "].");
}
}
String envValue = interpret("${env." + environmentVariableName + "}");
if(StringUtils.hasContent(envValue))
{
if(canParseAsInteger(envValue))
{
LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName);
return (Integer.parseInt(propertyValue));
}
else
{
LOG.warn("Unrecognized integer value [" + envValue + "] for env var [" + environmentVariableName + "].");
}
}
return defaultIfNotSet;
}
/*******************************************************************************
** we'd use NumberUtils.isDigits, but that doesn't allow negatives, or
** numberUtils.isParseable, but that allows decimals, so...
*******************************************************************************/
private boolean canParseAsInteger(String value)
{
if(value == null)
{
return (false);
}
return (value.matches("^-?[0-9]+$"));
}
}

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/ScheduleManager.java
View File

@ -110,20 +110,9 @@ public class ScheduleManager
*******************************************************************************/
public void start()
{
String propertyName = "qqq.scheduleManager.enabled";
String propertyValue = System.getProperty(propertyName);
if("false".equals(propertyValue))
if(!new QMetaDataVariableInterpreter().getBooleanFromPropertyOrEnvironment("qqq.scheduleManager.enabled", "QQQ_SCHEDULE_MANAGER_ENABLED", true))
{
LOG.info("Not starting ScheduleManager (per system property] [" + propertyName + "=" + propertyValue + "]).");
return;
}
QMetaDataVariableInterpreter qMetaDataVariableInterpreter = new QMetaDataVariableInterpreter();
String envName = "QQQ_SCHEDULE_MANAGER_ENABLED";
String envValue = qMetaDataVariableInterpreter.interpret("${env." + envName + "}");
if("false".equals(envValue))
{
LOG.info("Not starting ScheduleManager (per environment variable] [" + envName + "=" + envValue + "]).");
LOG.info("Not starting ScheduleManager per settings.");
return;
}

49
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java
View File

@ -30,8 +30,10 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
@ -224,6 +226,53 @@ class QMetaDataVariableInterpreterTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testGetBooleanFromPropertyOrEnvironment()
{
QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
//////////////////////////////////////////////////////////
// if neither prop nor env is set, get back the default //
//////////////////////////////////////////////////////////
assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("notSet", "NOT_SET", false));
assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("notSet", "NOT_SET", true));
/////////////////////////////////////////////
// unrecognized values are same as not set //
/////////////////////////////////////////////
System.setProperty("unrecognized", "asdf");
interpreter.setEnvironmentOverrides(Map.of("UNRECOGNIZED", "1234"));
assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", false));
assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", true));
/////////////////////////////////
// if only prop is set, get it //
/////////////////////////////////
assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("foo.enabled", "FOO_ENABLED", false));
System.setProperty("foo.enabled", "true");
assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("foo.enabled", "FOO_ENABLED", false));
////////////////////////////////
// if only env is set, get it //
////////////////////////////////
assertFalse(interpreter.getBooleanFromPropertyOrEnvironment("bar.enabled", "BAR_ENABLED", false));
interpreter.setEnvironmentOverrides(Map.of("BAR_ENABLED", "true"));
assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("bar.enabled", "BAR_ENABLED", false));
///////////////////////////////////
// if both are set, get the prop //
///////////////////////////////////
System.setProperty("baz.enabled", "true");
interpreter.setEnvironmentOverrides(Map.of("BAZ_ENABLED", "false"));
assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("baz.enabled", "BAZ_ENABLED", true));
assertTrue(interpreter.getBooleanFromPropertyOrEnvironment("baz.enabled", "BAZ_ENABLED", false));
}
/*******************************************************************************
**
*******************************************************************************/

46
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -68,6 +68,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -86,6 +87,8 @@ public abstract class AbstractRDBMSAction implements QActionInterface
{
private static final QLogger LOG = QLogger.getLogger(AbstractRDBMSAction.class);
protected QueryStat queryStat;
/*******************************************************************************
@ -1037,4 +1040,47 @@ public abstract class AbstractRDBMSAction implements QActionInterface
return (false);
}
/*******************************************************************************
**
*******************************************************************************/
protected void setSqlAndJoinsInQueryStat(CharSequence sql, JoinsContext joinsContext)
{
if(queryStat != null)
{
queryStat.setQueryText(sql.toString());
if(CollectionUtils.nullSafeHasContents(joinsContext.getQueryJoins()))
{
Set<String> joinTableNames = new HashSet<>();
for(QueryJoin queryJoin : joinsContext.getQueryJoins())
{
joinTableNames.add(queryJoin.getJoinTable());
}
queryStat.setJoinTableNames(joinTableNames);
}
}
}
/*******************************************************************************
** Getter for queryStat
*******************************************************************************/
public QueryStat getQueryStat()
{
return (this.queryStat);
}
/*******************************************************************************
** Setter for queryStat
*******************************************************************************/
public void setQueryStat(QueryStat queryStat)
{
this.queryStat = queryStat;
}
}

4
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java
View File

@ -92,6 +92,8 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
// todo sql customization - can edit sql and/or param list
setSqlAndJoinsInQueryStat(sql, joinsContext);
AggregateOutput rs = new AggregateOutput();
List<AggregateResult> results = new ArrayList<>();
rs.setResults(results);
@ -104,6 +106,8 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
{
while(resultSet.next())
{
setQueryStatFirstResultTime();
AggregateResult result = new AggregateResult();
results.add(result);

4
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java
View File

@ -77,6 +77,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
sql += " WHERE " + makeWhereClause(countInput.getInstance(), countInput.getSession(), table, joinsContext, filter, params);
// todo sql customization - can edit sql and/or param list
setSqlAndJoinsInQueryStat(sql, joinsContext);
CountOutput rs = new CountOutput();
try(Connection connection = getConnection(countInput))
{
@ -86,6 +88,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
{
if(resultSet.next())
{
setQueryStatFirstResultTime();
rs.setCount(resultSet.getInt("record_count"));
if(BooleanUtils.isTrue(countInput.getIncludeDistinctCount()))

44
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -30,11 +30,9 @@ import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -49,7 +47,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
@ -63,8 +60,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
{
private static final QLogger LOG = QLogger.getLogger(RDBMSQueryAction.class);
private QueryStat queryStat;
/*******************************************************************************
@ -105,6 +100,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
// todo sql customization - can edit sql and/or param list
setSqlAndJoinsInQueryStat(sql, joinsContext);
Connection connection;
boolean needToCloseConnection = false;
if(queryInput.getTransaction() != null && queryInput.getTransaction() instanceof RDBMSTransaction rdbmsTransaction)
@ -144,21 +141,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
//////////////////////////////////////////////
QueryOutput queryOutput = new QueryOutput(queryInput);
if(queryStat != null)
{
queryStat.setQueryText(sql.toString());
if(CollectionUtils.nullSafeHasContents(joinsContext.getQueryJoins()))
{
Set<String> joinTableNames = new HashSet<>();
for(QueryJoin queryJoin : joinsContext.getQueryJoins())
{
joinTableNames.add(queryJoin.getJoinTable());
}
setQueryStatJoinTables(joinTableNames);
}
}
PreparedStatement statement = createStatement(connection, sql.toString(), queryInput);
QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
{
@ -352,26 +334,4 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
return (statement);
}
/*******************************************************************************
** Getter for queryStat
*******************************************************************************/
@Override
public QueryStat getQueryStat()
{
return (this.queryStat);
}
/*******************************************************************************
** Setter for queryStat
*******************************************************************************/
@Override
public void setQueryStat(QueryStat queryStat)
{
this.queryStat = queryStat;
}
}