QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feature/query-stats' into feature/query-stats

Browse Source 
# Conflicts:
#	qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/interfaces/QueryInterface.java
#	qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java
#	qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java
This commit is contained in:
darin.kelkhoff
2023-06-16 08:40:04 -05:00
parent a38d57c7af 0b525f8775
commit 4105f034aa
5 changed files with 815 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

397
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/querystats/QueryStat.java Normal file
View File

@ -0,0 +1,397 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers.querystats;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QAssociation;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
/*******************************************************************************
**
*******************************************************************************/
public class QueryStat extends QRecordEntity
{
public static final String TABLE_NAME = "queryStat";
@QField()
private Integer id;
@QField()
private String tableName;
@QField()
private Instant startTimestamp;
@QField()
private Instant firstResultTimestamp;
@QField()
private Integer firstResultMillis;
@QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE)
private String joinTables;
@QField(maxLength = 100, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE)
private String orderBys;
@QAssociation(name = "queryStatFilterCriteria")
private List<QueryStatFilterCriteria> queryStatFilterCriteriaList;
/*******************************************************************************
**
*******************************************************************************/
public void setJoinTables(Collection<String> joinTableNames)
{
if(CollectionUtils.nullSafeIsEmpty(joinTableNames))
{
setJoinTables((String) null);
}
setJoinTables(joinTableNames.stream().sorted().collect(Collectors.joining(",")));
}
/*******************************************************************************
**
*******************************************************************************/
public void setQQueryFilter(QQueryFilter filter)
{
if(filter == null)
{
setQueryStatFilterCriteriaList(null);
setOrderBys(null);
}
else
{
/////////////////////////////////////////////
// manage list of sub-records for criteria //
/////////////////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(filter.getCriteria()) && CollectionUtils.nullSafeIsEmpty(filter.getSubFilters()))
{
setQueryStatFilterCriteriaList(null);
}
else
{
ArrayList<QueryStatFilterCriteria> criteriaList = new ArrayList<>();
setQueryStatFilterCriteriaList(criteriaList);
processFilterCriteria(filter, criteriaList);
}
//////////////////////////////////////////////////////////////
// set orderBys (comma-delimited concatenated string field) //
//////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeIsEmpty(filter.getOrderBys()))
{
setOrderBys(null);
}
else
{
setOrderBys(filter.getOrderBys().stream().map(ob -> ob.getFieldName()).collect(Collectors.joining(",")));
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private static void processFilterCriteria(QQueryFilter filter, ArrayList<QueryStatFilterCriteria> criteriaList)
{
for(QFilterCriteria criterion : CollectionUtils.nonNullList(filter.getCriteria()))
{
criteriaList.add(new QueryStatFilterCriteria()
.withFieldName(criterion.getFieldName())
.withOperator(criterion.getOperator().name())
.withValues(CollectionUtils.nonNullList(criterion.getValues()).stream().map(v -> ValueUtils.getValueAsString(v)).collect(Collectors.joining(","))));
}
for(QQueryFilter subFilter : CollectionUtils.nonNullList(filter.getSubFilters()))
{
processFilterCriteria(subFilter, criteriaList);
}
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
*******************************************************************************/
public QueryStat withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for startTimestamp
*******************************************************************************/
public Instant getStartTimestamp()
{
return (this.startTimestamp);
}
/*******************************************************************************
** Setter for startTimestamp
*******************************************************************************/
public void setStartTimestamp(Instant startTimestamp)
{
this.startTimestamp = startTimestamp;
}
/*******************************************************************************
** Fluent setter for startTimestamp
*******************************************************************************/
public QueryStat withStartTimestamp(Instant startTimestamp)
{
this.startTimestamp = startTimestamp;
return (this);
}
/*******************************************************************************
** Getter for firstResultTimestamp
*******************************************************************************/
public Instant getFirstResultTimestamp()
{
return (this.firstResultTimestamp);
}
/*******************************************************************************
** Setter for firstResultTimestamp
*******************************************************************************/
public void setFirstResultTimestamp(Instant firstResultTimestamp)
{
this.firstResultTimestamp = firstResultTimestamp;
}
/*******************************************************************************
** Fluent setter for firstResultTimestamp
*******************************************************************************/
public QueryStat withFirstResultTimestamp(Instant firstResultTimestamp)
{
this.firstResultTimestamp = firstResultTimestamp;
return (this);
}
/*******************************************************************************
** Getter for firstResultMillis
*******************************************************************************/
public Integer getFirstResultMillis()
{
return (this.firstResultMillis);
}
/*******************************************************************************
** Setter for firstResultMillis
*******************************************************************************/
public void setFirstResultMillis(Integer firstResultMillis)
{
this.firstResultMillis = firstResultMillis;
}
/*******************************************************************************
** Fluent setter for firstResultMillis
*******************************************************************************/
public QueryStat withFirstResultMillis(Integer firstResultMillis)
{
this.firstResultMillis = firstResultMillis;
return (this);
}
/*******************************************************************************
** Getter for joinTables
*******************************************************************************/
public String getJoinTables()
{
return (this.joinTables);
}
/*******************************************************************************
** Setter for joinTables
*******************************************************************************/
public void setJoinTables(String joinTables)
{
this.joinTables = joinTables;
}
/*******************************************************************************
** Fluent setter for joinTables
*******************************************************************************/
public QueryStat withJoinTables(String joinTables)
{
this.joinTables = joinTables;
return (this);
}
/*******************************************************************************
** Getter for queryStatFilterCriteriaList
*******************************************************************************/
public List<QueryStatFilterCriteria> getQueryStatFilterCriteriaList()
{
return (this.queryStatFilterCriteriaList);
}
/*******************************************************************************
** Setter for queryStatFilterCriteriaList
*******************************************************************************/
public void setQueryStatFilterCriteriaList(List<QueryStatFilterCriteria> queryStatFilterCriteriaList)
{
this.queryStatFilterCriteriaList = queryStatFilterCriteriaList;
}
/*******************************************************************************
** Fluent setter for queryStatFilterCriteriaList
*******************************************************************************/
public QueryStat withQueryStatFilterCriteriaList(List<QueryStatFilterCriteria> queryStatFilterCriteriaList)
{
this.queryStatFilterCriteriaList = queryStatFilterCriteriaList;
return (this);
}
/*******************************************************************************
** Getter for orderBys
*******************************************************************************/
public String getOrderBys()
{
return (this.orderBys);
}
/*******************************************************************************
** Setter for orderBys
*******************************************************************************/
public void setOrderBys(String orderBys)
{
this.orderBys = orderBys;
}
/*******************************************************************************
** Fluent setter for orderBys
*******************************************************************************/
public QueryStat withOrderBys(String orderBys)
{
this.orderBys = orderBys;
return (this);
}
/*******************************************************************************
** Getter for id
*******************************************************************************/
public Integer getId()
{
return (this.id);
}
/*******************************************************************************
** Setter for id
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Fluent setter for id
*******************************************************************************/
public QueryStat withId(Integer id)
{
this.id = id;
return (this);
}
}

162
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/querystats/QueryStatFilterCriteria.java Normal file
View File

@ -0,0 +1,162 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers.querystats;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
**
*******************************************************************************/
public class QueryStatFilterCriteria extends QRecordEntity
{
private Integer queryStatId;
private String fieldName;
private String operator;
private String values;
/*******************************************************************************
** Getter for queryStatId
*******************************************************************************/
public Integer getQueryStatId()
{
return (this.queryStatId);
}
/*******************************************************************************
** Setter for queryStatId
*******************************************************************************/
public void setQueryStatId(Integer queryStatId)
{
this.queryStatId = queryStatId;
}
/*******************************************************************************
** Fluent setter for queryStatId
*******************************************************************************/
public QueryStatFilterCriteria withQueryStatId(Integer queryStatId)
{
this.queryStatId = queryStatId;
return (this);
}
/*******************************************************************************
** Getter for fieldName
*******************************************************************************/
public String getFieldName()
{
return (this.fieldName);
}
/*******************************************************************************
** Setter for fieldName
*******************************************************************************/
public void setFieldName(String fieldName)
{
this.fieldName = fieldName;
}
/*******************************************************************************
** Fluent setter for fieldName
*******************************************************************************/
public QueryStatFilterCriteria withFieldName(String fieldName)
{
this.fieldName = fieldName;
return (this);
}
/*******************************************************************************
** Getter for operator
*******************************************************************************/
public String getOperator()
{
return (this.operator);
}
/*******************************************************************************
** Setter for operator
*******************************************************************************/
public void setOperator(String operator)
{
this.operator = operator;
}
/*******************************************************************************
** Fluent setter for operator
*******************************************************************************/
public QueryStatFilterCriteria withOperator(String operator)
{
this.operator = operator;
return (this);
}
/*******************************************************************************
** Getter for values
*******************************************************************************/
public String getValues()
{
return (this.values);
}
/*******************************************************************************
** Setter for values
*******************************************************************************/
public void setValues(String values)
{
this.values = values;
}
/*******************************************************************************
** Fluent setter for values
*******************************************************************************/
public QueryStatFilterCriteria withValues(String values)
{
this.values = values;
return (this);
}
}

217
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/querystats/QueryStatManager.java Normal file
View File

@ -0,0 +1,217 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.tables.helpers.querystats;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
**
*******************************************************************************/
public class QueryStatManager
{
private static QueryStatManager queryStatManager = null;
// todo - support multiple qInstances?
private QInstance qInstance;
private Supplier<QSession> sessionSupplier;
private boolean active = false;
private List<QueryStat> queryStats = new ArrayList<>();
private ScheduledExecutorService executorService;
/*******************************************************************************
** Singleton constructor
*******************************************************************************/
private QueryStatManager()
{
}
/*******************************************************************************
** Singleton accessor
*******************************************************************************/
public static QueryStatManager getInstance()
{
if(queryStatManager == null)
{
queryStatManager = new QueryStatManager();
}
return (queryStatManager);
}
/*******************************************************************************
**
*******************************************************************************/
public void start(Supplier<QSession> sessionSupplier)
{
qInstance = QContext.getQInstance();
this.sessionSupplier = sessionSupplier;
active = true;
queryStats = new ArrayList<>();
executorService = Executors.newSingleThreadScheduledExecutor();
executorService.scheduleAtFixedRate(new QueryStatManagerInsertJob(), 60, 60, TimeUnit.SECONDS);
}
/*******************************************************************************
**
*******************************************************************************/
public void stop()
{
active = false;
queryStats.clear();
if(executorService != null)
{
executorService.shutdown();
executorService = null;
}
}
/*******************************************************************************
**
*******************************************************************************/
public void add(QueryStat queryStat)
{
if(active)
{
synchronized(this)
{
if(queryStat.getFirstResultTimestamp() == null)
{
////////////////////////////////////////////////
// in case it didn't get set in the interface //
////////////////////////////////////////////////
queryStat.setFirstResultTimestamp(Instant.now());
}
///////////////////////////////////////////////
// compute the millis (so you don't have to) //
///////////////////////////////////////////////
if(queryStat.getStartTimestamp() != null && queryStat.getFirstResultTimestamp() != null && queryStat.getFirstResultMillis() == null)
{
long millis = queryStat.getFirstResultTimestamp().toEpochMilli() - queryStat.getStartTimestamp().toEpochMilli();
queryStat.setFirstResultMillis((int) millis);
}
queryStats.add(queryStat);
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private List<QueryStat> getListAndReset()
{
if(queryStats.isEmpty())
{
return Collections.emptyList();
}
synchronized(this)
{
List<QueryStat> returnList = queryStats;
queryStats = new ArrayList<>();
return (returnList);
}
}
/*******************************************************************************
**
*******************************************************************************/
private static class QueryStatManagerInsertJob implements Runnable
{
private static final QLogger LOG = QLogger.getLogger(QueryStatManagerInsertJob.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run()
{
try
{
QContext.init(getInstance().qInstance, getInstance().sessionSupplier.get());
List<QueryStat> list = getInstance().getListAndReset();
LOG.info(logPair("queryStatListSize", list.size()));
if(list.isEmpty())
{
return;
}
try
{
InsertInput insertInput = new InsertInput();
insertInput.setTableName(QueryStat.TABLE_NAME);
insertInput.setRecords(list.stream().map(qs -> qs.toQRecord()).toList());
new InsertAction().execute(insertInput);
}
catch(Exception e)
{
LOG.error("Error inserting query stats", e);
}
}
finally
{
QContext.clear();
}
}
}
}

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java
View File

@ -118,6 +118,24 @@ public abstract class QRecordEntity
qRecordEntityAssociation.getSetter().invoke(this, associatedEntityList); qRecordEntityAssociation.getSetter().invoke(this, associatedEntityList);
} }
} }
for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
{
List<QRecord> associatedRecords = qRecord.getAssociatedRecords().get(qRecordEntityAssociation.getAssociationAnnotation().name());
if(associatedRecords == null)
{
qRecordEntityAssociation.getSetter().invoke(this, (Object) null);
}
else
{
List<QRecordEntity> associatedEntityList = new ArrayList<>();
for(QRecord associatedRecord : CollectionUtils.nonNullList(associatedRecords))
{
associatedEntityList.add(QRecordEntity.fromQRecord(qRecordEntityAssociation.getAssociatedType(), associatedRecord));
}
qRecordEntityAssociation.getSetter().invoke(this, associatedEntityList);
}
}
} }
catch(Exception e) catch(Exception e)
{ {
@ -179,8 +197,7 @@ public abstract class QRecordEntity
{ {
QRecord qRecord = new QRecord(); QRecord qRecord = new QRecord();
List<QRecordEntityField> fieldList = getFieldList(this.getClass()); for(QRecordEntityField qRecordEntityField : getFieldList(this.getClass()))
for(QRecordEntityField qRecordEntityField : fieldList)
{ {
Serializable thisValue = (Serializable) qRecordEntityField.getGetter().invoke(this); Serializable thisValue = (Serializable) qRecordEntityField.getGetter().invoke(this);
Serializable originalValue = null; Serializable originalValue = null;
@ -195,6 +212,25 @@ public abstract class QRecordEntity
} }
} }
for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass()))
{
List<? extends QRecordEntity> associatedEntities = (List<? extends QRecordEntity>) qRecordEntityAssociation.getGetter().invoke(this);
String associationName = qRecordEntityAssociation.getAssociationAnnotation().name();
if(associatedEntities != null)
{
/////////////////////////////////////////////////////////////////////////////////
// do this so an empty list in the entity becomes an empty list in the QRecord //
/////////////////////////////////////////////////////////////////////////////////
qRecord.withAssociatedRecords(associationName, new ArrayList<>());
}
for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities))
{
qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord());
}
}
return (qRecord); return (qRecord);
} }
catch(Exception e) catch(Exception e)

1
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -37,6 +37,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
import com.kingsrook.qqq.backend.core.actions.tables.helpers.querystats.QueryStat;
import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;