QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 12:40:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #119 from Kingsrook/feature/CE-1460-export-and-join-bugs

Browse Source 
Feature/ce 1460 export and join bugs
This commit is contained in:
Tim Chamberlain
2024-07-18 13:39:43 -05:00
committed by GitHub
parent d094208a74 ccce1a3d1f
commit 61ec57af02
29 changed files with 1451 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java
View File

@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -78,6 +79,12 @@ public class PermissionsHelper
warnAboutPermissionSubTypeForTables(permissionSubType);
QTableMetaData table = QContext.getQInstance().getTable(tableName);
if(table == null)
{
LOG.info("Throwing a permission denied exception in response to a non-existent table name", logPair("tableName", tableName));
throw (new QPermissionDeniedException("Permission denied."));
}
commonCheckPermissionThrowing(getEffectivePermissionRules(table, QContext.getQInstance()), permissionSubType, table.getName());
}
@ -184,7 +191,14 @@ public class PermissionsHelper
*******************************************************************************/
public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName, Map<String, Serializable> processValues) throws QPermissionDeniedException
{
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
QProcessMetaData process = QContext.getQInstance().getProcess(processName);
if(process == null)
{
LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("processName", processName));
throw (new QPermissionDeniedException("Permission denied."));
}
QPermissionRules effectivePermissionRules = getEffectivePermissionRules(process, QContext.getQInstance());
if(effectivePermissionRules.getCustomPermissionChecker() != null)
@ -226,6 +240,13 @@ public class PermissionsHelper
public static void checkAppPermissionThrowing(AbstractActionInput actionInput, String appName) throws QPermissionDeniedException
{
QAppMetaData app = QContext.getQInstance().getApp(appName);
if(app == null)
{
LOG.info("Throwing a permission denied exception in response to a non-existent app name", logPair("appName", appName));
throw (new QPermissionDeniedException("Permission denied."));
}
commonCheckPermissionThrowing(getEffectivePermissionRules(app, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, app.getName());
}
@ -255,6 +276,13 @@ public class PermissionsHelper
public static void checkReportPermissionThrowing(AbstractActionInput actionInput, String reportName) throws QPermissionDeniedException
{
QReportMetaData report = QContext.getQInstance().getReport(reportName);
if(report == null)
{
LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("reportName", reportName));
throw (new QPermissionDeniedException("Permission denied."));
}
commonCheckPermissionThrowing(getEffectivePermissionRules(report, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, report.getName());
}
@ -284,6 +312,13 @@ public class PermissionsHelper
public static void checkWidgetPermissionThrowing(AbstractActionInput actionInput, String widgetName) throws QPermissionDeniedException
{
QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName);
if(widget == null)
{
LOG.info("Throwing a permission denied exception in response to a non-existent widget name", logPair("widgetName", widgetName));
throw (new QPermissionDeniedException("Permission denied."));
}
commonCheckPermissionThrowing(getEffectivePermissionRules(widget, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, widget.getName());
}

202
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java Normal file
View File

@ -0,0 +1,202 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Utility for verifying that the ExportAction works for all tables, and all
** exposed joins.
**
** Meant for use within a unit test, or maybe as part of an instance's boot-up/
** validation.
*******************************************************************************/
public class ExportsFullInstanceVerifier
{
private static final QLogger LOG = QLogger.getLogger(ExportsFullInstanceVerifier.class);
private boolean filterForAtMostOneRowPerExport = true;
/*******************************************************************************
**
*******************************************************************************/
public void verify(Collection<QTableMetaData> tables) throws QException
{
Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
for(QTableMetaData table : tables)
{
if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY))
{
LOG.info("Verifying Exports on table", logPair("tableName", table.getName()));
//////////////////////////////////////////////
// run the table by itself (no join fields) //
//////////////////////////////////////////////
runExport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions);
///////////////////////////////////////////////////
// run once w/ the fields from each exposed join //
///////////////////////////////////////////////////
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
runExport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions);
}
/////////////////////////////////////////////////
// run w/ all exposed joins (if there are any) //
/////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(table.getExposedJoins()))
{
runExport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions);
}
}
}
//////////////////////////////////
// log out an exceptions caught //
//////////////////////////////////
if(!caughtExceptions.isEmpty())
{
for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
{
LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
}
throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void runExport(String tableName, List<ExposedJoin> exposedJoinList, String description, Map<Pair<String, String>, Exception> caughtExceptions)
{
try
{
////////////////////////////////////////////////////////////////////////////////////
// build the list of fieldNames to export - starting with all fields in the table //
////////////////////////////////////////////////////////////////////////////////////
List<String> fieldNames = new ArrayList<>();
for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values())
{
fieldNames.add(field.getName());
}
///////////////////////////////////////////////////
// add all fields from all exposed joins as well //
///////////////////////////////////////////////////
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList))
{
QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
for(QFieldMetaData field : joinTable.getFields().values())
{
fieldNames.add(joinTable.getName() + "." + field.getName());
}
}
LOG.info("Verifying export", logPair("description", description), logPair("fieldCount", fieldNames.size()));
QQueryFilter queryFilter = new QQueryFilter();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(filterForAtMostOneRowPerExport)
{
queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1);
}
ExportInput exportInput = new ExportInput();
exportInput.setTableName(tableName);
exportInput.setFieldNames(fieldNames);
exportInput.setReportDestination(new ReportDestination()
.withReportOutputStream(new ByteArrayOutputStream())
.withReportFormat(ReportFormat.CSV));
exportInput.setQueryFilter(queryFilter);
new ExportAction().execute(exportInput);
}
catch(QException e)
{
caughtExceptions.put(Pair.of(tableName, description), e);
}
}
/*******************************************************************************
** Getter for filterForAtMostOneRowPerExport
*******************************************************************************/
public boolean getFilterForAtMostOneRowPerExport()
{
return (this.filterForAtMostOneRowPerExport);
}
/*******************************************************************************
** Setter for filterForAtMostOneRowPerExport
*******************************************************************************/
public void setFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport)
{
this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport;
}
/*******************************************************************************
** Fluent setter for filterForAtMostOneRowPerExport
*******************************************************************************/
public ExportsFullInstanceVerifier withFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport)
{
this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport;
return (this);
}
}

66
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java
View File

@ -66,6 +66,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
@ -303,7 +304,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
{
if(StringUtils.hasContent(dataSource.getSourceTable()))
{
joinsContext = new JoinsContext(exportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), dataSource.getQueryFilter());
joinsContext = new JoinsContext(QContext.getQInstance(), dataSource.getSourceTable(), cloneDataSourceQueryJoins(dataSource), dataSource.getQueryFilter() == null ? null : dataSource.getQueryFilter().clone());
countDataSourceRecords(reportInput, dataSource, reportFormat);
}
}
@ -351,7 +352,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
CountInput countInput = new CountInput();
countInput.setTableName(dataSource.getSourceTable());
countInput.setFilter(queryFilter);
countInput.setQueryJoins(dataSource.getQueryJoins());
countInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
CountOutput countOutput = new CountAction().execute(countInput);
if(countOutput.getCount() != null)
@ -369,6 +370,26 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
/*******************************************************************************
**
*******************************************************************************/
private static List<QueryJoin> cloneDataSourceQueryJoins(QReportDataSource dataSource)
{
if(dataSource == null || dataSource.getQueryJoins() == null)
{
return (null);
}
List<QueryJoin> rs = new ArrayList<>();
for(QueryJoin queryJoin : dataSource.getQueryJoins())
{
rs.add(queryJoin.clone());
}
return (rs);
}
/*******************************************************************************
**
*******************************************************************************/
@ -417,12 +438,12 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
queryInput.setRecordPipe(recordPipe);
queryInput.setTableName(dataSource.getSourceTable());
queryInput.setFilter(queryFilter);
queryInput.setQueryJoins(dataSource.getQueryJoins());
queryInput.setQueryJoins(cloneDataSourceQueryJoins(dataSource));
queryInput.withQueryHint(QueryHint.POTENTIALLY_LARGE_NUMBER_OF_RESULTS);
queryInput.withQueryHint(QueryHint.MAY_USE_READ_ONLY_BACKEND);
queryInput.setShouldTranslatePossibleValues(true);
queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource, new JoinsContext(reportInput.getInstance(), dataSource.getSourceTable(), dataSource.getQueryJoins(), queryInput.getFilter())));
queryInput.setFieldsToTranslatePossibleValues(setupFieldsToTranslatePossibleValues(reportInput, dataSource));
if(dataSource.getQueryInputCustomizer() != null)
{
@ -474,7 +495,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
}
consumedCount.getAndAdd(records.size());
return (consumeRecords(reportInput, dataSource, records, tableView, summaryViews, variantViews));
return (consumeRecords(dataSource, records, tableView, summaryViews, variantViews));
});
////////////////////////////////////////////////
@ -493,7 +514,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
/*******************************************************************************
**
*******************************************************************************/
private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException
private Set<String> setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource) throws QException
{
Set<String> fieldsToTranslatePossibleValues = new HashSet<>();
@ -574,9 +595,9 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
/*******************************************************************************
**
*******************************************************************************/
private Integer consumeRecords(ReportInput reportInput, QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
private Integer consumeRecords(QReportDataSource dataSource, List<QRecord> records, QReportView tableView, List<QReportView> summaryViews, List<QReportView> variantViews) throws QException
{
QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable());
////////////////////////////////////////////////////////////////////////////
// if this record goes on a table view, add it to the report streamer now //
@ -687,7 +708,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
/*******************************************************************************
**
*******************************************************************************/
private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key) throws QException
private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map<SummaryKey, Map<String, AggregatesInterface<?, ?>>> viewAggregates, SummaryKey key)
{
Map<String, AggregatesInterface<?, ?>> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>());
addRecordToAggregatesMap(table, record, keyAggregates);
@ -698,7 +719,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
/*******************************************************************************
**
*******************************************************************************/
private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap) throws QException
private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map<String, AggregatesInterface<?, ?>> aggregatesMap)
{
//////////////////////////////////////////////////////////////////////////////////////
// todo - an optimization could be, to only compute aggregates that we'll need... //
@ -706,7 +727,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
//////////////////////////////////////////////////////////////////////////////////////
for(String fieldName : record.getValues().keySet())
{
QFieldMetaData field = null;
QFieldMetaData field;
try
{
//////////////////////////////////////////////////////
@ -779,9 +800,14 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
List<QReportView> reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList();
for(QReportView view : reportViews)
{
QReportDataSource dataSource = getDataSource(view.getDataSourceName());
QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable());
SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table);
QReportDataSource dataSource = getDataSource(view.getDataSourceName());
if(dataSource == null)
{
throw new QReportingException("Data source for summary view was not found (viewName=" + view.getName() + ", dataSourceName=" + view.getDataSourceName() + ").");
}
QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable());
SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table);
ExportInput exportInput = new ExportInput();
exportInput.setReportDestination(reportInput.getReportDestination());
@ -867,9 +893,8 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
/*******************************************************************************
**
*******************************************************************************/
private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QReportingException, QFormulaException
private SummaryOutput computeSummaryRowsForView(ReportInput reportInput, QReportView view, QTableMetaData table) throws QFormulaException
{
QValueFormatter valueFormatter = new QValueFormatter();
QMetaDataVariableInterpreter variableInterpreter = new QMetaDataVariableInterpreter();
variableInterpreter.addValueMap("input", reportInput.getInputValues());
variableInterpreter.addValueMap("total", getSummaryValuesForInterpreter(totalAggregates));
@ -941,10 +966,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
//////////////////////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(view.getOrderByFields()))
{
summaryRows.sort((o1, o2) ->
{
return summaryRowComparator(view, o1, o2);
});
summaryRows.sort((o1, o2) -> summaryRowComparator(view, o1, o2));
}
////////////////
@ -979,8 +1001,6 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
Serializable serializable = getValueForColumn(variableInterpreter, column);
totalRow.setValue(column.getName(), serializable);
thisRowValues.put(column.getName(), serializable);
String formatted = valueFormatter.formatValue(column.getDisplayFormat(), serializable);
}
}
@ -1003,7 +1023,7 @@ public class GenerateReportAction extends AbstractQActionFunction<ReportInput, R
titleValues.add(variableInterpreter.interpret(titleField));
}
title = new QValueFormatter().formatStringWithValues(view.getTitleFormat(), titleValues);
title = QValueFormatter.formatStringWithValues(view.getTitleFormat(), titleValues);
}
else if(StringUtils.hasContent(view.getTitleFormat()))
{

30
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java
View File

@ -280,6 +280,16 @@ public class QLogger
/*******************************************************************************
**
*******************************************************************************/
public void debug(LogPair... logPairs)
{
logger.warn(() -> makeJsonString(null, null, logPairs));
}
/*******************************************************************************
**
*******************************************************************************/
@ -420,6 +430,16 @@ public class QLogger
/*******************************************************************************
**
*******************************************************************************/
public void warn(LogPair... logPairs)
{
logger.warn(() -> makeJsonString(null, null, logPairs));
}
/*******************************************************************************
**
*******************************************************************************/
@ -480,6 +500,16 @@ public class QLogger
/*******************************************************************************
**
*******************************************************************************/
public void error(LogPair... logPairs)
{
logger.warn(() -> makeJsonString(null, null, logPairs));
}
/*******************************************************************************
**
*******************************************************************************/

9
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java
View File

@ -378,7 +378,7 @@ public class JoinsContext
{
securityFieldTableAlias = matchedQueryJoin.getJoinTableOrItsAlias();
}
tmpTable = instance.getTable(securityFieldTableAlias);
tmpTable = instance.getTable(aliasToTableNameMap.getOrDefault(securityFieldTableAlias, securityFieldTableAlias));
////////////////////////////////////////////////////////////////////////////////////////
// set the baseTableOrAlias for the next iteration to be this join's joinTableOrAlias //
@ -466,8 +466,8 @@ public class JoinsContext
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
// check if the key type has an all-access key, and if so, if it's set to true for the current user/session //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
boolean haveAllAccessKey = false;
QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType());
boolean haveAllAccessKey = false;
if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName()))
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -1118,7 +1118,7 @@ public class JoinsContext
if(useExposedJoins)
{
QTableMetaData mainTable = QContext.getQInstance().getTable(mainTableName);
for(ExposedJoin exposedJoin : mainTable.getExposedJoins())
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins()))
{
if(exposedJoin.getJoinTable().equals(joinTableName))
{
@ -1159,6 +1159,7 @@ public class JoinsContext
}
/*******************************************************************************
**
*******************************************************************************/

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -237,6 +238,28 @@ public class QQueryFilter implements Serializable, Cloneable
/*******************************************************************************
** fluent method to add a new criteria
*******************************************************************************/
public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Collection<? extends Serializable> values)
{
addCriteria(new QFilterCriteria(fieldName, operator, values));
return (this);
}
/*******************************************************************************
** fluent method to add a new criteria
*******************************************************************************/
public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Serializable... values)
{
addCriteria(new QFilterCriteria(fieldName, operator, values));
return (this);
}
/*******************************************************************************
**
*******************************************************************************/

36
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java
View File

@ -56,7 +56,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
** JoinsContext is constructed before executing a query, and not meant to be set
** by users.
*******************************************************************************/
public class QueryJoin
public class QueryJoin implements Cloneable
{
private String baseTableOrAlias;
private String joinTable;
@ -69,6 +69,40 @@ public class QueryJoin
/*******************************************************************************
**
*******************************************************************************/
@Override
public QueryJoin clone()
{
try
{
QueryJoin clone = (QueryJoin) super.clone();
if(joinMetaData != null)
{
clone.joinMetaData = joinMetaData.clone();
}
if(securityCriteria != null)
{
clone.securityCriteria = new ArrayList<>();
for(QFilterCriteria securityCriterion : securityCriteria)
{
clone.securityCriteria.add(securityCriterion.clone());
}
}
return clone;
}
catch(CloneNotSupportedException e)
{
throw new AssertionError();
}
}
/*******************************************************************************
** define the types of joins - INNER, LEFT, RIGHT, or FULL.
*******************************************************************************/

20
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java
View File

@ -26,7 +26,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.joins;
** Specification for (at least part of) how two tables join together - e.g.,
** leftField = rightField. Used as part of a list in a QJoinMetaData.
*******************************************************************************/
public class JoinOn
public class JoinOn implements Cloneable
{
private String leftField;
private String rightField;
@ -131,4 +131,22 @@ public class JoinOn
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public JoinOn clone()
{
try
{
JoinOn clone = (JoinOn) super.clone();
return clone;
}
catch(CloneNotSupportedException e)
{
throw new AssertionError();
}
}
}

40
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java
View File

@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Definition of how 2 tables join together within a QQQ Instance.
*******************************************************************************/
public class QJoinMetaData implements TopLevelMetaDataInterface
public class QJoinMetaData implements TopLevelMetaDataInterface, Cloneable
{
private String name;
private JoinType type;
@ -62,6 +62,44 @@ public class QJoinMetaData implements TopLevelMetaDataInterface
/*******************************************************************************
**
*******************************************************************************/
@Override
public QJoinMetaData clone()
{
try
{
QJoinMetaData clone = (QJoinMetaData) super.clone();
if(joinOns != null)
{
clone.joinOns = new ArrayList<>();
for(JoinOn joinOn : joinOns)
{
clone.joinOns.add(joinOn.clone());
}
}
if(orderBys != null)
{
clone.orderBys = new ArrayList<>();
for(QFilterOrderBy orderBy : orderBys)
{
clone.orderBys.add(orderBy.clone());
}
}
return clone;
}
catch(CloneNotSupportedException e)
{
throw new AssertionError();
}
}
/*******************************************************************************
** Getter for name
**

126
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java Normal file
View File

@ -0,0 +1,126 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ExceptionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Utility for verifying that the ColumnStats process works for all fields,
** on all tables, and all exposed joins.
**
** Meant for use within a unit test, or maybe as part of an instance's boot-up/
** validation.
*******************************************************************************/
public class ColumnStatsFullInstanceVerifier
{
private static final QLogger LOG = QLogger.getLogger(ColumnStatsFullInstanceVerifier.class);
/*******************************************************************************
**
*******************************************************************************/
public void verify(Collection<QTableMetaData> tables) throws QException
{
Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
for(QTableMetaData table : tables)
{
if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.QUERY_STATS))
{
LOG.info("Verifying ColumnStats on table", logPair("tableName", table.getName()));
for(QFieldMetaData field : table.getFields().values())
{
runColumnStats(table.getName(), field.getName(), caughtExceptions);
}
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
for(QFieldMetaData field : joinTable.getFields().values())
{
runColumnStats(table.getName(), joinTable.getName() + "." + field.getName(), caughtExceptions);
}
}
}
}
// log out an exceptions caught
if(!caughtExceptions.isEmpty())
{
for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
{
LOG.info("Caught an exception verifying column stats", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
}
throw (new QException("Column Status Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void runColumnStats(String tableName, String fieldName, Map<Pair<String, String>, Exception> caughtExceptions) throws QException
{
try
{
RunBackendStepInput input = new RunBackendStepInput();
input.addValue("tableName", tableName);
input.addValue("fieldName", fieldName);
RunBackendStepOutput output = new RunBackendStepOutput();
new ColumnStatsStep().run(input, output);
}
catch(QException e)
{
Throwable rootException = ExceptionUtils.getRootException(e);
if(rootException instanceof QException && rootException.getMessage().contains("not supported for this field's data type"))
{
////////////////////////////////////////////////
// ignore this exception, it's kinda expected //
////////////////////////////////////////////////
LOG.debug("Caught an expected-exception in column stats", e, logPair("tableName", tableName), logPair("fieldName", fieldName));
}
else
{
caughtExceptions.put(Pair.of(tableName, fieldName), e);
}
}
}
}

77
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java
View File

@ -136,29 +136,11 @@ public class ColumnStatsStep implements BackendStep
filter = new QQueryFilter();
}
QueryJoin queryJoin = null;
QTableMetaData table = QContext.getQInstance().getTable(tableName);
QFieldMetaData field = null;
if(fieldName.contains("."))
{
String[] parts = fieldName.split("\\.", 2);
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
if(exposedJoin.getJoinTable().equals(parts[0]))
{
field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]);
queryJoin = new QueryJoin()
.withJoinTable(exposedJoin.getJoinTable())
.withSelect(true)
.withType(QueryJoin.Type.INNER);
break;
}
}
}
else
{
field = table.getField(fieldName);
}
QTableMetaData table = QContext.getQInstance().getTable(tableName);
FieldAndQueryJoin fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName);
QFieldMetaData field = fieldAndQueryJoin.field();
QueryJoin queryJoin = fieldAndQueryJoin.queryJoin();
if(field == null)
{
@ -213,7 +195,7 @@ public class ColumnStatsStep implements BackendStep
filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false));
filter.withOrderBy(new QFilterOrderByGroupBy(groupBy));
Integer limit = 1000; // too big?
Integer limit = 1000;
AggregateInput aggregateInput = new AggregateInput();
aggregateInput.withAggregate(aggregate);
aggregateInput.withGroupBy(groupBy);
@ -223,7 +205,11 @@ public class ColumnStatsStep implements BackendStep
if(queryJoin != null)
{
aggregateInput.withQueryJoin(queryJoin);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// re-construct this queryJoin object - just because, the JoinsContext edits the previous one, so we can make some failing-joins otherwise... //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName);
aggregateInput.withQueryJoin(fieldAndQueryJoin.queryJoin());
}
AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput);
@ -238,7 +224,7 @@ public class ColumnStatsStep implements BackendStep
value = Instant.parse(value + ":00:00Z");
}
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate));
valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count));
}
@ -526,4 +512,43 @@ public class ColumnStatsStep implements BackendStep
return (null);
}
/*******************************************************************************
**
*******************************************************************************/
private FieldAndQueryJoin getFieldAndQueryJoin(QTableMetaData table, String fieldName)
{
QFieldMetaData field = null;
QueryJoin queryJoin = null;
if(fieldName.contains("."))
{
String[] parts = fieldName.split("\\.", 2);
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
if(exposedJoin.getJoinTable().equals(parts[0]))
{
field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]);
queryJoin = new QueryJoin()
.withJoinTable(exposedJoin.getJoinTable())
.withSelect(true)
.withType(QueryJoin.Type.INNER);
break;
}
}
}
else
{
field = table.getField(fieldName);
}
return (new FieldAndQueryJoin(field, queryJoin));
}
/*******************************************************************************
**
*******************************************************************************/
private record FieldAndQueryJoin(QFieldMetaData field, QueryJoin queryJoin) {}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java
View File

@ -129,6 +129,7 @@ public class RenderSavedReportExecuteStep implements BackendStep
.withRenderedReportStatusId(RenderedReportStatus.RUNNING.getId())
.withReportFormat(ReportFormatPossibleValueEnum.valueOf(reportFormat.name()).getPossibleValueId())
)).getRecords().get(0);
runBackendStepOutput.addValue("renderedReportId", renderedReportRecord.getValue("id"));
////////////////////////////////////////////////////////////////////////////////////////////
// convert the report record to report meta-data, which the GenerateReportAction works on //

258
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java Normal file
View File

@ -0,0 +1,258 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport;
import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Utility for verifying that the RenderReports process works for all fields,
** on all tables, and all exposed joins.
**
** Meant for use within a unit test, or maybe as part of an instance's boot-up/
** validation.
*******************************************************************************/
public class ReportsFullInstanceVerifier
{
private static final QLogger LOG = QLogger.getLogger(ReportsFullInstanceVerifier.class);
private boolean removeRenderedReports = true;
private boolean filterForAtMostOneRowPerReport = true;
/*******************************************************************************
**
*******************************************************************************/
public void verify(Collection<QTableMetaData> tables, String storageTableName) throws QException
{
Map<Pair<String, String>, Exception> caughtExceptions = new LinkedHashMap<>();
for(QTableMetaData table : tables)
{
if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY))
{
LOG.info("Verifying Reports on table", logPair("tableName", table.getName()));
//////////////////////////////////////////////
// run the table by itself (no join fields) //
//////////////////////////////////////////////
runReport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions, storageTableName);
///////////////////////////////////////////////////
// run once w/ the fields from each exposed join //
///////////////////////////////////////////////////
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins()))
{
runReport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions, storageTableName);
}
/////////////////////////////////////////////////
// run w/ all exposed joins (if there are any) //
/////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(table.getExposedJoins()))
{
runReport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions, storageTableName);
}
}
}
//////////////////////////////////
// log out an exceptions caught //
//////////////////////////////////
if(!caughtExceptions.isEmpty())
{
for(Map.Entry<Pair<String, String>, Exception> entry : caughtExceptions.entrySet())
{
LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB()));
}
throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void runReport(String tableName, List<ExposedJoin> exposedJoinList, String description, Map<Pair<String, String>, Exception> caughtExceptions, String storageTableName)
{
try
{
////////////////////////////////////////////////////////////////////////////////////////////////
// build the list of reports to include in the column - starting with all fields in the table //
////////////////////////////////////////////////////////////////////////////////////////////////
ReportColumns reportColumns = new ReportColumns();
for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values())
{
reportColumns.withColumn(field.getName());
}
///////////////////////////////////////////////////
// add all fields from all exposed joins as well //
///////////////////////////////////////////////////
for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList))
{
QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable());
for(QFieldMetaData field : joinTable.getFields().values())
{
reportColumns.withColumn(joinTable.getName() + "." + field.getName());
}
}
QQueryFilter queryFilter = new QQueryFilter();
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(filterForAtMostOneRowPerReport)
{
queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1);
}
//////////////////////////////////
// insert a saved report record //
//////////////////////////////////
SavedReport savedReport = new SavedReport();
savedReport.setTableName(tableName);
savedReport.setLabel("Test " + tableName + " " + description);
savedReport.setColumnsJson(JsonUtils.toJson(reportColumns));
savedReport.setQueryFilterJson(JsonUtils.toJson(queryFilter));
List<QRecord> reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords();
///////////////////////
// render the report //
///////////////////////
RunBackendStepInput input = new RunBackendStepInput();
RunBackendStepOutput output = new RunBackendStepOutput();
input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.CSV.name());
input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName);
input.setRecords(reportRecordList);
new RenderSavedReportExecuteStep().run(input, output);
//////////////////////////////////////////
// clean up the report, if so requested //
//////////////////////////////////////////
if(removeRenderedReports)
{
new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId")));
}
}
catch(QException e)
{
caughtExceptions.put(Pair.of(tableName, description), e);
}
}
/*******************************************************************************
** Getter for removeRenderedReports
*******************************************************************************/
public boolean getRemoveRenderedReports()
{
return (this.removeRenderedReports);
}
/*******************************************************************************
** Setter for removeRenderedReports
*******************************************************************************/
public void setRemoveRenderedReports(boolean removeRenderedReports)
{
this.removeRenderedReports = removeRenderedReports;
}
/*******************************************************************************
** Fluent setter for removeRenderedReports
*******************************************************************************/
public ReportsFullInstanceVerifier withRemoveRenderedReports(boolean removeRenderedReports)
{
this.removeRenderedReports = removeRenderedReports;
return (this);
}
/*******************************************************************************
** Getter for filterForAtMostOneRowPerReport
*******************************************************************************/
public boolean getFilterForAtMostOneRowPerReport()
{
return (this.filterForAtMostOneRowPerReport);
}
/*******************************************************************************
** Setter for filterForAtMostOneRowPerReport
*******************************************************************************/
public void setFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport)
{
this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport;
}
/*******************************************************************************
** Fluent setter for filterForAtMostOneRowPerReport
*******************************************************************************/
public ReportsFullInstanceVerifier withFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport)
{
this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport;
return (this);
}
}

151
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java Normal file
View File

@ -0,0 +1,151 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
** Utility for verifying that the RenderReports process works for all report
** records stored in the saved reports table.
**
** Meant for use within a unit test, or maybe as part of an instance's boot-up/
** validation.
*******************************************************************************/
public class SavedReportsTableFullVerifier
{
private static final QLogger LOG = QLogger.getLogger(SavedReportsTableFullVerifier.class);
private boolean removeRenderedReports = true;
/*******************************************************************************
**
*******************************************************************************/
public void verify(List<QRecord> savedReportRecordList, String storageTableName) throws QException
{
Map<Integer, Exception> caughtExceptions = new LinkedHashMap<>();
for(QRecord report : savedReportRecordList)
{
runReport(report, caughtExceptions, storageTableName);
}
//////////////////////////////////
// log out an exceptions caught //
//////////////////////////////////
if(!caughtExceptions.isEmpty())
{
for(Map.Entry<Integer, Exception> entry : caughtExceptions.entrySet())
{
LOG.info("Caught an exception verifying saved reports", entry.getValue(), logPair("savdReportId", entry.getKey()));
}
throw (new QException("Saved Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size())));
}
}
/*******************************************************************************
**
*******************************************************************************/
private void runReport(QRecord savedReport, Map<Integer, Exception> caughtExceptions, String storageTableName)
{
try
{
///////////////////////
// render the report //
///////////////////////
RunBackendStepInput input = new RunBackendStepInput();
RunBackendStepOutput output = new RunBackendStepOutput();
input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.XLSX.name());
input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName);
input.setRecords(List.of(savedReport));
new RenderSavedReportExecuteStep().run(input, output);
Exception exception = output.getException();
if(exception != null)
{
throw (exception);
}
//////////////////////////////////////////
// clean up the report, if so requested //
//////////////////////////////////////////
if(removeRenderedReports)
{
new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId")));
}
}
catch(Exception e)
{
caughtExceptions.put(savedReport.getValueInteger("id"), e);
}
}
/*******************************************************************************
** Getter for removeRenderedReports
*******************************************************************************/
public boolean getRemoveRenderedReports()
{
return (this.removeRenderedReports);
}
/*******************************************************************************
** Setter for removeRenderedReports
*******************************************************************************/
public void setRemoveRenderedReports(boolean removeRenderedReports)
{
this.removeRenderedReports = removeRenderedReports;
}
/*******************************************************************************
** Fluent setter for removeRenderedReports
*******************************************************************************/
public SavedReportsTableFullVerifier withRemoveRenderedReports(boolean removeRenderedReports)
{
this.removeRenderedReports = removeRenderedReports;
return (this);
}
}

46
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java Normal file
View File

@ -0,0 +1,46 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.reporting;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for ExportsFullInstanceVerifier
*******************************************************************************/
class ExportsFullInstanceVerifierTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
new ExportsFullInstanceVerifier().verify(QContext.getQInstance().getTables().values());
}
}

48
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java Normal file
View File

@ -0,0 +1,48 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.columnstats;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for ColumnStatsFullInstanceVerifier
*******************************************************************************/
class ColumnStatsFullInstanceVerifierTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
new ColumnStatsFullInstanceVerifier().verify(List.of(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)));
}
}

61
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java Normal file
View File

@ -0,0 +1,61 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for ReportsFullInstanceVerifier
*******************************************************************************/
class ReportsFullInstanceVerifierTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void beforeEach() throws Exception
{
new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
new ReportsFullInstanceVerifier().verify(List.of(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)), SavedReportsMetaDataProvider.REPORT_STORAGE_TABLE_NAME);
}
}

84
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.savedreports;
import java.util.List;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport;
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for SavedReportsTableFullVerifier
*******************************************************************************/
class SavedReportsTableFullVerifierTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void beforeEach() throws Exception
{
new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null);
GenerateReportActionTest.insertPersonRecords(QContext.getQInstance());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
ReportColumns reportColumns = new ReportColumns();
reportColumns.withColumn("id");
//////////////////////////////////
// insert a saved report record //
//////////////////////////////////
SavedReport savedReport = new SavedReport();
savedReport.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
savedReport.setLabel("Test");
savedReport.setColumnsJson(JsonUtils.toJson(reportColumns));
savedReport.setQueryFilterJson(JsonUtils.toJson(new QQueryFilter()));
List<QRecord> reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords();
SavedReportsTableFullVerifier savedReportsTableFullVerifier = new SavedReportsTableFullVerifier();
savedReportsTableFullVerifier.verify(reportRecordList, SavedReportsMetaDataProvider.REPORT_STORAGE_TABLE_NAME);
}
}

55
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -245,31 +245,64 @@ public abstract class AbstractRDBMSAction
*******************************************************************************/
protected String makeFromClause(QInstance instance, String tableName, JoinsContext joinsContext, List<Serializable> params)
{
//////////////////////////////////////////////////////////////////////
// start with the main table - un-aliased (well, aliased as itself) //
//////////////////////////////////////////////////////////////////////
StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName));
////////////////////////////////////////////////////////////////////////////////////////////////////////
// sort the query joins from the main table "outward"... //
// this might not be perfect, e.g., for cases where what we actually might need is a tree of joins... //
////////////////////////////////////////////////////////////////////////////////////////////////////////
List<QueryJoin> queryJoins = sortQueryJoinsForFromClause(tableName, joinsContext.getQueryJoins());
////////////////////////////////////////////////////////
// iterate over joins, adding to the from clause (rs) //
////////////////////////////////////////////////////////
for(QueryJoin queryJoin : queryJoins)
{
QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable());
String joinTableNameOrAlias = queryJoin.getJoinTableOrItsAlias();
////////////////////////////////////////////////////////
// add the `<type> JOIN table AS alias` bit to the rs //
////////////////////////////////////////////////////////
rs.append(" ").append(queryJoin.getType()).append(" JOIN ")
.append(escapeIdentifier(getTableName(joinTable)))
.append(" AS ").append(escapeIdentifier(tableNameOrAlias));
.append(" AS ").append(escapeIdentifier(joinTableNameOrAlias));
////////////////////////////////////////////////////////////
// find the join in the instance, to set the 'on' clause //
////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// find the join in the instance, for building the ON clause //
// append each sub-clause (condition) into a list, for later joining with AND //
////////////////////////////////////////////////////////////////////////////////
List<String> joinClauseList = new ArrayList<>();
String baseTableName = Objects.requireNonNullElse(joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), tableName);
QJoinMetaData joinMetaData = Objects.requireNonNull(queryJoin.getJoinMetaData(), () -> "Could not find a join between tables [" + baseTableName + "][" + queryJoin.getJoinTable() + "]");
//////////////////////////////////////////////////
// loop over join-ons (e.g., multi-column join) //
//////////////////////////////////////////////////
for(JoinOn joinOn : joinMetaData.getJoinOns())
{
////////////////////////////////////////////////////////////////////////////////////////////////////////
// figure out if the join needs flipped. We want its left table to equal the queryJoin's base table. //
////////////////////////////////////////////////////////////////////////////////////////////////////////
QTableMetaData leftTable = instance.getTable(joinMetaData.getLeftTable());
QTableMetaData rightTable = instance.getTable(joinMetaData.getRightTable());
if(!joinMetaData.getLeftTable().equals(baseTableName))
{
joinOn = joinOn.flip();
QTableMetaData tmpTable = leftTable;
leftTable = rightTable;
rightTable = tmpTable;
}
////////////////////////////////////////////////////////////
// get the table-names-or-aliases to use in the ON clause //
////////////////////////////////////////////////////////////
String baseTableOrAlias = queryJoin.getBaseTableOrAlias();
String joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
if(baseTableOrAlias == null)
{
baseTableOrAlias = leftTable.getName();
@ -279,15 +312,6 @@ public abstract class AbstractRDBMSAction
}
}
String joinTableOrAlias = queryJoin.getJoinTableOrItsAlias();
if(!joinMetaData.getLeftTable().equals(baseTableName))
{
joinOn = joinOn.flip();
QTableMetaData tmpTable = leftTable;
leftTable = rightTable;
rightTable = tmpTable;
}
joinClauseList.add(escapeIdentifier(baseTableOrAlias)
+ "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField())))
+ " = " + escapeIdentifier(joinTableOrAlias)
@ -938,6 +962,7 @@ public abstract class AbstractRDBMSAction
{
try
{
params = Objects.requireNonNullElse(params, Collections.emptyList());
params = params.size() <= 100 ? params : params.subList(0, 99);
/////////////////////////////////////////////////////////////////////////////

8
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java
View File

@ -116,7 +116,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
actionTimeoutHelper = new ActionTimeoutHelper(aggregateInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql));
actionTimeoutHelper.start();
QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) ->
{
/////////////////////////////////////////////////////////////////////////
// once we've started getting results, go ahead and cancel the timeout //
@ -168,8 +168,10 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
}), params);
}
logSQL(sql, params, mark);
finally
{
logSQL(sql, params, mark);
}
return rs;
}

10
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java
View File

@ -84,10 +84,10 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
setSqlAndJoinsInQueryStat(sql, joinsContext);
CountOutput rs = new CountOutput();
long mark = System.currentTimeMillis();
try(Connection connection = getConnection(countInput))
{
long mark = System.currentTimeMillis();
statement = connection.prepareStatement(sql);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -96,7 +96,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
actionTimeoutHelper = new ActionTimeoutHelper(countInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql));
actionTimeoutHelper.start();
QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) ->
{
/////////////////////////////////////////////////////////////////////////
// once we've started getting results, go ahead and cancel the timeout //
@ -116,7 +116,9 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
setQueryStatFirstResultTime();
}), params);
}
finally
{
logSQL(sql, params, mark);
}

24
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java
View File

@ -212,13 +212,16 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
// LOG.debug("rowCount 0 trying to delete [" + tableName + "][" + primaryKey + "]");
// deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError("Record was not deleted (but no error was given from the database)"));
// }
logSQL(sql, List.of(primaryKey), mark);
}
catch(Exception e)
{
LOG.debug("Exception trying to delete [" + tableName + "][" + primaryKey + "]", e);
deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError(new SystemErrorStatusMessage("Record was not deleted: " + e.getMessage())));
}
finally
{
logSQL(sql, List.of(primaryKey), mark);
}
}
@ -228,13 +231,14 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
*******************************************************************************/
public void doDeleteList(Connection connection, QTableMetaData table, List<Serializable> primaryKeys, DeleteOutput deleteOutput) throws QException
{
long mark = System.currentTimeMillis();
String sql = null;
try
{
long mark = System.currentTimeMillis();
String tableName = getTableName(table);
String primaryKeyName = getColumnName(table.getField(table.getPrimaryKeyField()));
String sql = "DELETE FROM "
sql = "DELETE FROM "
+ escapeIdentifier(tableName)
+ " WHERE "
+ escapeIdentifier(primaryKeyName)
@ -246,13 +250,15 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys);
deleteOutput.addToDeletedRecordCount(rowCount);
logSQL(sql, primaryKeys, mark);
}
catch(Exception e)
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
finally
{
logSQL(sql, primaryKeys, mark);
}
}
@ -282,12 +288,14 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
{
int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params);
deleteOutput.setDeletedRecordCount(rowCount);
logSQL(sql, params, mark);
}
catch(Exception e)
{
throw new QException("Error executing delete with filter: " + e.getMessage(), e);
}
finally
{
logSQL(sql, params, mark);
}
}
}

17
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -57,9 +57,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
InsertOutput rs = new InsertOutput();
QTableMetaData table = insertInput.getTable();
Connection connection = null;
Connection connection = null;
boolean needToCloseConnection = false;
StringBuilder sql = null;
List<Object> params = null;
Long mark = null;
try
{
List<QFieldMetaData> insertableFields = table.getFields().values().stream()
@ -88,10 +92,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
for(List<QRecord> page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE))
{
String tableName = escapeIdentifier(getTableName(table));
StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES");
List<Object> params = new ArrayList<>();
int recordIndex = 0;
String tableName = escapeIdentifier(getTableName(table));
sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES");
params = new ArrayList<>();
int recordIndex = 0;
//////////////////////////////////////////////////////
// for each record in the page: //
@ -133,7 +137,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
continue;
}
Long mark = System.currentTimeMillis();
mark = System.currentTimeMillis();
///////////////////////////////////////////////////////////
// execute the insert, then foreach record in the input, //
@ -163,6 +167,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
}
catch(Exception e)
{
logSQL(sql, params, mark);
throw new QException("Error executing insert: " + e.getMessage(), e);
}
finally

16
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -170,7 +170,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
//////////////////////////////////////////////
QueryOutput queryOutput = new QueryOutput(queryInput);
QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) ->
{
/////////////////////////////////////////////////////////////////////////
// once we've started getting results, go ahead and cancel the timeout //
@ -223,17 +223,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
}), params);
logSQL(sql, params, mark);
return queryOutput;
}
catch(Exception e)
{
logSQL(sql, params, mark);
throw (e);
}
finally
{
logSQL(sql, params, mark);
if(actionTimeoutHelper != null)
{
/////////////////////////////////////////
@ -366,10 +361,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
{
RDBMSBackendMetaData rdbmsBackendMetaData = (RDBMSBackendMetaData) queryInput.getBackend();
////////////////////////////////////////////////////////////////////////////
// todo - remove "aurora" - it's a legacy value here for a staged rollout //
////////////////////////////////////////////////////////////////////////////
if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || "aurora".equals(rdbmsBackendMetaData.getVendor()))
if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor()))
{
//////////////////////////////////////////////////////////////////////////////////////////////////////
// mysql "optimization", presumably here - from Result Set section of //

26
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java
View File

@ -179,10 +179,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
////////////////////////////////////////////////////////////////////////////////
// let query manager do the batch updates - note that it will internally page //
////////////////////////////////////////////////////////////////////////////////
QueryManager.executeBatchUpdate(connection, sql, values);
incrementStatus(updateInput, recordList.size());
logSQL(sql, values, mark);
try
{
QueryManager.executeBatchUpdate(connection, sql, values);
incrementStatus(updateInput, recordList.size());
}
finally
{
logSQL(sql, values, mark);
}
}
@ -249,10 +254,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
/////////////////////////////////////
// let query manager do the update //
/////////////////////////////////////
QueryManager.executeUpdate(connection, sql, params);
incrementStatus(updateInput, page.size());
logSQL(sql, params, mark);
try
{
QueryManager.executeUpdate(connection, sql, params);
incrementStatus(updateInput, page.size());
}
finally
{
logSQL(sql, params, mark);
}
}
}

10
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java
View File

@ -150,10 +150,7 @@ public class ConnectionManager
return switch(backend.getVendor())
{
////////////////////////////////////////////////////////////////////////////
// todo - remove "aurora" - it's a legacy value here for a staged rollout //
////////////////////////////////////////////////////////////////////////////
case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "com.mysql.cj.jdbc.Driver";
case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "com.mysql.cj.jdbc.Driver";
case RDBMSBackendMetaData.VENDOR_H2 -> "org.h2.Driver";
default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data."));
};
@ -178,10 +175,7 @@ public class ConnectionManager
////////////////////////////////////////////////////////////////
// jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL";
////////////////////////////////////////////////////////////////////////////
// todo - remove "aurora" - it's a legacy value here for a staged rollout //
////////////////////////////////////////////////////////////////////////////
case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false";
case RDBMSBackendMetaData.VENDOR_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull";
case RDBMSBackendMetaData.VENDOR_H2 -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1";
default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor());

73
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java
View File

@ -32,6 +32,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.time.Instant;
@ -56,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValu
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import org.apache.commons.lang.NotImplementedException;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
/*******************************************************************************
@ -121,6 +123,17 @@ public class QueryManager
** customized settings/optimizations).
*******************************************************************************/
public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException, QException
{
executeStatement(statement, null, processor, params);
}
/*******************************************************************************
** Let the caller provide their own prepared statement (e.g., possibly with some
** customized settings/optimizations).
*******************************************************************************/
public static void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException
{
ResultSet resultSet = null;
@ -136,6 +149,14 @@ public class QueryManager
processor.processResultSet(resultSet);
}
}
catch(SQLException e)
{
if(sql != null)
{
LOG.warn("SQLException", e, logPair("sql", sql));
}
throw (e);
}
finally
{
if(resultSet != null)
@ -372,10 +393,17 @@ public class QueryManager
*******************************************************************************/
public static PreparedStatement executeUpdate(Connection connection, String sql, Object... params) throws SQLException
{
PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params);
incrementStatistic(STAT_QUERIES_RAN);
statement.executeUpdate();
return (statement);
try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params))
{
incrementStatistic(STAT_QUERIES_RAN);
statement.executeUpdate();
return (statement);
}
catch(SQLException e)
{
LOG.warn("SQLException", e, logPair("sql", sql));
throw (e);
}
}
@ -385,10 +413,17 @@ public class QueryManager
*******************************************************************************/
public static PreparedStatement executeUpdate(Connection connection, String sql, List<Object> params) throws SQLException
{
PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params);
incrementStatistic(STAT_QUERIES_RAN);
statement.executeUpdate();
return (statement);
try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params))
{
incrementStatistic(STAT_QUERIES_RAN);
statement.executeUpdate();
return (statement);
}
catch(SQLException e)
{
LOG.warn("SQLException", e, logPair("sql", sql));
throw (e);
}
}
@ -436,6 +471,11 @@ public class QueryManager
statement.executeUpdate();
return (statement.getUpdateCount());
}
catch(SQLException e)
{
LOG.warn("SQLException", e, logPair("sql", sql));
throw (e);
}
}
@ -488,19 +528,24 @@ public class QueryManager
*******************************************************************************/
public static List<Integer> executeInsertForGeneratedIds(Connection connection, String sql, List<Object> params) throws SQLException
{
List<Integer> rs = new ArrayList<>();
try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS))
{
bindParams(params.toArray(), statement);
incrementStatistic(STAT_QUERIES_RAN);
statement.executeUpdate();
ResultSet generatedKeys = statement.getGeneratedKeys();
ResultSet generatedKeys = statement.getGeneratedKeys();
List<Integer> rs = new ArrayList<>();
while(generatedKeys.next())
{
rs.add(getInteger(generatedKeys, 1));
}
return (rs);
}
catch(SQLException e)
{
LOG.warn("SQLException", e, logPair("sql", sql));
throw (e);
}
return (rs);
}
@ -747,14 +792,14 @@ public class QueryManager
else if(value instanceof LocalDate ld)
{
@SuppressWarnings("deprecation")
java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth());
Date date = new Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth());
statement.setDate(index, date);
return (1);
}
else if(value instanceof LocalTime lt)
{
@SuppressWarnings("deprecation")
java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond());
Time time = new Time(lt.getHour(), lt.getMinute(), lt.getSecond());
statement.setTime(index, time);
return (1);
}
@ -943,7 +988,7 @@ public class QueryManager
}
else
{
statement.setDate(index, new java.sql.Date(value.getTime()));
statement.setDate(index, new Date(value.getTime()));
}
}

7
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
View File

@ -273,6 +273,7 @@ public class TestUtils
.withJoinNameChain(List.of("orderInstructionsJoinOrder")))
.withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id"))
.withField(new QFieldMetaData("instructions", QFieldType.STRING))
.withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderInstructionsJoinOrder")))
);
qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item")
@ -395,10 +396,10 @@ public class TestUtils
qInstance.addJoin(new QJoinMetaData()
.withName("orderInstructionsJoinOrder")
.withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS)
.withRightTable(TABLE_NAME_ORDER)
.withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS)
.withLeftTable(TABLE_NAME_ORDER)
.withType(JoinType.MANY_TO_ONE)
.withJoinOn(new JoinOn("orderId", "id"))
.withJoinOn(new JoinOn("id", "orderId"))
);
qInstance.addPossibleValueSource(new QPossibleValueSource()

33
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java
View File

@ -54,6 +54,7 @@ import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
/*******************************************************************************
@ -69,10 +70,6 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
public void beforeEach() throws Exception
{
super.primeTestDatabase();
AbstractRDBMSAction.setLogSQL(true);
AbstractRDBMSAction.setLogSQLReformat(true);
AbstractRDBMSAction.setLogSQLOutput("system.out");
}
@ -909,7 +906,7 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
**
*******************************************************************************/
@Test
void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException
void testMultipleReversedDirectionJoinsBetweenSameTablesAllAccessKey() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
@ -992,6 +989,32 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest
/*******************************************************************************
** We had, at one time, a bug where, for tables with 2 joins between each other,
** an ON clause could get written using the wrong table name in one part.
**
** With that bug, this QueryAction.execute would throw an SQL Exception.
**
** So this test, just makes sure that no such exception gets thrown.
*******************************************************************************/
@Test
void testFlippedJoinForOnClause() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS);
queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER));
QueryOutput queryOutput = new QueryAction().execute(queryInput);
assertFalse(queryOutput.getRecords().isEmpty());
////////////////////////////////////
// if no exception, then we pass. //
////////////////////////////////////
}
/*******************************************************************************
** Addressing a regression where a table was brought into a query for its
** security field, but it was a write-scope lock, so, it shouldn't have been.