diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java index ab1b56ca..82af5ed7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/permissions/PermissionsHelper.java @@ -49,6 +49,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -78,6 +79,12 @@ public class PermissionsHelper warnAboutPermissionSubTypeForTables(permissionSubType); QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent table name", logPair("tableName", tableName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(table, QContext.getQInstance()), permissionSubType, table.getName()); } @@ -184,7 +191,14 @@ public class PermissionsHelper *******************************************************************************/ public static void checkProcessPermissionThrowing(AbstractActionInput actionInput, String processName, Map processValues) throws QPermissionDeniedException { - QProcessMetaData process = QContext.getQInstance().getProcess(processName); + QProcessMetaData process = QContext.getQInstance().getProcess(processName); + + if(process == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("processName", processName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + QPermissionRules effectivePermissionRules = getEffectivePermissionRules(process, QContext.getQInstance()); if(effectivePermissionRules.getCustomPermissionChecker() != null) @@ -226,6 +240,13 @@ public class PermissionsHelper public static void checkAppPermissionThrowing(AbstractActionInput actionInput, String appName) throws QPermissionDeniedException { QAppMetaData app = QContext.getQInstance().getApp(appName); + + if(app == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent app name", logPair("appName", appName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(app, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, app.getName()); } @@ -255,6 +276,13 @@ public class PermissionsHelper public static void checkReportPermissionThrowing(AbstractActionInput actionInput, String reportName) throws QPermissionDeniedException { QReportMetaData report = QContext.getQInstance().getReport(reportName); + + if(report == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent process name", logPair("reportName", reportName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(report, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, report.getName()); } @@ -284,6 +312,13 @@ public class PermissionsHelper public static void checkWidgetPermissionThrowing(AbstractActionInput actionInput, String widgetName) throws QPermissionDeniedException { QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); + + if(widget == null) + { + LOG.info("Throwing a permission denied exception in response to a non-existent widget name", logPair("widgetName", widgetName)); + throw (new QPermissionDeniedException("Permission denied.")); + } + commonCheckPermissionThrowing(getEffectivePermissionRules(widget, QContext.getQInstance()), PrivatePermissionSubType.HAS_ACCESS, widget.getName()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java new file mode 100644 index 00000000..bd4c3dfa --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifier.java @@ -0,0 +1,202 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the ExportAction works for all tables, and all + ** exposed joins. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class ExportsFullInstanceVerifier +{ + private static final QLogger LOG = QLogger.getLogger(ExportsFullInstanceVerifier.class); + + private boolean filterForAtMostOneRowPerExport = true; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(Collection tables) throws QException + { + Map, Exception> caughtExceptions = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY)) + { + LOG.info("Verifying Exports on table", logPair("tableName", table.getName())); + + ////////////////////////////////////////////// + // run the table by itself (no join fields) // + ////////////////////////////////////////////// + runExport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions); + + /////////////////////////////////////////////////// + // run once w/ the fields from each exposed join // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + runExport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions); + } + + ///////////////////////////////////////////////// + // run w/ all exposed joins (if there are any) // + ///////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(table.getExposedJoins())) + { + runExport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions); + } + } + } + + ////////////////////////////////// + // log out an exceptions caught // + ////////////////////////////////// + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry, Exception> entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB())); + } + throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runExport(String tableName, List exposedJoinList, String description, Map, Exception> caughtExceptions) + { + try + { + //////////////////////////////////////////////////////////////////////////////////// + // build the list of fieldNames to export - starting with all fields in the table // + //////////////////////////////////////////////////////////////////////////////////// + List fieldNames = new ArrayList<>(); + for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values()) + { + fieldNames.add(field.getName()); + } + + /////////////////////////////////////////////////// + // add all fields from all exposed joins as well // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList)) + { + QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + fieldNames.add(joinTable.getName() + "." + field.getName()); + } + } + + LOG.info("Verifying export", logPair("description", description), logPair("fieldCount", fieldNames.size())); + + QQueryFilter queryFilter = new QQueryFilter(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(filterForAtMostOneRowPerExport) + { + queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1); + } + + ExportInput exportInput = new ExportInput(); + exportInput.setTableName(tableName); + exportInput.setFieldNames(fieldNames); + exportInput.setReportDestination(new ReportDestination() + .withReportOutputStream(new ByteArrayOutputStream()) + .withReportFormat(ReportFormat.CSV)); + exportInput.setQueryFilter(queryFilter); + new ExportAction().execute(exportInput); + } + catch(QException e) + { + caughtExceptions.put(Pair.of(tableName, description), e); + } + } + + + + /******************************************************************************* + ** Getter for filterForAtMostOneRowPerExport + *******************************************************************************/ + public boolean getFilterForAtMostOneRowPerExport() + { + return (this.filterForAtMostOneRowPerExport); + } + + + + /******************************************************************************* + ** Setter for filterForAtMostOneRowPerExport + *******************************************************************************/ + public void setFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport) + { + this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport; + } + + + + /******************************************************************************* + ** Fluent setter for filterForAtMostOneRowPerExport + *******************************************************************************/ + public ExportsFullInstanceVerifier withFilterForAtMostOneRowPerExport(boolean filterForAtMostOneRowPerExport) + { + this.filterForAtMostOneRowPerExport = filterForAtMostOneRowPerExport; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java index a984bed3..a216e9e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportAction.java @@ -66,6 +66,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -303,7 +304,7 @@ public class GenerateReportAction extends AbstractQActionFunction cloneDataSourceQueryJoins(QReportDataSource dataSource) + { + if(dataSource == null || dataSource.getQueryJoins() == null) + { + return (null); + } + + List rs = new ArrayList<>(); + for(QueryJoin queryJoin : dataSource.getQueryJoins()) + { + rs.add(queryJoin.clone()); + } + return (rs); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -417,12 +438,12 @@ public class GenerateReportAction extends AbstractQActionFunction setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource, JoinsContext joinsContext) throws QException + private Set setupFieldsToTranslatePossibleValues(ReportInput reportInput, QReportDataSource dataSource) throws QException { Set fieldsToTranslatePossibleValues = new HashSet<>(); @@ -574,9 +595,9 @@ public class GenerateReportAction extends AbstractQActionFunction records, QReportView tableView, List summaryViews, List variantViews) throws QException + private Integer consumeRecords(QReportDataSource dataSource, List records, QReportView tableView, List summaryViews, List variantViews) throws QException { - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); + QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable()); //////////////////////////////////////////////////////////////////////////// // if this record goes on a table view, add it to the report streamer now // @@ -687,7 +708,7 @@ public class GenerateReportAction extends AbstractQActionFunction>> viewAggregates, SummaryKey key) throws QException + private void addRecordToSummaryKeyAggregates(QTableMetaData table, QRecord record, Map>> viewAggregates, SummaryKey key) { Map> keyAggregates = viewAggregates.computeIfAbsent(key, (name) -> new HashMap<>()); addRecordToAggregatesMap(table, record, keyAggregates); @@ -698,7 +719,7 @@ public class GenerateReportAction extends AbstractQActionFunction> aggregatesMap) throws QException + private void addRecordToAggregatesMap(QTableMetaData table, QRecord record, Map> aggregatesMap) { ////////////////////////////////////////////////////////////////////////////////////// // todo - an optimization could be, to only compute aggregates that we'll need... // @@ -706,7 +727,7 @@ public class GenerateReportAction extends AbstractQActionFunction reportViews = views.stream().filter(v -> v.getType().equals(ReportType.SUMMARY)).toList(); for(QReportView view : reportViews) { - QReportDataSource dataSource = getDataSource(view.getDataSourceName()); - QTableMetaData table = reportInput.getInstance().getTable(dataSource.getSourceTable()); - SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table); + QReportDataSource dataSource = getDataSource(view.getDataSourceName()); + if(dataSource == null) + { + throw new QReportingException("Data source for summary view was not found (viewName=" + view.getName() + ", dataSourceName=" + view.getDataSourceName() + ")."); + } + + QTableMetaData table = QContext.getQInstance().getTable(dataSource.getSourceTable()); + SummaryOutput summaryOutput = computeSummaryRowsForView(reportInput, view, table); ExportInput exportInput = new ExportInput(); exportInput.setReportDestination(reportInput.getReportDestination()); @@ -867,9 +893,8 @@ public class GenerateReportAction extends AbstractQActionFunction - { - return summaryRowComparator(view, o1, o2); - }); + summaryRows.sort((o1, o2) -> summaryRowComparator(view, o1, o2)); } //////////////// @@ -979,8 +1001,6 @@ public class GenerateReportAction extends AbstractQActionFunction makeJsonString(null, null, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -420,6 +430,16 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public void warn(LogPair... logPairs) + { + logger.warn(() -> makeJsonString(null, null, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -480,6 +500,16 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public void error(LogPair... logPairs) + { + logger.warn(() -> makeJsonString(null, null, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java index 26767cce..7f2adde6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java @@ -378,7 +378,7 @@ public class JoinsContext { securityFieldTableAlias = matchedQueryJoin.getJoinTableOrItsAlias(); } - tmpTable = instance.getTable(securityFieldTableAlias); + tmpTable = instance.getTable(aliasToTableNameMap.getOrDefault(securityFieldTableAlias, securityFieldTableAlias)); //////////////////////////////////////////////////////////////////////////////////////// // set the baseTableOrAlias for the next iteration to be this join's joinTableOrAlias // @@ -466,8 +466,8 @@ public class JoinsContext ////////////////////////////////////////////////////////////////////////////////////////////////////////////// // check if the key type has an all-access key, and if so, if it's set to true for the current user/session // ////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); - boolean haveAllAccessKey = false; + QSecurityKeyType securityKeyType = instance.getSecurityKeyType(recordSecurityLock.getSecurityKeyType()); + boolean haveAllAccessKey = false; if(StringUtils.hasContent(securityKeyType.getAllAccessKeyName())) { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1118,7 +1118,7 @@ public class JoinsContext if(useExposedJoins) { QTableMetaData mainTable = QContext.getQInstance().getTable(mainTableName); - for(ExposedJoin exposedJoin : mainTable.getExposedJoins()) + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(mainTable.getExposedJoins())) { if(exposedJoin.getJoinTable().equals(joinTableName)) { @@ -1159,6 +1159,7 @@ public class JoinsContext } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java index dba1a93e..6de05ddc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QQueryFilter.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.query; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -237,6 +238,28 @@ public class QQueryFilter implements Serializable, Cloneable + /******************************************************************************* + ** fluent method to add a new criteria + *******************************************************************************/ + public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Collection values) + { + addCriteria(new QFilterCriteria(fieldName, operator, values)); + return (this); + } + + + + /******************************************************************************* + ** fluent method to add a new criteria + *******************************************************************************/ + public QQueryFilter withCriteria(String fieldName, QCriteriaOperator operator, Serializable... values) + { + addCriteria(new QFilterCriteria(fieldName, operator, values)); + return (this); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java index b55fbea2..d7a433cc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryJoin.java @@ -56,7 +56,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** JoinsContext is constructed before executing a query, and not meant to be set ** by users. *******************************************************************************/ -public class QueryJoin +public class QueryJoin implements Cloneable { private String baseTableOrAlias; private String joinTable; @@ -69,6 +69,40 @@ public class QueryJoin + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryJoin clone() + { + try + { + QueryJoin clone = (QueryJoin) super.clone(); + + if(joinMetaData != null) + { + clone.joinMetaData = joinMetaData.clone(); + } + + if(securityCriteria != null) + { + clone.securityCriteria = new ArrayList<>(); + for(QFilterCriteria securityCriterion : securityCriteria) + { + clone.securityCriteria.add(securityCriterion.clone()); + } + } + + return clone; + } + catch(CloneNotSupportedException e) + { + throw new AssertionError(); + } + } + + + /******************************************************************************* ** define the types of joins - INNER, LEFT, RIGHT, or FULL. *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java index 0dc8ed98..78666893 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/JoinOn.java @@ -26,7 +26,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.joins; ** Specification for (at least part of) how two tables join together - e.g., ** leftField = rightField. Used as part of a list in a QJoinMetaData. *******************************************************************************/ -public class JoinOn +public class JoinOn implements Cloneable { private String leftField; private String rightField; @@ -131,4 +131,22 @@ public class JoinOn return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public JoinOn clone() + { + try + { + JoinOn clone = (JoinOn) super.clone(); + return clone; + } + catch(CloneNotSupportedException e) + { + throw new AssertionError(); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java index 1f35d165..51c8dd30 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/joins/QJoinMetaData.java @@ -33,7 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* ** Definition of how 2 tables join together within a QQQ Instance. *******************************************************************************/ -public class QJoinMetaData implements TopLevelMetaDataInterface +public class QJoinMetaData implements TopLevelMetaDataInterface, Cloneable { private String name; private JoinType type; @@ -62,6 +62,44 @@ public class QJoinMetaData implements TopLevelMetaDataInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QJoinMetaData clone() + { + try + { + QJoinMetaData clone = (QJoinMetaData) super.clone(); + + if(joinOns != null) + { + clone.joinOns = new ArrayList<>(); + for(JoinOn joinOn : joinOns) + { + clone.joinOns.add(joinOn.clone()); + } + } + + if(orderBys != null) + { + clone.orderBys = new ArrayList<>(); + for(QFilterOrderBy orderBy : orderBys) + { + clone.orderBys.add(orderBy.clone()); + } + } + + return clone; + } + catch(CloneNotSupportedException e) + { + throw new AssertionError(); + } + } + + + /******************************************************************************* ** Getter for name ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java new file mode 100644 index 00000000..8e21c3a6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifier.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; + + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the ColumnStats process works for all fields, + ** on all tables, and all exposed joins. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class ColumnStatsFullInstanceVerifier +{ + private static final QLogger LOG = QLogger.getLogger(ColumnStatsFullInstanceVerifier.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(Collection tables) throws QException + { + Map, Exception> caughtExceptions = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.QUERY_STATS)) + { + LOG.info("Verifying ColumnStats on table", logPair("tableName", table.getName())); + for(QFieldMetaData field : table.getFields().values()) + { + runColumnStats(table.getName(), field.getName(), caughtExceptions); + } + + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + runColumnStats(table.getName(), joinTable.getName() + "." + field.getName(), caughtExceptions); + } + } + } + } + + // log out an exceptions caught + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry, Exception> entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying column stats", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB())); + } + throw (new QException("Column Status Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runColumnStats(String tableName, String fieldName, Map, Exception> caughtExceptions) throws QException + { + try + { + RunBackendStepInput input = new RunBackendStepInput(); + input.addValue("tableName", tableName); + input.addValue("fieldName", fieldName); + RunBackendStepOutput output = new RunBackendStepOutput(); + new ColumnStatsStep().run(input, output); + } + catch(QException e) + { + Throwable rootException = ExceptionUtils.getRootException(e); + if(rootException instanceof QException && rootException.getMessage().contains("not supported for this field's data type")) + { + //////////////////////////////////////////////// + // ignore this exception, it's kinda expected // + //////////////////////////////////////////////// + LOG.debug("Caught an expected-exception in column stats", e, logPair("tableName", tableName), logPair("fieldName", fieldName)); + } + else + { + caughtExceptions.put(Pair.of(tableName, fieldName), e); + } + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index 21584e8e..cbe1e1a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -136,29 +136,11 @@ public class ColumnStatsStep implements BackendStep filter = new QQueryFilter(); } - QueryJoin queryJoin = null; - QTableMetaData table = QContext.getQInstance().getTable(tableName); - QFieldMetaData field = null; - if(fieldName.contains(".")) - { - String[] parts = fieldName.split("\\.", 2); - for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) - { - if(exposedJoin.getJoinTable().equals(parts[0])) - { - field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]); - queryJoin = new QueryJoin() - .withJoinTable(exposedJoin.getJoinTable()) - .withSelect(true) - .withType(QueryJoin.Type.INNER); - break; - } - } - } - else - { - field = table.getField(fieldName); - } + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + FieldAndQueryJoin fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName); + QFieldMetaData field = fieldAndQueryJoin.field(); + QueryJoin queryJoin = fieldAndQueryJoin.queryJoin(); if(field == null) { @@ -213,7 +195,7 @@ public class ColumnStatsStep implements BackendStep filter.withOrderBy(new QFilterOrderByAggregate(aggregate, false)); filter.withOrderBy(new QFilterOrderByGroupBy(groupBy)); - Integer limit = 1000; // too big? + Integer limit = 1000; AggregateInput aggregateInput = new AggregateInput(); aggregateInput.withAggregate(aggregate); aggregateInput.withGroupBy(groupBy); @@ -223,7 +205,11 @@ public class ColumnStatsStep implements BackendStep if(queryJoin != null) { - aggregateInput.withQueryJoin(queryJoin); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // re-construct this queryJoin object - just because, the JoinsContext edits the previous one, so we can make some failing-joins otherwise... // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + fieldAndQueryJoin = getFieldAndQueryJoin(table, fieldName); + aggregateInput.withQueryJoin(fieldAndQueryJoin.queryJoin()); } AggregateOutput aggregateOutput = new AggregateAction().execute(aggregateInput); @@ -238,7 +224,7 @@ public class ColumnStatsStep implements BackendStep value = Instant.parse(value + ":00:00Z"); } - Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); + Integer count = ValueUtils.getValueAsInteger(result.getAggregateValue(aggregate)); valueCounts.add(new QRecord().withValue(fieldName, value).withValue("count", count)); } @@ -526,4 +512,43 @@ public class ColumnStatsStep implements BackendStep return (null); } + + + /******************************************************************************* + ** + *******************************************************************************/ + private FieldAndQueryJoin getFieldAndQueryJoin(QTableMetaData table, String fieldName) + { + QFieldMetaData field = null; + QueryJoin queryJoin = null; + if(fieldName.contains(".")) + { + String[] parts = fieldName.split("\\.", 2); + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + if(exposedJoin.getJoinTable().equals(parts[0])) + { + field = QContext.getQInstance().getTable(exposedJoin.getJoinTable()).getField(parts[1]); + queryJoin = new QueryJoin() + .withJoinTable(exposedJoin.getJoinTable()) + .withSelect(true) + .withType(QueryJoin.Type.INNER); + break; + } + } + } + else + { + field = table.getField(fieldName); + } + + return (new FieldAndQueryJoin(field, queryJoin)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private record FieldAndQueryJoin(QFieldMetaData field, QueryJoin queryJoin) {} } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java index 506d2857..f91a9000 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -129,6 +129,7 @@ public class RenderSavedReportExecuteStep implements BackendStep .withRenderedReportStatusId(RenderedReportStatus.RUNNING.getId()) .withReportFormat(ReportFormatPossibleValueEnum.valueOf(reportFormat.name()).getPossibleValueId()) )).getRecords().get(0); + runBackendStepOutput.addValue("renderedReportId", renderedReportRecord.getValue("id")); //////////////////////////////////////////////////////////////////////////////////////////// // convert the report record to report meta-data, which the GenerateReportAction works on // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java new file mode 100644 index 00000000..60c2f509 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifier.java @@ -0,0 +1,258 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the RenderReports process works for all fields, + ** on all tables, and all exposed joins. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class ReportsFullInstanceVerifier +{ + private static final QLogger LOG = QLogger.getLogger(ReportsFullInstanceVerifier.class); + + private boolean removeRenderedReports = true; + private boolean filterForAtMostOneRowPerReport = true; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(Collection tables, String storageTableName) throws QException + { + Map, Exception> caughtExceptions = new LinkedHashMap<>(); + for(QTableMetaData table : tables) + { + if(table.isCapabilityEnabled(QContext.getQInstance().getBackendForTable(table.getName()), Capability.TABLE_QUERY)) + { + LOG.info("Verifying Reports on table", logPair("tableName", table.getName())); + + ////////////////////////////////////////////// + // run the table by itself (no join fields) // + ////////////////////////////////////////////// + runReport(table.getName(), Collections.emptyList(), "main-table-only", caughtExceptions, storageTableName); + + /////////////////////////////////////////////////// + // run once w/ the fields from each exposed join // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(table.getExposedJoins())) + { + runReport(table.getName(), List.of(exposedJoin), "join-" + exposedJoin.getLabel(), caughtExceptions, storageTableName); + } + + ///////////////////////////////////////////////// + // run w/ all exposed joins (if there are any) // + ///////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(table.getExposedJoins())) + { + runReport(table.getName(), table.getExposedJoins(), "all-joins", caughtExceptions, storageTableName); + } + } + } + + ////////////////////////////////// + // log out an exceptions caught // + ////////////////////////////////// + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry, Exception> entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying reports", entry.getValue(), logPair("tableName", entry.getKey().getA()), logPair("fieldName", entry.getKey().getB())); + } + throw (new QException("Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runReport(String tableName, List exposedJoinList, String description, Map, Exception> caughtExceptions, String storageTableName) + { + try + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // build the list of reports to include in the column - starting with all fields in the table // + //////////////////////////////////////////////////////////////////////////////////////////////// + ReportColumns reportColumns = new ReportColumns(); + for(QFieldMetaData field : QContext.getQInstance().getTable(tableName).getFields().values()) + { + reportColumns.withColumn(field.getName()); + } + + /////////////////////////////////////////////////// + // add all fields from all exposed joins as well // + /////////////////////////////////////////////////// + for(ExposedJoin exposedJoin : CollectionUtils.nonNullList(exposedJoinList)) + { + QTableMetaData joinTable = QContext.getQInstance().getTable(exposedJoin.getJoinTable()); + for(QFieldMetaData field : joinTable.getFields().values()) + { + reportColumns.withColumn(joinTable.getName() + "." + field.getName()); + } + } + + QQueryFilter queryFilter = new QQueryFilter(); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if caller is okay with a filter that should limit the report to a small number of rows (could be more than 1 for to-many joins), then do so // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(filterForAtMostOneRowPerReport) + { + queryFilter.withCriteria(QContext.getQInstance().getTable(tableName).getPrimaryKeyField(), QCriteriaOperator.EQUALS, 1); + } + + ////////////////////////////////// + // insert a saved report record // + ////////////////////////////////// + SavedReport savedReport = new SavedReport(); + savedReport.setTableName(tableName); + savedReport.setLabel("Test " + tableName + " " + description); + savedReport.setColumnsJson(JsonUtils.toJson(reportColumns)); + savedReport.setQueryFilterJson(JsonUtils.toJson(queryFilter)); + List reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords(); + + /////////////////////// + // render the report // + /////////////////////// + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.CSV.name()); + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName); + input.setRecords(reportRecordList); + + new RenderSavedReportExecuteStep().run(input, output); + + ////////////////////////////////////////// + // clean up the report, if so requested // + ////////////////////////////////////////// + if(removeRenderedReports) + { + new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId"))); + } + } + catch(QException e) + { + caughtExceptions.put(Pair.of(tableName, description), e); + } + } + + /******************************************************************************* + ** Getter for removeRenderedReports + *******************************************************************************/ + public boolean getRemoveRenderedReports() + { + return (this.removeRenderedReports); + } + + + + /******************************************************************************* + ** Setter for removeRenderedReports + *******************************************************************************/ + public void setRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + } + + + + /******************************************************************************* + ** Fluent setter for removeRenderedReports + *******************************************************************************/ + public ReportsFullInstanceVerifier withRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + return (this); + } + + + + /******************************************************************************* + ** Getter for filterForAtMostOneRowPerReport + *******************************************************************************/ + public boolean getFilterForAtMostOneRowPerReport() + { + return (this.filterForAtMostOneRowPerReport); + } + + + + /******************************************************************************* + ** Setter for filterForAtMostOneRowPerReport + *******************************************************************************/ + public void setFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport) + { + this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport; + } + + + + /******************************************************************************* + ** Fluent setter for filterForAtMostOneRowPerReport + *******************************************************************************/ + public ReportsFullInstanceVerifier withFilterForAtMostOneRowPerReport(boolean filterForAtMostOneRowPerReport) + { + this.filterForAtMostOneRowPerReport = filterForAtMostOneRowPerReport; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java new file mode 100644 index 00000000..d6e7e687 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifier.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.RenderedReport; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Utility for verifying that the RenderReports process works for all report + ** records stored in the saved reports table. + ** + ** Meant for use within a unit test, or maybe as part of an instance's boot-up/ + ** validation. + *******************************************************************************/ +public class SavedReportsTableFullVerifier +{ + private static final QLogger LOG = QLogger.getLogger(SavedReportsTableFullVerifier.class); + + private boolean removeRenderedReports = true; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void verify(List savedReportRecordList, String storageTableName) throws QException + { + Map caughtExceptions = new LinkedHashMap<>(); + for(QRecord report : savedReportRecordList) + { + runReport(report, caughtExceptions, storageTableName); + } + + ////////////////////////////////// + // log out an exceptions caught // + ////////////////////////////////// + if(!caughtExceptions.isEmpty()) + { + for(Map.Entry entry : caughtExceptions.entrySet()) + { + LOG.info("Caught an exception verifying saved reports", entry.getValue(), logPair("savdReportId", entry.getKey())); + } + throw (new QException("Saved Reports Verification failed with " + caughtExceptions.size() + " exception" + StringUtils.plural(caughtExceptions.size()))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runReport(QRecord savedReport, Map caughtExceptions, String storageTableName) + { + try + { + /////////////////////// + // render the report // + /////////////////////// + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT, ReportFormat.XLSX.name()); + input.addValue(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME, storageTableName); + input.setRecords(List.of(savedReport)); + + new RenderSavedReportExecuteStep().run(input, output); + Exception exception = output.getException(); + if(exception != null) + { + throw (exception); + } + + ////////////////////////////////////////// + // clean up the report, if so requested // + ////////////////////////////////////////// + if(removeRenderedReports) + { + new DeleteAction().execute(new DeleteInput(RenderedReport.TABLE_NAME).withPrimaryKey(output.getValue("renderedReportId"))); + } + } + catch(Exception e) + { + caughtExceptions.put(savedReport.getValueInteger("id"), e); + } + } + + + + /******************************************************************************* + ** Getter for removeRenderedReports + *******************************************************************************/ + public boolean getRemoveRenderedReports() + { + return (this.removeRenderedReports); + } + + + + /******************************************************************************* + ** Setter for removeRenderedReports + *******************************************************************************/ + public void setRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + } + + + + /******************************************************************************* + ** Fluent setter for removeRenderedReports + *******************************************************************************/ + public SavedReportsTableFullVerifier withRemoveRenderedReports(boolean removeRenderedReports) + { + this.removeRenderedReports = removeRenderedReports; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java new file mode 100644 index 00000000..bf3e28b7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportsFullInstanceVerifierTest.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.reporting; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ExportsFullInstanceVerifier + *******************************************************************************/ +class ExportsFullInstanceVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new ExportsFullInstanceVerifier().verify(QContext.getQInstance().getTables().values()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java new file mode 100644 index 00000000..b2476f9b --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsFullInstanceVerifierTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.columnstats; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ColumnStatsFullInstanceVerifier + *******************************************************************************/ +class ColumnStatsFullInstanceVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new ColumnStatsFullInstanceVerifier().verify(List.of(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY))); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java new file mode 100644 index 00000000..a3730f28 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/ReportsFullInstanceVerifierTest.java @@ -0,0 +1,61 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ReportsFullInstanceVerifier + *******************************************************************************/ +class ReportsFullInstanceVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + new ReportsFullInstanceVerifier().verify(List.of(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)), SavedReportsMetaDataProvider.REPORT_STORAGE_TABLE_NAME); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java new file mode 100644 index 00000000..431bf0a6 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/SavedReportsTableFullVerifierTest.java @@ -0,0 +1,84 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedreports; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedreports.ReportColumns; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReport; +import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for SavedReportsTableFullVerifier + *******************************************************************************/ +class SavedReportsTableFullVerifierTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws Exception + { + new SavedReportsMetaDataProvider().defineAll(QContext.getQInstance(), TestUtils.MEMORY_BACKEND_NAME, TestUtils.MEMORY_BACKEND_NAME, null); + GenerateReportActionTest.insertPersonRecords(QContext.getQInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ReportColumns reportColumns = new ReportColumns(); + reportColumns.withColumn("id"); + + ////////////////////////////////// + // insert a saved report record // + ////////////////////////////////// + SavedReport savedReport = new SavedReport(); + savedReport.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + savedReport.setLabel("Test"); + savedReport.setColumnsJson(JsonUtils.toJson(reportColumns)); + savedReport.setQueryFilterJson(JsonUtils.toJson(new QQueryFilter())); + List reportRecordList = new InsertAction().execute(new InsertInput(SavedReport.TABLE_NAME).withRecordEntity(savedReport)).getRecords(); + + SavedReportsTableFullVerifier savedReportsTableFullVerifier = new SavedReportsTableFullVerifier(); + savedReportsTableFullVerifier.verify(reportRecordList, SavedReportsMetaDataProvider.REPORT_STORAGE_TABLE_NAME); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 082759bc..f9e12361 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -245,31 +245,64 @@ public abstract class AbstractRDBMSAction *******************************************************************************/ protected String makeFromClause(QInstance instance, String tableName, JoinsContext joinsContext, List params) { + ////////////////////////////////////////////////////////////////////// + // start with the main table - un-aliased (well, aliased as itself) // + ////////////////////////////////////////////////////////////////////// StringBuilder rs = new StringBuilder(escapeIdentifier(getTableName(instance.getTable(tableName))) + " AS " + escapeIdentifier(tableName)); + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // sort the query joins from the main table "outward"... // + // this might not be perfect, e.g., for cases where what we actually might need is a tree of joins... // + //////////////////////////////////////////////////////////////////////////////////////////////////////// List queryJoins = sortQueryJoinsForFromClause(tableName, joinsContext.getQueryJoins()); + + //////////////////////////////////////////////////////// + // iterate over joins, adding to the from clause (rs) // + //////////////////////////////////////////////////////// for(QueryJoin queryJoin : queryJoins) { - QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); - String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); + QTableMetaData joinTable = instance.getTable(queryJoin.getJoinTable()); + String joinTableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); + //////////////////////////////////////////////////////// + // add the ` JOIN table AS alias` bit to the rs // + //////////////////////////////////////////////////////// rs.append(" ").append(queryJoin.getType()).append(" JOIN ") .append(escapeIdentifier(getTableName(joinTable))) - .append(" AS ").append(escapeIdentifier(tableNameOrAlias)); + .append(" AS ").append(escapeIdentifier(joinTableNameOrAlias)); - //////////////////////////////////////////////////////////// - // find the join in the instance, to set the 'on' clause // - //////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + // find the join in the instance, for building the ON clause // + // append each sub-clause (condition) into a list, for later joining with AND // + //////////////////////////////////////////////////////////////////////////////// List joinClauseList = new ArrayList<>(); String baseTableName = Objects.requireNonNullElse(joinsContext.resolveTableNameOrAliasToTableName(queryJoin.getBaseTableOrAlias()), tableName); QJoinMetaData joinMetaData = Objects.requireNonNull(queryJoin.getJoinMetaData(), () -> "Could not find a join between tables [" + baseTableName + "][" + queryJoin.getJoinTable() + "]"); + ////////////////////////////////////////////////// + // loop over join-ons (e.g., multi-column join) // + ////////////////////////////////////////////////// for(JoinOn joinOn : joinMetaData.getJoinOns()) { + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // figure out if the join needs flipped. We want its left table to equal the queryJoin's base table. // + //////////////////////////////////////////////////////////////////////////////////////////////////////// QTableMetaData leftTable = instance.getTable(joinMetaData.getLeftTable()); QTableMetaData rightTable = instance.getTable(joinMetaData.getRightTable()); + if(!joinMetaData.getLeftTable().equals(baseTableName)) + { + joinOn = joinOn.flip(); + QTableMetaData tmpTable = leftTable; + leftTable = rightTable; + rightTable = tmpTable; + } + + //////////////////////////////////////////////////////////// + // get the table-names-or-aliases to use in the ON clause // + //////////////////////////////////////////////////////////// String baseTableOrAlias = queryJoin.getBaseTableOrAlias(); + String joinTableOrAlias = queryJoin.getJoinTableOrItsAlias(); if(baseTableOrAlias == null) { baseTableOrAlias = leftTable.getName(); @@ -279,15 +312,6 @@ public abstract class AbstractRDBMSAction } } - String joinTableOrAlias = queryJoin.getJoinTableOrItsAlias(); - if(!joinMetaData.getLeftTable().equals(baseTableName)) - { - joinOn = joinOn.flip(); - QTableMetaData tmpTable = leftTable; - leftTable = rightTable; - rightTable = tmpTable; - } - joinClauseList.add(escapeIdentifier(baseTableOrAlias) + "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField()))) + " = " + escapeIdentifier(joinTableOrAlias) @@ -938,6 +962,7 @@ public abstract class AbstractRDBMSAction { try { + params = Objects.requireNonNullElse(params, Collections.emptyList()); params = params.size() <= 100 ? params : params.subList(0, 99); ///////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index 21a2052f..b174373b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -116,7 +116,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega actionTimeoutHelper = new ActionTimeoutHelper(aggregateInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql)); actionTimeoutHelper.start(); - QueryManager.executeStatement(statement, ((ResultSet resultSet) -> + QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // @@ -168,8 +168,10 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega }), params); } - - logSQL(sql, params, mark); + finally + { + logSQL(sql, params, mark); + } return rs; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index ba167674..a24890f1 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -84,10 +84,10 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf setSqlAndJoinsInQueryStat(sql, joinsContext); CountOutput rs = new CountOutput(); + long mark = System.currentTimeMillis(); + try(Connection connection = getConnection(countInput)) { - long mark = System.currentTimeMillis(); - statement = connection.prepareStatement(sql); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -96,7 +96,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf actionTimeoutHelper = new ActionTimeoutHelper(countInput.getTimeoutSeconds(), TimeUnit.SECONDS, new StatementTimeoutCanceller(statement, sql)); actionTimeoutHelper.start(); - QueryManager.executeStatement(statement, ((ResultSet resultSet) -> + QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // @@ -116,7 +116,9 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf setQueryStatFirstResultTime(); }), params); - + } + finally + { logSQL(sql, params, mark); } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index dd9cf209..5d4cddf8 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -212,13 +212,16 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // LOG.debug("rowCount 0 trying to delete [" + tableName + "][" + primaryKey + "]"); // deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError("Record was not deleted (but no error was given from the database)")); // } - logSQL(sql, List.of(primaryKey), mark); } catch(Exception e) { LOG.debug("Exception trying to delete [" + tableName + "][" + primaryKey + "]", e); deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError(new SystemErrorStatusMessage("Record was not deleted: " + e.getMessage()))); } + finally + { + logSQL(sql, List.of(primaryKey), mark); + } } @@ -228,13 +231,14 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte *******************************************************************************/ public void doDeleteList(Connection connection, QTableMetaData table, List primaryKeys, DeleteOutput deleteOutput) throws QException { + long mark = System.currentTimeMillis(); + String sql = null; + try { - long mark = System.currentTimeMillis(); - String tableName = getTableName(table); String primaryKeyName = getColumnName(table.getField(table.getPrimaryKeyField())); - String sql = "DELETE FROM " + sql = "DELETE FROM " + escapeIdentifier(tableName) + " WHERE " + escapeIdentifier(primaryKeyName) @@ -246,13 +250,15 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys); deleteOutput.addToDeletedRecordCount(rowCount); - - logSQL(sql, primaryKeys, mark); } catch(Exception e) { throw new QException("Error executing delete: " + e.getMessage(), e); } + finally + { + logSQL(sql, primaryKeys, mark); + } } @@ -282,12 +288,14 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte { int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params); deleteOutput.setDeletedRecordCount(rowCount); - - logSQL(sql, params, mark); } catch(Exception e) { throw new QException("Error executing delete with filter: " + e.getMessage(), e); } + finally + { + logSQL(sql, params, mark); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 101d96f4..b58d1386 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -57,9 +57,13 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte InsertOutput rs = new InsertOutput(); QTableMetaData table = insertInput.getTable(); - Connection connection = null; + Connection connection = null; boolean needToCloseConnection = false; + StringBuilder sql = null; + List params = null; + Long mark = null; + try { List insertableFields = table.getFields().values().stream() @@ -88,10 +92,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) { - String tableName = escapeIdentifier(getTableName(table)); - StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); - List params = new ArrayList<>(); - int recordIndex = 0; + String tableName = escapeIdentifier(getTableName(table)); + sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); + params = new ArrayList<>(); + int recordIndex = 0; ////////////////////////////////////////////////////// // for each record in the page: // @@ -133,7 +137,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte continue; } - Long mark = System.currentTimeMillis(); + mark = System.currentTimeMillis(); /////////////////////////////////////////////////////////// // execute the insert, then foreach record in the input, // @@ -163,6 +167,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte } catch(Exception e) { + logSQL(sql, params, mark); throw new QException("Error executing insert: " + e.getMessage(), e); } finally diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index f39ab0f2..5b687cd7 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -170,7 +170,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf ////////////////////////////////////////////// QueryOutput queryOutput = new QueryOutput(queryInput); - QueryManager.executeStatement(statement, ((ResultSet resultSet) -> + QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // @@ -223,17 +223,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf }), params); - logSQL(sql, params, mark); - return queryOutput; } - catch(Exception e) - { - logSQL(sql, params, mark); - throw (e); - } finally { + logSQL(sql, params, mark); + if(actionTimeoutHelper != null) { ///////////////////////////////////////// @@ -366,10 +361,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { RDBMSBackendMetaData rdbmsBackendMetaData = (RDBMSBackendMetaData) queryInput.getBackend(); - //////////////////////////////////////////////////////////////////////////// - // todo - remove "aurora" - it's a legacy value here for a staged rollout // - //////////////////////////////////////////////////////////////////////////// - if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || "aurora".equals(rdbmsBackendMetaData.getVendor())) + if(RDBMSBackendMetaData.VENDOR_MYSQL.equals(rdbmsBackendMetaData.getVendor()) || RDBMSBackendMetaData.VENDOR_AURORA_MYSQL.equals(rdbmsBackendMetaData.getVendor())) { ////////////////////////////////////////////////////////////////////////////////////////////////////// // mysql "optimization", presumably here - from Result Set section of // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index 54276782..627b9c60 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -179,10 +179,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte //////////////////////////////////////////////////////////////////////////////// // let query manager do the batch updates - note that it will internally page // //////////////////////////////////////////////////////////////////////////////// - QueryManager.executeBatchUpdate(connection, sql, values); - incrementStatus(updateInput, recordList.size()); - - logSQL(sql, values, mark); + try + { + QueryManager.executeBatchUpdate(connection, sql, values); + incrementStatus(updateInput, recordList.size()); + } + finally + { + logSQL(sql, values, mark); + } } @@ -249,10 +254,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte ///////////////////////////////////// // let query manager do the update // ///////////////////////////////////// - QueryManager.executeUpdate(connection, sql, params); - incrementStatus(updateInput, page.size()); - - logSQL(sql, params, mark); + try + { + QueryManager.executeUpdate(connection, sql, params); + incrementStatus(updateInput, page.size()); + } + finally + { + logSQL(sql, params, mark); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 084e5df1..5ec62e34 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -150,10 +150,7 @@ public class ConnectionManager return switch(backend.getVendor()) { - //////////////////////////////////////////////////////////////////////////// - // todo - remove "aurora" - it's a legacy value here for a staged rollout // - //////////////////////////////////////////////////////////////////////////// - case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "com.mysql.cj.jdbc.Driver"; + case RDBMSBackendMetaData.VENDOR_MYSQL, RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "com.mysql.cj.jdbc.Driver"; case RDBMSBackendMetaData.VENDOR_H2 -> "org.h2.Driver"; default -> throw (new IllegalStateException("We do not know what jdbc driver to use for vendor name [" + backend.getVendor() + "]. Try setting jdbcDriverClassName in your backend meta data.")); }; @@ -178,10 +175,7 @@ public class ConnectionManager //////////////////////////////////////////////////////////////// // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; - //////////////////////////////////////////////////////////////////////////// - // todo - remove "aurora" - it's a legacy value here for a staged rollout // - //////////////////////////////////////////////////////////////////////////// - case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL, "aurora" -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; + case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; case RDBMSBackendMetaData.VENDOR_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; case RDBMSBackendMetaData.VENDOR_H2 -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index 94d81dfd..f1e31ffe 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -32,6 +32,7 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; import java.time.Instant; @@ -56,6 +57,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValu import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -121,6 +123,17 @@ public class QueryManager ** customized settings/optimizations). *******************************************************************************/ public static void executeStatement(PreparedStatement statement, ResultSetProcessor processor, Object... params) throws SQLException, QException + { + executeStatement(statement, null, processor, params); + } + + + + /******************************************************************************* + ** Let the caller provide their own prepared statement (e.g., possibly with some + ** customized settings/optimizations). + *******************************************************************************/ + public static void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException { ResultSet resultSet = null; @@ -136,6 +149,14 @@ public class QueryManager processor.processResultSet(resultSet); } } + catch(SQLException e) + { + if(sql != null) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + } + throw (e); + } finally { if(resultSet != null) @@ -372,10 +393,17 @@ public class QueryManager *******************************************************************************/ public static PreparedStatement executeUpdate(Connection connection, String sql, Object... params) throws SQLException { - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - incrementStatistic(STAT_QUERIES_RAN); - statement.executeUpdate(); - return (statement); + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) + { + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + return (statement); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } } @@ -385,10 +413,17 @@ public class QueryManager *******************************************************************************/ public static PreparedStatement executeUpdate(Connection connection, String sql, List params) throws SQLException { - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - incrementStatistic(STAT_QUERIES_RAN); - statement.executeUpdate(); - return (statement); + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) + { + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + return (statement); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } } @@ -436,6 +471,11 @@ public class QueryManager statement.executeUpdate(); return (statement.getUpdateCount()); } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } } @@ -488,19 +528,24 @@ public class QueryManager *******************************************************************************/ public static List executeInsertForGeneratedIds(Connection connection, String sql, List params) throws SQLException { - List rs = new ArrayList<>(); try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { bindParams(params.toArray(), statement); incrementStatistic(STAT_QUERIES_RAN); statement.executeUpdate(); - ResultSet generatedKeys = statement.getGeneratedKeys(); + ResultSet generatedKeys = statement.getGeneratedKeys(); + List rs = new ArrayList<>(); while(generatedKeys.next()) { rs.add(getInteger(generatedKeys, 1)); } + return (rs); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); } - return (rs); } @@ -747,14 +792,14 @@ public class QueryManager else if(value instanceof LocalDate ld) { @SuppressWarnings("deprecation") - java.sql.Date date = new java.sql.Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); + Date date = new Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); statement.setDate(index, date); return (1); } else if(value instanceof LocalTime lt) { @SuppressWarnings("deprecation") - java.sql.Time time = new java.sql.Time(lt.getHour(), lt.getMinute(), lt.getSecond()); + Time time = new Time(lt.getHour(), lt.getMinute(), lt.getSecond()); statement.setTime(index, time); return (1); } @@ -943,7 +988,7 @@ public class QueryManager } else { - statement.setDate(index, new java.sql.Date(value.getTime())); + statement.setDate(index, new Date(value.getTime())); } } diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 2570f6ea..7a84cc63 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -273,6 +273,7 @@ public class TestUtils .withJoinNameChain(List.of("orderInstructionsJoinOrder"))) .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id")) .withField(new QFieldMetaData("instructions", QFieldType.STRING)) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderInstructionsJoinOrder"))) ); qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") @@ -395,10 +396,10 @@ public class TestUtils qInstance.addJoin(new QJoinMetaData() .withName("orderInstructionsJoinOrder") - .withLeftTable(TABLE_NAME_ORDER_INSTRUCTIONS) - .withRightTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withLeftTable(TABLE_NAME_ORDER) .withType(JoinType.MANY_TO_ONE) - .withJoinOn(new JoinOn("orderId", "id")) + .withJoinOn(new JoinOn("id", "orderId")) ); qInstance.addPossibleValueSource(new QPossibleValueSource() diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java index 12993de3..44dfbad0 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionJoinsTest.java @@ -54,6 +54,7 @@ import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; /******************************************************************************* @@ -69,10 +70,6 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); - - AbstractRDBMSAction.setLogSQL(true); - AbstractRDBMSAction.setLogSQLReformat(true); - AbstractRDBMSAction.setLogSQLOutput("system.out"); } @@ -909,7 +906,7 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest ** *******************************************************************************/ @Test - void testMultipleReversedDirectionJoinsBetweenSameTables() throws QException + void testMultipleReversedDirectionJoinsBetweenSameTablesAllAccessKey() throws QException { QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); @@ -992,6 +989,32 @@ public class RDBMSQueryActionJoinsTest extends RDBMSActionTest + /******************************************************************************* + ** We had, at one time, a bug where, for tables with 2 joins between each other, + ** an ON clause could get written using the wrong table name in one part. + ** + ** With that bug, this QueryAction.execute would throw an SQL Exception. + ** + ** So this test, just makes sure that no such exception gets thrown. + *******************************************************************************/ + @Test + void testFlippedJoinForOnClause() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_INSTRUCTIONS); + queryInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_ORDER)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertFalse(queryOutput.getRecords().isEmpty()); + + //////////////////////////////////// + // if no exception, then we pass. // + //////////////////////////////////// + } + + + /******************************************************************************* ** Addressing a regression where a table was brought into a query for its ** security field, but it was a write-scope lock, so, it shouldn't have been.