diff --git a/.circleci/config.yml b/.circleci/config.yml index ce065e1..0ac03b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,7 @@ commands: name: Run react app and mvn verify command: | echo "HTTPS=true" >> ./.env + npm run build export REACT_APP_PROXY_LOCALHOST_PORT=8001; export PORT=3001; npm run start & dockerize -wait tcp://localhost:3001 -timeout 3m export QQQ_SELENIUM_HEADLESS=true; mvn verify diff --git a/src/qqq/pages/records/query/ColumnStats.tsx b/src/qqq/pages/records/query/ColumnStats.tsx new file mode 100644 index 0000000..c42c91e --- /dev/null +++ b/src/qqq/pages/records/query/ColumnStats.tsx @@ -0,0 +1,240 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import {QFieldMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QFieldMetaData"; +import {QTableMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QTableMetaData"; +import {QTableSection} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QTableSection"; +import {QJobComplete} from "@kingsrook/qqq-frontend-core/lib/model/processes/QJobComplete"; +import {QJobError} from "@kingsrook/qqq-frontend-core/lib/model/processes/QJobError"; +import {QRecord} from "@kingsrook/qqq-frontend-core/lib/model/QRecord"; +import {QQueryFilter} from "@kingsrook/qqq-frontend-core/lib/model/query/QQueryFilter"; +import {TablePagination} from "@mui/material"; +import Box from "@mui/material/Box"; +import Button from "@mui/material/Button"; +import Grid from "@mui/material/Grid"; +import Icon from "@mui/material/Icon"; +import LinearProgress from "@mui/material/LinearProgress"; +import Typography from "@mui/material/Typography"; +import {DataGridPro, GridSortModel} from "@mui/x-data-grid-pro"; +import FormData from "form-data"; +import React, {useEffect, useState} from "react"; +import DataGridUtils from "qqq/utils/DataGridUtils"; +import Client from "qqq/utils/qqq/Client"; +import ValueUtils from "qqq/utils/qqq/ValueUtils"; + +interface Props +{ + tableMetaData: QTableMetaData; + fieldMetaData: QFieldMetaData; + filter: QQueryFilter; +} + +ColumnStats.defaultProps = { +}; + +const qController = Client.getInstance(); + +function ColumnStats({tableMetaData, fieldMetaData, filter}: Props): JSX.Element +{ + const [statusString, setStatusString] = useState("Calculating statistics..."); + const [loading, setLoading] = useState(true); + const [valueCounts, setValueCounts] = useState(null as QRecord[]); + const [statsRecord, setStatsRecord] = useState(null as QRecord); + const [orderBy, setOrderBy] = useState(null as string); + const [statsFields, setStatsFields] = useState([] as QFieldMetaData[]); + const [countDistinct, setCountDistinct] = useState(null as number); + const [rows, setRows] = useState([]); + const [columns, setColumns] = useState([]); + + useEffect(() => + { + if(!loading) + { + return; + } + + (async () => + { + const formData = new FormData(); + formData.append("tableName", tableMetaData.name); + formData.append("fieldName", fieldMetaData.name); + formData.append("filterJSON", JSON.stringify(filter)); + if(orderBy) + { + formData.append("orderBy", orderBy); + } + const processResult = await qController.processRun("columnStats", formData); + + setStatusString(null) + if (processResult instanceof QJobError) + { + const jobError = processResult as QJobError; + setStatusString("Error fetching column stats: " + jobError.error); + setLoading(false); + } + else + { + const result = processResult as QJobComplete; + + const statFieldObjects = result.values.statsFields; + if(statFieldObjects && statFieldObjects.length) + { + const newStatsFields = [] as QFieldMetaData[]; + for(let i = 0; i(); + fakeTableMetaData.fields.set(fieldMetaData.name, fieldMetaData); + fakeTableMetaData.fields.set("count", new QFieldMetaData({name: "count", label: "Count", type: "INTEGER"})); + fakeTableMetaData.sections = [] as QTableSection[]; + fakeTableMetaData.sections.push(new QTableSection({fieldNames: [fieldMetaData.name, "count"]})); + + const {rows, columnsToRender} = DataGridUtils.makeRows(valueCounts, fakeTableMetaData); + const columns = DataGridUtils.setupGridColumns(fakeTableMetaData, columnsToRender); + columns.forEach((c) => + { + c.width = 200; + c.filterable = false; + c.hideable = false; + }) + + setRows(rows); + setColumns(columns); + setLoading(false); + } + })(); + }, [loading]); + + function CustomPagination() + { + return ( + + {rows && rows.length && countDistinct && rows.length < countDistinct ? Showing the first {rows.length.toLocaleString()} of {countDistinct.toLocaleString()} values : <>} + {rows && rows.length && countDistinct && rows.length >= countDistinct && rows.length == 1 ? Showing the only value : <>} + {rows && rows.length && countDistinct && rows.length >= countDistinct && rows.length > 1 ? Showing all {rows.length.toLocaleString()} values : <>} + + ); + } + + const refresh = () => + { + setLoading(true) + setStatusString("Refreshing...") + } + + function Loading() + { + return ( + + ); + } + + const handleSortChange = (gridSort: GridSortModel) => + { + if (gridSort && gridSort.length > 0) + { + console.log("Sort: ", gridSort[0]); + setOrderBy(`${gridSort[0].field}.${gridSort[0].sort}`); + refresh(); + } + }; + + return ( + + + + Column Statistics for {fieldMetaData.label} + + {statusString ?? <> } + + + + + + + + (params.indexRelativeToCurrentPage % 2 === 0 ? "even" : "odd")} + sortingMode={"server"} + onSortModelChange={handleSortChange} + sortingOrder={["desc", "asc"]} + pagination={true} + paginationMode={"server"} + initialState={{ + sorting: { + sortModel: [ + { + field: "count", + sort: "desc", + }, + ], + }, + }} + /> + + + + + { + statsFields && statsFields.map((field) => + ( + +
{field.label}:
+
{ValueUtils.getValueForDisplay(field, statsRecord?.values.get(field.name), statsRecord?.displayValues.get(field.name))}
+ + )) + } + + + + ); +} + +export default ColumnStats; diff --git a/src/qqq/pages/records/query/RecordQuery.tsx b/src/qqq/pages/records/query/RecordQuery.tsx index f9f5098..f341867 100644 --- a/src/qqq/pages/records/query/RecordQuery.tsx +++ b/src/qqq/pages/records/query/RecordQuery.tsx @@ -49,10 +49,11 @@ import FormData from "form-data"; import React, {forwardRef, useContext, useEffect, useReducer, useRef, useState} from "react"; import {useLocation, useNavigate, useSearchParams} from "react-router-dom"; import QContext from "QContext"; -import {QActionsMenuButton, QCreateNewButton} from "qqq/components/buttons/DefaultButtons"; +import {QActionsMenuButton, QCancelButton, QCreateNewButton} from "qqq/components/buttons/DefaultButtons"; import SavedFilters from "qqq/components/misc/SavedFilters"; import BaseLayout from "qqq/layouts/BaseLayout"; import ProcessRun from "qqq/pages/processes/ProcessRun"; +import ColumnStats from "qqq/pages/records/query/ColumnStats"; import DataGridUtils from "qqq/utils/DataGridUtils"; import Client from "qqq/utils/qqq/Client"; import FilterUtils from "qqq/utils/qqq/FilterUtils"; @@ -167,6 +168,8 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element const [activeModalProcess, setActiveModalProcess] = useState(null as QProcessMetaData); const [launchingProcess, setLaunchingProcess] = useState(launchProcess); const [recordIdsForProcess, setRecordIdsForProcess] = useState(null as string | QQueryFilter); + const [columnStatsFieldName, setColumnStatsFieldName] = useState(null as string) + const [filterForColumnStats, setFilterForColumnStats] = useState(null as QQueryFilter) const instance = useRef({timer: null}); @@ -779,6 +782,16 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element closeActionsMenu(); }; + const closeColumnStats = (event: object, reason: string) => + { + if (reason === "backdropClick" || reason === "escapeKeyDown") + { + return; + } + + setColumnStatsFieldName(null); + }; + const closeModalProcess = (event: object, reason: string) => { if (reason === "backdropClick" || reason === "escapeKeyDown") @@ -981,6 +994,12 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element } } + const openColumnStatistics = async (column: GridColDef) => + { + setFilterForColumnStats(buildQFilter(tableMetaData, filterModel)); + setColumnStatsFieldName(column.field); + } + const CustomColumnMenu = forwardRef( function GridColumnMenu(props: GridColumnMenuProps, ref) { @@ -1023,6 +1042,14 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element */} + + { + hideMenu(e); + openColumnStatistics(currentColumn); + }}> + Column statistics + + ); }); @@ -1360,6 +1387,24 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element } + + { + columnStatsFieldName && + closeColumnStats(event, reason)}> +
+ + + + + + closeColumnStats(null, null)} disabled={false} /> + + + + +
+ + } ); diff --git a/src/qqq/styles/qqq-override-styles.css b/src/qqq/styles/qqq-override-styles.css index c01fa06..13393ff 100644 --- a/src/qqq/styles/qqq-override-styles.css +++ b/src/qqq/styles/qqq-override-styles.css @@ -350,12 +350,14 @@ input[type="search"]::-webkit-search-results-decoration { display: none; } color: rgb(52, 71, 103); font-weight: 700; padding-right: .5em; + display: inline; } .fieldValue { color: rgb(123, 128, 154); font-weight: 400; + display: inline; } .fullScreenWidget diff --git a/src/qqq/utils/DataGridUtils.tsx b/src/qqq/utils/DataGridUtils.tsx index 6d6bee0..99c5d72 100644 --- a/src/qqq/utils/DataGridUtils.tsx +++ b/src/qqq/utils/DataGridUtils.tsx @@ -20,6 +20,7 @@ */ import {AdornmentType} from "@kingsrook/qqq-frontend-core/lib/model/metaData/AdornmentType"; +import {QFieldMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QFieldMetaData"; import {QFieldType} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QFieldType"; import {QTableMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QTableMetaData"; import {QRecord} from "@kingsrook/qqq-frontend-core/lib/model/QRecord"; @@ -55,7 +56,14 @@ export default class DataGridUtils if(!row["id"]) { - row["id"] = row[tableMetaData.primaryKeyField]; + row["id"] = record.values.get(tableMetaData.primaryKeyField) ?? row[tableMetaData.primaryKeyField]; + if(row["id"] === null || row["id"] === undefined) + { + ///////////////////////////////////////////////////////////////////////////////////////// + // DataGrid gets very upset about a null or undefined here, so, try to make it happier // + ///////////////////////////////////////////////////////////////////////////////////////// + row["id"] = "--"; + } } rows.push(row); @@ -101,89 +109,7 @@ export default class DataGridUtils sortedKeys.forEach((key) => { const field = tableMetaData.fields.get(key); - - let columnType = "string"; - let columnWidth = 200; - let filterOperators: GridFilterOperator[] = QGridStringOperators; - - if (field.possibleValueSourceName) - { - filterOperators = buildQGridPvsOperators(tableMetaData.name, field); - } - else - { - switch (field.type) - { - case QFieldType.DECIMAL: - case QFieldType.INTEGER: - columnType = "number"; - columnWidth = 100; - - if (key === tableMetaData.primaryKeyField && field.label.length < 3) - { - columnWidth = 75; - } - - filterOperators = QGridNumericOperators; - break; - case QFieldType.DATE: - columnType = "date"; - columnWidth = 100; - filterOperators = getGridDateOperators(); - break; - case QFieldType.DATE_TIME: - columnType = "dateTime"; - columnWidth = 200; - filterOperators = getGridDateOperators(true); - break; - case QFieldType.BOOLEAN: - columnType = "string"; // using boolean gives an odd 'no' for nulls. - columnWidth = 75; - filterOperators = QGridBooleanOperators; - break; - default: - // noop - leave as string - } - } - - if (field.hasAdornment(AdornmentType.SIZE)) - { - const sizeAdornment = field.getAdornment(AdornmentType.SIZE); - const width: string = sizeAdornment.getValue("width"); - const widths: Map = new Map([ - ["small", 100], - ["medium", 200], - ["large", 400], - ["xlarge", 600] - ]); - if (widths.has(width)) - { - columnWidth = widths.get(width); - } - else - { - console.log("Unrecognized size.width adornment value: " + width); - } - } - - const column = { - field: field.name, - type: columnType, - headerName: field.label, - width: columnWidth, - renderCell: null as any, - filterOperators: filterOperators, - }; - - ///////////////////////////////////////////////////////////////////////////////////////// - // looks like, maybe we can just always render all columns, and remove this parameter? // - ///////////////////////////////////////////////////////////////////////////////////////// - if (columnsToRender == null || columnsToRender[field.name]) - { - column.renderCell = (cellValues: any) => ( - (cellValues.value) - ); - } + const column = this.makeColumnFromField(field, tableMetaData, columnsToRender); if (key === tableMetaData.primaryKeyField && linkBase) { @@ -201,5 +127,97 @@ export default class DataGridUtils return (columns); }; + + + /******************************************************************************* + ** + *******************************************************************************/ + public static makeColumnFromField = (field: QFieldMetaData, tableMetaData: QTableMetaData, columnsToRender: any): GridColDef => + { + let columnType = "string"; + let columnWidth = 200; + let filterOperators: GridFilterOperator[] = QGridStringOperators; + + if (field.possibleValueSourceName) + { + filterOperators = buildQGridPvsOperators(tableMetaData.name, field); + } + else + { + switch (field.type) + { + case QFieldType.DECIMAL: + case QFieldType.INTEGER: + columnType = "number"; + columnWidth = 100; + + if (field.name === tableMetaData.primaryKeyField && field.label.length < 3) + { + columnWidth = 75; + } + + filterOperators = QGridNumericOperators; + break; + case QFieldType.DATE: + columnType = "date"; + columnWidth = 100; + filterOperators = getGridDateOperators(); + break; + case QFieldType.DATE_TIME: + columnType = "dateTime"; + columnWidth = 200; + filterOperators = getGridDateOperators(true); + break; + case QFieldType.BOOLEAN: + columnType = "string"; // using boolean gives an odd 'no' for nulls. + columnWidth = 75; + filterOperators = QGridBooleanOperators; + break; + default: + // noop - leave as string + } + } + + if (field.hasAdornment(AdornmentType.SIZE)) + { + const sizeAdornment = field.getAdornment(AdornmentType.SIZE); + const width: string = sizeAdornment.getValue("width"); + const widths: Map = new Map([ + ["small", 100], + ["medium", 200], + ["large", 400], + ["xlarge", 600] + ]); + if (widths.has(width)) + { + columnWidth = widths.get(width); + } + else + { + console.log("Unrecognized size.width adornment value: " + width); + } + } + + const column = { + field: field.name, + type: columnType, + headerName: field.label, + width: columnWidth, + renderCell: null as any, + filterOperators: filterOperators, + }; + + ///////////////////////////////////////////////////////////////////////////////////////// + // looks like, maybe we can just always render all columns, and remove this parameter? // + ///////////////////////////////////////////////////////////////////////////////////////// + if (columnsToRender == null || columnsToRender[field.name]) + { + column.renderCell = (cellValues: any) => ( + (cellValues.value) + ); + } + + return (column); + } }