Merge branch 'feature/CTLE-207-query-joins' into dev

pom.xml

@@ -85,7 +85,7 @@
       <dependency>
          <groupId>io.javalin</groupId>
          <artifactId>javalin</artifactId>
-         <version>5.1.4</version>
+         <version>5.4.2</version>
          <scope>test</scope>
       </dependency>
       <dependency>

src/App.tsx

@@ -549,7 +549,7 @@ export default function App()
       },
    );
 
-   const [pageHeader, setPageHeader] = useState("");
+   const [pageHeader, setPageHeader] = useState("" as string | JSX.Element);
    const [accentColor, setAccentColor] = useState("#0062FF");
    return (
 
@@ -557,7 +557,7 @@ export default function App()
          <QContext.Provider value={{
             pageHeader: pageHeader,
             accentColor: accentColor,
-            setPageHeader: (header: string) => setPageHeader(header),
+            setPageHeader: (header: string | JSX.Element) => setPageHeader(header),
             setAccentColor: (accentColor: string) => setAccentColor(accentColor)
          }}>
             <ThemeProvider theme={theme}>

src/QContext.tsx

@@ -24,8 +24,8 @@ import {createContext} from "react";
 
 interface QContext
 {
-   pageHeader: string;
+   pageHeader: string | JSX.Element;
-   setPageHeader?: (header: string) => void;
+   setPageHeader?: (header: string | JSX.Element) => void;
    accentColor: string;
    setAccentColor?: (header: string) => void;
 }

src/qqq/components/audits/AuditBody.tsx

@@ -194,7 +194,7 @@ function AuditBody({tableMetaData, recordId, record}: Props): JSX.Element
             new QFilterOrderBy("timestamp", sortDirection),
             new QFilterOrderBy("id", sortDirection),
             new QFilterOrderBy("auditDetail.id", true)
-         ]);
+         ], "AND", 0, limit);
 
          ///////////////////////////////
          // fetch audits in try-catch //
@@ -202,7 +202,7 @@ function AuditBody({tableMetaData, recordId, record}: Props): JSX.Element
          let audits = [] as QRecord[]
          try
          {
-            audits = await qController.query("audit", filter, limit, 0, [new QueryJoin("auditDetail", true, "LEFT")]);
+            audits = await qController.query("audit", filter, [new QueryJoin("auditDetail", true, "LEFT")]);
             setAudits(audits);
          }
          catch(e)
@@ -222,8 +222,8 @@ function AuditBody({tableMetaData, recordId, record}: Props): JSX.Element
          // if we fetched the limit
          if (audits.length == limit)
          {
-            const count = await qController.count("audit", filter);
+            const [count, distinctCount] = await qController.count("audit", filter, null, true); // todo validate distinct working here!
-            setTotal(count);
+            setTotal(distinctCount);
          }
 
          //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/qqq/components/buttons/DefaultButtons.tsx

@@ -50,18 +50,20 @@ export function QCreateNewButton({tablePath}: QCreateNewButtonProps): JSX.Elemen
 interface QSaveButtonProps
 {
    label?: string;
+   iconName?: string;
    onClickHandler?: any,
    disabled: boolean
 }
 QSaveButton.defaultProps = {
-   label: "Save"
+   label: "Save",
+   iconName: "save"
 };
 
-export function QSaveButton({label, onClickHandler, disabled}: QSaveButtonProps): JSX.Element
+export function QSaveButton({label, iconName, onClickHandler, disabled}: QSaveButtonProps): JSX.Element
 {
    return (
       <Box ml={3} width={standardWidth}>
-         <MDButton type="submit" variant="gradient" color="info" size="small" onClick={onClickHandler} fullWidth startIcon={<Icon>save</Icon>} disabled={disabled}>
+         <MDButton type="submit" variant="gradient" color="info" size="small" onClick={onClickHandler} fullWidth startIcon={<Icon>{iconName}</Icon>} disabled={disabled}>
             {label}
          </MDButton>
       </Box>

src/qqq/components/buttons/MenuButton.tsx

@@ -0,0 +1,144 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2023.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {ClickAwayListener, Grow, MenuList, Paper, Popper} from "@mui/material";
+import Button from "@mui/material/Button/Button";
+import Icon from "@mui/material/Icon";
+import MenuItem from "@mui/material/MenuItem";
+import React, {useEffect, useRef, useState} from "react";
+
+
+interface Props
+{
+   label: string;
+   iconName?: string
+   options: string[];
+   disabled?: boolean;
+   callback: (selectedIndex: number) => void;
+}
+
+MenuButton.defaultProps =
+{
+   disabled: false
+};
+
+function MenuButton({label, iconName, options, disabled, callback}: Props)
+{
+   const [open, setOpen] = useState(false);
+   const anchorRef = useRef<HTMLButtonElement>(null);
+
+   const handleToggle = () => 
+   {
+      setOpen((prevOpen) => !prevOpen);
+   };
+
+   const handleClose = (event: Event | React.SyntheticEvent) => 
+   {
+      if (anchorRef.current && anchorRef.current.contains(event.target as HTMLElement))
+      {
+         return;
+      }
+
+      setOpen(false);
+   };
+
+   function handleListKeyDown(event: React.KeyboardEvent) 
+   {
+      if (event.key === "Tab") 
+      {
+         event.preventDefault();
+         setOpen(false);
+      }
+      else if (event.key === "Escape") 
+      {
+         setOpen(false);
+      }
+   }
+
+   // return focus to the button when we transitioned from !open -> open
+   const prevOpen = useRef(open);
+   useEffect(() =>
+   {
+      if (prevOpen.current === true && open === false) 
+      {
+         anchorRef.current!.focus();
+      }
+
+      prevOpen.current = open;
+   }, [open]);
+
+
+   const menuItemClicked = (e: React.MouseEvent<HTMLLIElement, MouseEvent>, newIndex: number) =>
+   {
+      callback(newIndex);
+      handleClose(e);
+   }
+
+   const menuItems: JSX.Element[] = []
+   options.map((option, index) =>
+   {
+      menuItems.push(<MenuItem key={index} onClick={e => menuItemClicked(e, index)}>
+         {option}
+      </MenuItem>);
+   })
+
+   return (
+      <div>
+         <Button
+            ref={anchorRef}
+            id="composition-button"
+            aria-controls={open ? "composition-menu" : undefined}
+            aria-expanded={open ? "true" : undefined}
+            aria-haspopup="true"
+            onClick={handleToggle}
+            startIcon={iconName ? <Icon>{iconName}</Icon> : undefined}
+            sx={{pl: "1.25rem"}}
+            disabled={disabled}
+         >
+            {label}
+         </Button>
+         <Popper
+            open={open}
+            anchorEl={anchorRef.current}
+            role={undefined}
+            placement="bottom-start"
+            transition
+            disablePortal nonce={undefined} onResize={undefined} onResizeCapture={undefined}
+            sx={{zIndex: 1}}
+         >
+            {({TransitionProps, placement}) => (
+               <Grow{...TransitionProps} style={{transformOrigin: placement === "bottom-start" ? "left top" : "left bottom"}}>
+                  <Paper elevation={3}>
+                     <ClickAwayListener onClickAway={handleClose}>
+                        <MenuList onKeyDown={handleListKeyDown}>
+                           {menuItems}
+                        </MenuList>
+                     </ClickAwayListener>
+                  </Paper>
+               </Grow>
+            )}
+         </Popper>
+      </div>
+   );
+}
+
+export default MenuButton;
+

src/qqq/components/widgets/misc/DataBagViewer.tsx

@@ -100,8 +100,8 @@ export default function DataBagViewer({dataBagId}: Props): JSX.Element
 
             const criteria = [new QFilterCriteria("dataBagId", QCriteriaOperator.EQUALS, [dataBagId])];
             const orderBys = [new QFilterOrderBy("sequenceNo", false)];
-            const filter = new QQueryFilter(criteria, orderBys);
+            const filter = new QQueryFilter(criteria, orderBys, "AND", 0, 25);
-            const versions = await qController.query("dataBagVersion", filter, 25, 0);
+            const versions = await qController.query("dataBagVersion", filter);
             console.log("Fetched versions:");
             console.log(versions);
             setVersionRecordList(versions);

src/qqq/components/widgets/misc/RecordGridWidget.tsx

@@ -60,13 +60,13 @@ function RecordGridWidget({widgetMetaData, data}: Props): JSX.Element
          }
 
          const tableMetaData = new QTableMetaData(data.childTableMetaData);
-         const {rows, columnsToRender} = DataGridUtils.makeRows(records, tableMetaData);
+         const rows = DataGridUtils.makeRows(records, tableMetaData);
 
          /////////////////////////////////////////////////////////////////////////////////
          // note - tablePath may be null, if the user doesn't have access to the table. //
          /////////////////////////////////////////////////////////////////////////////////
          const childTablePath = data.tablePath ? data.tablePath + (data.tablePath.endsWith("/") ? "" : "/") : data.tablePath;
-         const columns = DataGridUtils.setupGridColumns(tableMetaData, columnsToRender, childTablePath);
+         const columns = DataGridUtils.setupGridColumns(tableMetaData, childTablePath, null, "bySection");
 
          ////////////////////////////////////////////////////////////////
          // do not not show the foreign-key column of the parent table //

src/qqq/components/widgets/misc/ScriptViewer.tsx

@@ -133,8 +133,8 @@ export default function ScriptViewer({scriptId, associatedScriptTableName, assoc
 
             const criteria = [new QFilterCriteria("scriptId", QCriteriaOperator.EQUALS, [scriptId])];
             const orderBys = [new QFilterOrderBy("sequenceNo", false)];
-            const filter = new QQueryFilter(criteria, orderBys);
+            const filter = new QQueryFilter(criteria, orderBys, "AND", 0, 25);
-            const versions = await qController.query("scriptRevision", filter, 25, 0);
+            const versions = await qController.query("scriptRevision", filter);
             console.log("Fetched versions:");
             console.log(versions);
             setVersionRecordList(versions);
@@ -281,7 +281,8 @@ export default function ScriptViewer({scriptId, associatedScriptTableName, assoc
       {
          (async () =>
          {
-            scriptLogs[scriptRevisionId] = await qController.query("scriptLog", new QQueryFilter([new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, [scriptRevisionId])]), 100, 0);
+            let filter = new QQueryFilter([new QFilterCriteria("scriptRevisionId", QCriteriaOperator.EQUALS, [scriptRevisionId])], [new QFilterOrderBy("id", false)], "AND", 0, 100);
+            scriptLogs[scriptRevisionId] = await qController.query("scriptLog", filter);
             setScriptLogs(scriptLogs);
             forceUpdate();
          })();

src/qqq/pages/apps/Home.tsx

@@ -127,7 +127,7 @@ function AppHome({app}: Props): JSX.Element
             let countResult = null;
             if(tableMetaData.capabilities.has(Capability.TABLE_COUNT) && tableMetaData.readPermission)
             {
-               countResult = await qController.count(table.name);
+               [countResult] = await qController.count(table.name);
 
                if (countResult !== null && countResult !== undefined)
                {

src/qqq/pages/records/query/ColumnStats.tsx

@@ -126,8 +126,8 @@ function ColumnStats({tableMetaData, fieldMetaData, filter}: Props): JSX.Element
             fakeTableMetaData.sections = [] as QTableSection[];
             fakeTableMetaData.sections.push(new QTableSection({fieldNames: [fieldMetaData.name, "count"]}));
 
-            const {rows, columnsToRender} = DataGridUtils.makeRows(valueCounts, fakeTableMetaData);
+            const rows = DataGridUtils.makeRows(valueCounts, fakeTableMetaData);
-            const columns = DataGridUtils.setupGridColumns(fakeTableMetaData, columnsToRender);
+            const columns = DataGridUtils.setupGridColumns(fakeTableMetaData, null, null, "bySection");
             columns.forEach((c) =>
             {
                c.width = 200;

src/qqq/pages/records/query/RecordQuery.tsx

@@ -26,7 +26,9 @@ import {QTableMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QT
 import {QJobComplete} from "@kingsrook/qqq-frontend-core/lib/model/processes/QJobComplete";
 import {QJobError} from "@kingsrook/qqq-frontend-core/lib/model/processes/QJobError";
 import {QRecord} from "@kingsrook/qqq-frontend-core/lib/model/QRecord";
+import {QFilterCriteria} from "@kingsrook/qqq-frontend-core/lib/model/query/QFilterCriteria";
 import {QQueryFilter} from "@kingsrook/qqq-frontend-core/lib/model/query/QQueryFilter";
+import {QueryJoin} from "@kingsrook/qqq-frontend-core/lib/model/query/QueryJoin";
 import {Alert, Box, Collapse, TablePagination} from "@mui/material";
 import Button from "@mui/material/Button";
 import Card from "@mui/material/Card";
@@ -37,19 +39,27 @@ import DialogContentText from "@mui/material/DialogContentText";
 import DialogTitle from "@mui/material/DialogTitle";
 import Divider from "@mui/material/Divider";
 import Icon from "@mui/material/Icon";
+import IconButton from "@mui/material/IconButton";
 import LinearProgress from "@mui/material/LinearProgress";
 import ListItemIcon from "@mui/material/ListItemIcon";
 import Menu from "@mui/material/Menu";
 import MenuItem from "@mui/material/MenuItem";
 import Modal from "@mui/material/Modal";
+import Stack from "@mui/material/Stack";
+import TextField from "@mui/material/TextField";
 import Tooltip from "@mui/material/Tooltip";
-import {DataGridPro, GridCallbackDetails, GridColDef, GridColumnMenuContainer, GridColumnMenuProps, GridColumnOrderChangeParams, GridColumnPinningMenuItems, GridColumnsMenuItem, GridColumnVisibilityModel, GridDensity, GridEventListener, GridExportMenuItemProps, GridFilterMenuItem, GridFilterModel, GridPinnedColumns, gridPreferencePanelStateSelector, GridRowId, GridRowParams, GridRowsProp, GridSelectionModel, GridSortItem, GridSortModel, GridState, GridToolbarColumnsButton, GridToolbarContainer, GridToolbarDensitySelector, GridToolbarExportContainer, GridToolbarFilterButton, HideGridColMenuItem, MuiEvent, SortGridMenuItems, useGridApiContext, useGridApiEventHandler, useGridSelector} from "@mui/x-data-grid-pro";
+import {DataGridPro, GridCallbackDetails, GridColDef, GridColumnMenuContainer, GridColumnMenuProps, GridColumnOrderChangeParams, GridColumnPinningMenuItems, GridColumnsMenuItem, GridColumnVisibilityModel, GridDensity, GridEventListener, GridExportMenuItemProps, GridFilterMenuItem, GridFilterModel, GridPinnedColumns, gridPreferencePanelStateSelector, GridRowId, GridRowParams, GridRowProps, GridRowsProp, GridSelectionModel, GridSortItem, GridSortModel, GridState, GridToolbarColumnsButton, GridToolbarContainer, GridToolbarDensitySelector, GridToolbarExportContainer, GridToolbarFilterButton, HideGridColMenuItem, MuiEvent, SortGridMenuItems, useGridApiContext, useGridApiEventHandler, useGridSelector} from "@mui/x-data-grid-pro";
+import {GridColumnsPanelProps} from "@mui/x-data-grid/components/panel/GridColumnsPanel";
+import {gridColumnDefinitionsSelector, gridColumnVisibilityModelSelector} from "@mui/x-data-grid/hooks/features/columns/gridColumnsSelector";
+import {GridRowModel} from "@mui/x-data-grid/models/gridRows";
 import FormData from "form-data";
 import React, {forwardRef, useContext, useEffect, useReducer, useRef, useState} from "react";
 import {useLocation, useNavigate, useSearchParams} from "react-router-dom";
 import QContext from "QContext";
-import {QActionsMenuButton, QCancelButton, QCreateNewButton} from "qqq/components/buttons/DefaultButtons";
+import {QActionsMenuButton, QCancelButton, QCreateNewButton, QSaveButton} from "qqq/components/buttons/DefaultButtons";
+import MenuButton from "qqq/components/buttons/MenuButton";
 import SavedFilters from "qqq/components/misc/SavedFilters";
+import CustomWidthTooltip from "qqq/components/tooltips/CustomWidthTooltip";
 import BaseLayout from "qqq/layouts/BaseLayout";
 import ProcessRun from "qqq/pages/processes/ProcessRun";
 import ColumnStats from "qqq/pages/records/query/ColumnStats";
@@ -86,7 +96,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    const [searchParams] = useSearchParams();
 
    const [showSuccessfullyDeletedAlert, setShowSuccessfullyDeletedAlert] = useState(searchParams.has("deleteSuccess"));
-   const [successAlert, setSuccessAlert] = useState(null as string)
+   const [successAlert, setSuccessAlert] = useState(null as string);
 
    const location = useLocation();
    const navigate = useNavigate();
@@ -103,7 +113,8 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    const columnVisibilityLocalStorageKey = `${COLUMN_VISIBILITY_LOCAL_STORAGE_KEY_ROOT}.${tableName}`;
    const filterLocalStorageKey = `${FILTER_LOCAL_STORAGE_KEY_ROOT}.${tableName}`;
    let defaultSort = [] as GridSortItem[];
-   let defaultVisibility = {};
+   let defaultVisibility = {} as { [index: string]: boolean };
+   let didDefaultVisibilityComeFromLocalStorage = false;
    let defaultRowsPerPage = 10;
    let defaultDensity = "standard" as GridDensity;
    let defaultPinnedColumns = {left: ["__check__", "id"]} as GridPinnedColumns;
@@ -120,6 +131,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    if (localStorage.getItem(columnVisibilityLocalStorageKey))
    {
       defaultVisibility = JSON.parse(localStorage.getItem(columnVisibilityLocalStorageKey));
+      didDefaultVisibilityComeFromLocalStorage = true;
    }
    if (localStorage.getItem(pinnedColumnsLocalStorageKey))
    {
@@ -137,6 +149,8 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    const [filterModel, setFilterModel] = useState({items: []} as GridFilterModel);
    const [columnSortModel, setColumnSortModel] = useState(defaultSort);
    const [columnVisibilityModel, setColumnVisibilityModel] = useState(defaultVisibility);
+   const [didDefaultVisibilityModelComeFromLocalStorage, setDidDefaultVisibilityModelComeFromLocalStorage] = useState(didDefaultVisibilityComeFromLocalStorage);
+   const [visibleJoinTables, setVisibleJoinTables] = useState(new Set<string>());
    const [rowsPerPage, setRowsPerPage] = useState(defaultRowsPerPage);
    const [density, setDensity] = useState(defaultDensity);
    const [pinnedColumns, setPinnedColumns] = useState(defaultPinnedColumns);
@@ -150,8 +164,13 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    const [allTableProcesses, setAllTableProcesses] = useState([] as QProcessMetaData[]);
    const [pageNumber, setPageNumber] = useState(0);
    const [totalRecords, setTotalRecords] = useState(null);
+   const [distinctRecords, setDistinctRecords] = useState(null);
    const [selectedIds, setSelectedIds] = useState([] as string[]);
-   const [selectFullFilterState, setSelectFullFilterState] = useState("n/a" as "n/a" | "checked" | "filter");
+   const [distinctRecordsOnPageCount, setDistinctRecordsOnPageCount] = useState(null as number);
+   const [selectionSubsetSize, setSelectionSubsetSize] = useState(null as number);
+   const [selectionSubsetSizePromptOpen, setSelectionSubsetSizePromptOpen] = useState(false);
+   const [selectFullFilterState, setSelectFullFilterState] = useState("n/a" as "n/a" | "checked" | "filter" | "filterSubset");
+   const [rowSelectionModel, setRowSelectionModel] = useState<GridSelectionModel>([]);
    const [columnsModel, setColumnsModel] = useState([] as GridColDef[]);
    const [rows, setRows] = useState([] as GridRowsProp[]);
    const [loading, setLoading] = useState(true);
@@ -167,8 +186,8 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    const [activeModalProcess, setActiveModalProcess] = useState(null as QProcessMetaData);
    const [launchingProcess, setLaunchingProcess] = useState(launchProcess);
    const [recordIdsForProcess, setRecordIdsForProcess] = useState(null as string | QQueryFilter);
-   const [columnStatsFieldName, setColumnStatsFieldName] = useState(null as string)
+   const [columnStatsFieldName, setColumnStatsFieldName] = useState(null as string);
-   const [filterForColumnStats, setFilterForColumnStats] = useState(null as QQueryFilter)
+   const [filterForColumnStats, setFilterForColumnStats] = useState(null as QQueryFilter);
 
    const instance = useRef({timer: null});
 
@@ -184,6 +203,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    const [queryErrors, setQueryErrors] = useState({} as any);
    const [receivedQueryErrorTimestamp, setReceivedQueryErrorTimestamp] = useState(new Date());
 
+
    const {setPageHeader} = useContext(QContext);
    const [, forceUpdate] = useReducer((x) => x + 1, 0);
 
@@ -296,7 +316,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    if (tableMetaData && tableMetaData.name !== tableName)
    {
       console.log("  it looks like we changed tables - try to reload the things");
-      setTableMetaData(null)
+      setTableMetaData(null);
       setColumnSortModel([]);
       setColumnVisibilityModel({});
       setColumnsModel([]);
@@ -310,13 +330,111 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    // first time we call in here, we may not yet have set it in state (but will have fetched it async) //
    // so we'll pass in the local version of it!                                                        //
    //////////////////////////////////////////////////////////////////////////////////////////////////////
-   const buildQFilter = (tableMetaData: QTableMetaData, filterModel: GridFilterModel) =>
+   const buildQFilter = (tableMetaData: QTableMetaData, filterModel: GridFilterModel, limit?: number) =>
    {
-      const filter = FilterUtils.buildQFilterFromGridFilter(tableMetaData, filterModel, columnSortModel);
+      const filter = FilterUtils.buildQFilterFromGridFilter(tableMetaData, filterModel, columnSortModel, limit);
       setHasValidFilters(filter.criteria && filter.criteria.length > 0);
       return (filter);
    };
 
+   const getVisibleJoinTables = (): Set<string> =>
+   {
+      const visibleJoinTables = new Set<string>();
+      columnsModel.forEach((gridColumn) =>
+      {
+         const fieldName = gridColumn.field;
+         if (columnVisibilityModel[fieldName] !== false)
+         {
+            if (fieldName.indexOf(".") > -1)
+            {
+               visibleJoinTables.add(fieldName.split(".")[0]);
+            }
+         }
+      });
+
+      filterModel.items.forEach((item) =>
+      {
+         // todo - some test if there is a value?  see FilterUtils.buildQFilterFromGridFilter (re-use if needed)
+
+         const fieldName = item.columnField;
+         if(fieldName.indexOf(".") > -1)
+         {
+            visibleJoinTables.add(fieldName.split(".")[0]);
+         }
+      });
+
+      return (visibleJoinTables);
+   };
+
+   const isJoinMany = (tableMetaData: QTableMetaData, visibleJoinTables: Set<string>): boolean =>
+   {
+      if (tableMetaData?.exposedJoins)
+      {
+         for (let i = 0; i < tableMetaData.exposedJoins.length; i++)
+         {
+            const join = tableMetaData.exposedJoins[i];
+            if (visibleJoinTables.has(join.joinTable.name))
+            {
+               if(join.isMany)
+               {
+                  return (true);
+               }
+            }
+         }
+      }
+      return (false);
+   }
+
+   const getPageHeader = (tableMetaData: QTableMetaData, visibleJoinTables: Set<string>): string | JSX.Element =>
+   {
+      if (visibleJoinTables.size > 0)
+      {
+         let joinLabels = [];
+         if (tableMetaData?.exposedJoins)
+         {
+            for (let i = 0; i < tableMetaData.exposedJoins.length; i++)
+            {
+               const join = tableMetaData.exposedJoins[i];
+               if (visibleJoinTables.has(join.joinTable.name))
+               {
+                  joinLabels.push(join.label);
+               }
+            }
+         }
+
+         let joinLabelsString = joinLabels.join(", ");
+         if(joinLabels.length == 2)
+         {
+            let lastCommaIndex = joinLabelsString.lastIndexOf(",");
+            joinLabelsString = joinLabelsString.substring(0, lastCommaIndex) + " and " + joinLabelsString.substring(lastCommaIndex + 1);
+         }
+         if(joinLabels.length > 2)
+         {
+            let lastCommaIndex = joinLabelsString.lastIndexOf(",");
+            joinLabelsString = joinLabelsString.substring(0, lastCommaIndex) + ", and " + joinLabelsString.substring(lastCommaIndex + 1);
+         }
+
+         let tooltipHTML = <div>
+            You are viewing results from the {tableMetaData.label} table joined with {joinLabels.length} other table{joinLabels.length == 1 ? "" : "s"}:
+            <ul style={{marginLeft: "1rem"}}>
+               {joinLabels.map((name) => <li key={name}>{name}</li>)}
+            </ul>
+         </div>
+
+         return(
+            <div>
+               {tableMetaData?.label}
+               <CustomWidthTooltip title={tooltipHTML}>
+                  <IconButton sx={{p: 0, fontSize: "0.5rem", mb: 1, color: "#9f9f9f", fontVariationSettings: "'wght' 100"}}><Icon fontSize="small">emergency</Icon></IconButton>
+               </CustomWidthTooltip>
+            </div>);
+      }
+      else
+      {
+         return (tableMetaData?.label);
+      }
+   };
+
    const updateTable = () =>
    {
       setLoading(true);
@@ -324,7 +442,29 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
       (async () =>
       {
          const tableMetaData = await qController.loadTableMetaData(tableName);
-         setPageHeader(tableMetaData.label);
+
+         const visibleJoinTables = getVisibleJoinTables();
+         setPageHeader(getPageHeader(tableMetaData, visibleJoinTables));
+
+         if (!didDefaultVisibilityModelComeFromLocalStorage)
+         {
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // if we didn't load the column visibility from local storage, then by default, it'll be an empty array, and all fields will be visible. //
+            // but - if the table has join tables, we don't want them on by default, so, flip them off!                                              //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            if (tableMetaData?.exposedJoins)
+            {
+               for (let i = 0; i < tableMetaData.exposedJoins.length; i++)
+               {
+                  const join = tableMetaData.exposedJoins[i];
+                  for (let fieldName of join.joinTable.fields.keys())
+                  {
+                     columnVisibilityModel[`${join.joinTable.name}.${fieldName}`] = false;
+                  }
+               }
+            }
+            setColumnVisibilityModel(columnVisibilityModel);
+         }
 
          ////////////////////////////////////////////////////////////////////////////////////////////////
          // we need the table meta data to look up the default filter (if it comes from query string), //
@@ -347,12 +487,40 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
 
          if (columnsModel.length == 0)
          {
-            let linkBase = metaData.getTablePath(table)
+            let linkBase = metaData.getTablePath(table);
             linkBase += linkBase.endsWith("/") ? "" : "/";
-            const columns = DataGridUtils.setupGridColumns(tableMetaData, null, linkBase);
+            const columns = DataGridUtils.setupGridColumns(tableMetaData, linkBase, metaData, "alphabetical");
             setColumnsModel(columns);
+
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // let the next render (since columnsModel is watched below) build the filter, using the new columnsModel (in case of joins) //
+            ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            return;
          }
 
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         // make sure that any if any sort columns are from a join table, that the join table is visible //
+         //////////////////////////////////////////////////////////////////////////////////////////////////
+         let resetColumnSortModel = false;
+         for (let i = 0; i < columnSortModel.length; i++)
+         {
+            const gridSortItem = columnSortModel[i];
+            if (gridSortItem.field.indexOf(".") > -1)
+            {
+               const tableName = gridSortItem.field.split(".")[0];
+               if (!visibleJoinTables?.has(tableName))
+               {
+                  columnSortModel.splice(i, 1);
+                  setColumnSortModel(columnSortModel);
+                  resetColumnSortModel = true;
+                  i--;
+               }
+            }
+         }
+
+         ///////////////////////////////////////////////////////////
+         // if there's no column sort, make a default - pkey desc //
+         ///////////////////////////////////////////////////////////
          if (columnSortModel.length === 0)
          {
             columnSortModel.push({
@@ -360,9 +528,39 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                sort: "desc",
             });
             setColumnSortModel(columnSortModel);
+            resetColumnSortModel = true;
+         }
+
+         if (resetColumnSortModel && latestQueryId > 0)
+         {
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            // let the next render (since columnSortModel is watched below) build the filter, using the new columnSort //
+            /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+            return;
          }
 
          const qFilter = buildQFilter(tableMetaData, localFilterModel);
+         qFilter.skip = pageNumber * rowsPerPage;
+         qFilter.limit = rowsPerPage;
+
+         //////////////////////////////////////////
+         // figure out joins to use in the query //
+         //////////////////////////////////////////
+         let queryJoins = null;
+         if (tableMetaData?.exposedJoins)
+         {
+            const visibleJoinTables = getVisibleJoinTables();
+
+            queryJoins = [];
+            for (let i = 0; i < tableMetaData.exposedJoins.length; i++)
+            {
+               const join = tableMetaData.exposedJoins[i];
+               if (visibleJoinTables.has(join.joinTable.name))
+               {
+                  queryJoins.push(new QueryJoin(join.joinTable.name, true, "LEFT"));
+               }
+            }
+         }
 
          //////////////////////////////////////////////////////////////////////////////////////////////////
          // assign a new query id to the query being issued here.  then run both the count & query async //
@@ -374,15 +572,19 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          console.log(`Issuing query: ${thisQueryId}`);
          if (tableMetaData.capabilities.has(Capability.TABLE_COUNT))
          {
-            qController.count(tableName, qFilter).then((count) =>
+            let includeDistinct = isJoinMany(tableMetaData, getVisibleJoinTables());
+            qController.count(tableName, qFilter, queryJoins, includeDistinct).then(([count, distinctCount]) =>
             {
-               countResults[thisQueryId] = count;
+               console.log(`Received count results for query ${thisQueryId}: ${count} ${distinctCount}`);
+               countResults[thisQueryId] = [];
+               countResults[thisQueryId].push(count);
+               countResults[thisQueryId].push(distinctCount);
                setCountResults(countResults);
                setReceivedCountTimestamp(new Date());
             });
          }
 
-         qController.query(tableName, qFilter, rowsPerPage, pageNumber * rowsPerPage).then((results) =>
+         qController.query(tableName, qFilter, queryJoins).then((results) =>
          {
             console.log(`Received results for query ${thisQueryId}`);
             queryResults[thisQueryId] = results;
@@ -422,7 +624,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    ///////////////////////////
    useEffect(() =>
    {
-      if (countResults[latestQueryId] === null)
+      if (countResults[latestQueryId] == null || countResults[latestQueryId].length == 0)
       {
          ///////////////////////////////////////////////
          // see same idea in displaying query results //
@@ -430,8 +632,16 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          console.log(`No count results for id ${latestQueryId}...`);
          return;
       }
-      setTotalRecords(countResults[latestQueryId]);
+      try
+      {
+         setTotalRecords(countResults[latestQueryId][0]);
+         setDistinctRecords(countResults[latestQueryId][1]);
          delete countResults[latestQueryId];
+      }
+      catch(e)
+      {
+         console.log(e);
+      }
    }, [receivedCountTimestamp]);
 
    ///////////////////////////
@@ -454,8 +664,20 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
       delete queryResults[latestQueryId];
       setLatestQueryResults(results);
 
-      const {rows, columnsToRender} = DataGridUtils.makeRows(results, tableMetaData);
+      ///////////////////////////////////////////////////////////
+      // count how many distinct primary keys are on this page //
+      ///////////////////////////////////////////////////////////
+      let distinctPrimaryKeySet = new Set<string>();
+      for(let i = 0; i < results.length; i++)
+      {
+         distinctPrimaryKeySet.add(results[i].values.get(tableMetaData.primaryKeyField) as string);
+      }
+      setDistinctRecordsOnPageCount(distinctPrimaryKeySet.size);
 
+      ////////////////////////////////
+      // make the rows for the grid //
+      ////////////////////////////////
+      const rows = DataGridUtils.makeRows(results, tableMetaData);
       setRows(rows);
 
       setLoading(false);
@@ -561,19 +783,22 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
 
    const selectionChanged = (selectionModel: GridSelectionModel, details: GridCallbackDetails) =>
    {
-      const newSelectedIds: string[] = [];
+      ////////////////////////////////////////////////////
+      // since we manage this object, we must re-set it //
+      ////////////////////////////////////////////////////
+      setRowSelectionModel(selectionModel);
+
+      let checkboxesChecked = 0;
+      let selectedPrimaryKeys = new Set<string>();
       selectionModel.forEach((value: GridRowId, index: number) =>
       {
-         let valueToPush = value as string;
+         checkboxesChecked++
-         if(tableMetaData.primaryKeyField !== "id")
+         const valueToPush = latestQueryResults[value as number].values.get(tableMetaData.primaryKeyField);
-         {
+         selectedPrimaryKeys.add(valueToPush as string);
-            valueToPush = latestQueryResults[index].values.get(tableMetaData.primaryKeyField);
-         }
-         newSelectedIds.push(valueToPush as string);
       });
-      setSelectedIds(newSelectedIds);
+      setSelectedIds([...selectedPrimaryKeys.values()]);
 
-      if (newSelectedIds.length === rowsPerPage)
+      if (checkboxesChecked === rowsPerPage)
       {
          setSelectFullFilterState("checked");
       }
@@ -595,6 +820,19 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
       }
    };
 
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   // if, after a column was turned on or off, the set of visibleJoinTables is changed, then update the table //
+   // check this on each render - it should only be different if there was a change.  note that putting this  //
+   // in handleColumnVisibilityChange "didn't work" - it was always "behind by one" (like, maybe data grid    //
+   // calls that function before it updates the visible model or some-such).                                  //
+   /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+   const newVisibleJoinTables = getVisibleJoinTables();
+   if (JSON.stringify([...newVisibleJoinTables.keys()]) != JSON.stringify([...visibleJoinTables.keys()]))
+   {
+      updateTable();
+      setVisibleJoinTables(newVisibleJoinTables);
+   }
+
    const handleColumnOrderChange = (columnOrderChangeParams: GridColumnOrderChangeParams) =>
    {
       // TODO: make local storaged
@@ -666,7 +904,6 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                columnsModel.forEach((gridColumn) =>
                {
                   const fieldName = gridColumn.field;
-                  // @ts-ignore
                   if (columnVisibilityModel[fieldName] !== false)
                   {
                      visibleFields.push(fieldName);
@@ -741,6 +978,10 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    {
       if (selectFullFilterState === "filter")
       {
+         if(isJoinMany(tableMetaData, getVisibleJoinTables()))
+         {
+            return (distinctRecords);
+         }
          return (totalRecords);
       }
 
@@ -754,6 +995,11 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          return `?recordsParam=filterJSON&filterJSON=${JSON.stringify(buildQFilter(tableMetaData, filterModel))}`;
       }
 
+      if (selectFullFilterState === "filterSubset")
+      {
+         return `?recordsParam=filterJSON&filterJSON=${JSON.stringify(buildQFilter(tableMetaData, filterModel, selectionSubsetSize))}`;
+      }
+
       if (selectedIds.length > 0)
       {
          return `?recordsParam=recordIds&recordIds=${selectedIds.join(",")}`;
@@ -768,6 +1014,10 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
       {
          setRecordIdsForProcess(buildQFilter(tableMetaData, filterModel));
       }
+      else if (selectFullFilterState === "filterSubset")
+      {
+         setRecordIdsForProcess(buildQFilter(tableMetaData, filterModel, selectionSubsetSize));
+      }
       else if (selectedIds.length > 0)
       {
          setRecordIdsForProcess(selectedIds.join(","));
@@ -859,6 +1109,18 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    // @ts-ignore
    const defaultLabelDisplayedRows = ({from, to, count}) =>
    {
+      const tooltipHTML = <>
+         The number of rows shown on this screen may be greater than the number of {tableMetaData?.label} records
+         that match your query, because you have included fields from other tables which may have
+         more than one record associated with each {tableMetaData?.label}.
+      </>
+      let distinctPart = isJoinMany(tableMetaData, getVisibleJoinTables()) ? (<Box display="inline" textAlign="right">
+         &nbsp;({distinctRecords} distinct<CustomWidthTooltip title={tooltipHTML}>
+            <IconButton sx={{p: 0, pl: 0.25, mb: 0.25}}><Icon fontSize="small" sx={{fontSize: "1.125rem !important", color: "#9f9f9f"}}>info_outlined</Icon></IconButton>
+         </CustomWidthTooltip>
+         )
+      </Box>) : <></>;
+
       if (tableMetaData && !tableMetaData.capabilities.has(Capability.TABLE_COUNT))
       {
          ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -879,13 +1141,21 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
       {
          if (count === 0)
          {
-            return (loading ? "Counting records..." : "No rows");
+            return (loading ? "Counting..." : "No rows");
          }
-         return (`Showing ${from.toLocaleString()} to ${to.toLocaleString()} of ${count !== -1 ? `${count.toLocaleString()} records` : `more than ${to.toLocaleString()} records`}`);
+
+         return <>
+            Showing {from.toLocaleString()} to {to.toLocaleString()} of
+            {
+               count == -1 ?
+                  <>more than {to.toLocaleString()}</>
+                  : <> {count.toLocaleString()}{distinctPart}</>
+            }
+         </>;
       }
       else
       {
-         return ("Counting records...");
+         return ("Counting...");
       }
    };
 
@@ -895,7 +1165,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          <TablePagination
             component="div"
             // note - passing null here makes the 'to' param in the defaultLabelDisplayedRows also be null,
-            // so pass some sentinel value...
+            // so pass a sentinel value of -1...
             count={totalRecords === null || totalRecords === undefined ? -1 : totalRecords}
             page={pageNumber}
             rowsPerPageOptions={[10, 25, 50, 100, 250]}
@@ -982,7 +1252,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
 
          if (counter > 0)
          {
-            await navigator.clipboard.writeText(data)
+            await navigator.clipboard.writeText(data);
             setSuccessAlert(`Copied ${counter} ${qFieldMetaData.label} value${counter == 1 ? "" : "s"}.`);
          }
          else
@@ -991,13 +1261,13 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          }
          setTimeout(() => setSuccessAlert(null), 3000);
       }
-   }
+   };
 
    const openColumnStatistics = async (column: GridColDef) =>
    {
       setFilterForColumnStats(buildQFilter(tableMetaData, filterModel));
       setColumnStatsFieldName(column.field);
-   }
+   };
 
    const CustomColumnMenu = forwardRef<HTMLUListElement, GridColumnMenuProps>(
       function GridColumnMenu(props: GridColumnMenuProps, ref)
@@ -1028,7 +1298,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                <MenuItem sx={{justifyContent: "space-between"}} onClick={(e) =>
                {
                   hideMenu(e);
-                  copyColumnValues(currentColumn)
+                  copyColumnValues(currentColumn);
                }}>
                   Copy values
 
@@ -1053,6 +1323,96 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          );
       });
 
+   ////////////////////////////////////////////////////////////////////////////
+   // this is a WIP example of how we could do a custom "columns" panel/menu //
+   ////////////////////////////////////////////////////////////////////////////
+   const CustomColumnsPanel = forwardRef<any, GridColumnsPanelProps>(
+      function MyCustomColumnsPanel(props: GridColumnsPanelProps, ref)
+      {
+         const apiRef = useGridApiContext();
+         const columns = useGridSelector(apiRef, gridColumnDefinitionsSelector);
+         const columnVisibilityModel = useGridSelector(apiRef, gridColumnVisibilityModelSelector);
+
+         const [openGroups, setOpenGroups] = useState({} as { [name: string]: boolean });
+
+         const groups = ["Order", "Line Item"];
+
+         const onColumnVisibilityChange = (fieldName: string) =>
+         {
+            /*
+            if(columnVisibilityModel[fieldName] === undefined)
+            {
+               columnVisibilityModel[fieldName] = true;
+            }
+            columnVisibilityModel[fieldName] = !columnVisibilityModel[fieldName];
+            setColumnVisibilityModel(JSON.parse(JSON.stringify(columnVisibilityModel)))
+            */
+
+            console.log(`${fieldName} = ${columnVisibilityModel[fieldName]}`);
+            // columnVisibilityModel[fieldName] = Math.random() < 0.5;
+            apiRef.current.setColumnVisibility(fieldName, columnVisibilityModel[fieldName] === false);
+            // handleColumnVisibilityChange(JSON.parse(JSON.stringify(columnVisibilityModel)));
+         };
+
+         const toggleColumnGroup = (groupName: string) =>
+         {
+            if (openGroups[groupName] === undefined)
+            {
+               openGroups[groupName] = true;
+            }
+            openGroups[groupName] = !openGroups[groupName];
+            setOpenGroups(JSON.parse(JSON.stringify(openGroups)));
+         };
+
+         return (
+            <div ref={ref} className="custom-columns-panel" style={{width: "350px", height: "450px"}}>
+               <Box height="55px" padding="5px">
+                  <TextField label="Find column" placeholder="Column title" variant="standard" fullWidth={true}></TextField>
+               </Box>
+               <Box overflow="auto" height="calc( 100% - 105px )">
+
+                  <Stack direction="column" spacing={1} pl="0.5rem">
+
+                     {groups.map((groupName: string) =>
+                        (
+                           <>
+                              <IconButton
+                                 key={groupName}
+                                 size="small"
+                                 onClick={() => toggleColumnGroup(groupName)}
+                                 sx={{width: "100%", justifyContent: "flex-start", fontSize: "0.875rem"}}
+                                 disableRipple={true}
+                              >
+                                 <Icon>{openGroups[groupName] === false ? "expand_less" : "expand_more"}</Icon>
+                                 <Box sx={{pl: "0.25rem", fontWeight: "bold"}} textAlign="left">{groupName} fields</Box>
+                              </IconButton>
+
+                              {openGroups[groupName] !== false && columnsModel.map((gridColumn: any) => (
+                                 <IconButton
+                                    key={gridColumn.field}
+                                    size="small"
+                                    onClick={() => onColumnVisibilityChange(gridColumn.field)}
+                                    sx={{width: "100%", justifyContent: "flex-start", fontSize: "0.875rem", pl: "1.375rem"}}
+                                    disableRipple={true}
+                                 >
+                                    <Icon>{columnVisibilityModel[gridColumn.field] === false ? "visibility_off" : "visibility"}</Icon>
+                                    <Box sx={{pl: "0.25rem"}} textAlign="left">{gridColumn.headerName}</Box>
+                                 </IconButton>
+                              ))}
+                           </>
+                        ))}
+
+                  </Stack>
+               </Box>
+               <Box height="50px" padding="5px" display="flex" justifyContent="space-between">
+                  <Button>hide all</Button>
+                  <Button>show all</Button>
+               </Box>
+            </div>
+         );
+      }
+   );
+
    function CustomToolbar()
    {
       const handleMouseDown: GridEventListener<"cellMouseDown"> = (
@@ -1085,14 +1445,84 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
          setGridPreferencesWindow(preferencePanelState.openedPanelValue);
       });
 
+      const joinIsMany = isJoinMany(tableMetaData, visibleJoinTables);
+
+      const safeToLocaleString = (n: Number): string =>
+      {
+         if(n != null && n != undefined)
+         {
+            return (n.toLocaleString());
+         }
+         return ("");
+      }
+
+      const selectionMenuOptions: string[] = [];
+      selectionMenuOptions.push(`This page (${safeToLocaleString(distinctRecordsOnPageCount)} ${joinIsMany ? "distinct " : ""}record${distinctRecordsOnPageCount == 1 ? "" : "s"})`);
+      selectionMenuOptions.push(`Full query result (${joinIsMany ? safeToLocaleString(distinctRecords) + ` distinct record${distinctRecords == 1 ? "" : "s"}` : safeToLocaleString(totalRecords) + ` record${totalRecords == 1 ? "" : "s"}`})`);
+      selectionMenuOptions.push(`Subset of the query result ${selectionSubsetSize ? `(${safeToLocaleString(selectionSubsetSize)} ${joinIsMany ? "distinct " : ""}record${selectionSubsetSize == 1 ? "" : "s"})` : "..."}`);
+      selectionMenuOptions.push("Clear selection");
+
+      function programmaticallySelectSomeOrAllRows(max?: number)
+      {
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         // any time the user selects one of the options from our selection menu,                 //
+         // we want to check all the boxes on the screen - and - "select" all of the primary keys //
+         // unless they did the subset option - then we'll only go up to a 'max' number           //
+         ///////////////////////////////////////////////////////////////////////////////////////////
+         const rowSelectionModel: GridSelectionModel = [];
+         let selectedPrimaryKeys = new Set<string>();
+         rows.forEach((value: GridRowModel, index: number) =>
+         {
+            const primaryKeyValue = latestQueryResults[index].values.get(tableMetaData.primaryKeyField);
+            if(max)
+            {
+               if(selectedPrimaryKeys.size < max)
+               {
+                  if(!selectedPrimaryKeys.has(primaryKeyValue))
+                  {
+                     rowSelectionModel.push(value.__rowIndex);
+                     selectedPrimaryKeys.add(primaryKeyValue as string);
+                  }
+               }
+            }
+            else
+            {
+               rowSelectionModel.push(value.__rowIndex);
+               selectedPrimaryKeys.add(primaryKeyValue as string);
+            }
+         });
+         setRowSelectionModel(rowSelectionModel);
+         setSelectedIds([...selectedPrimaryKeys.values()]);
+      }
+
+      const selectionMenuCallback = (selectedIndex: number) =>
+      {
+         if(selectedIndex == 0)
+         {
+            programmaticallySelectSomeOrAllRows();
+            setSelectFullFilterState("checked")
+         }
+         else if(selectedIndex == 1)
+         {
+            programmaticallySelectSomeOrAllRows();
+            setSelectFullFilterState("filter")
+         }
+         else if(selectedIndex == 2)
+         {
+            setSelectionSubsetSizePromptOpen(true);
+         }
+         else if(selectedIndex == 3)
+         {
+            setSelectFullFilterState("n/a")
+            setRowSelectionModel([]);
+            setSelectedIds([]);
+         }
+      };
+
       return (
          <GridToolbarContainer>
             <div>
-               <Button
+               <Button id="refresh-button" onClick={updateTable} startIcon={<Icon>refresh</Icon>} sx={{pr: "1.25rem"}}>
-                  id="refresh-button"
-                  onClick={updateTable}
-                  startIcon={<Icon>refresh</Icon>}
-               >
                   Refresh
                </Button>
             </div>
@@ -1106,19 +1536,27 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                         <Tooltip title="Clear All Filters">
                            <Icon sx={{cursor: "pointer"}} onClick={() => setShowClearFiltersWarning(true)}>clear</Icon>
                         </Tooltip>
-                        <Dialog open={showClearFiltersWarning} onClose={() => setShowClearFiltersWarning(false)}>
+                        <Dialog open={showClearFiltersWarning} onClose={() => setShowClearFiltersWarning(false)} onKeyPress={(e) =>
+                        {
+                           if (e.key == "Enter")
+                           {
+                              setShowClearFiltersWarning(false)
+                              navigate(metaData.getTablePathByName(tableName));
+                              handleFilterChange({items: []} as GridFilterModel);
+                           }
+                        }}>
                            <DialogTitle id="alert-dialog-title">Confirm</DialogTitle>
                            <DialogContent>
                               <DialogContentText>Are you sure you want to clear all filters?</DialogContentText>
                            </DialogContent>
                            <DialogActions>
-                              <Button onClick={() => setShowClearFiltersWarning(false)}>No</Button>
+                              <QCancelButton label="No" disabled={false} onClickHandler={() => setShowClearFiltersWarning(false)} />
-                              <Button onClick={() =>
+                              <QSaveButton label="Yes" iconName="check" disabled={false} onClickHandler={() =>
                               {
                                  setShowClearFiltersWarning(false);
                                  navigate(metaData.getTablePathByName(tableName));
                                  handleFilterChange({items: []} as GridFilterModel);
-                              }}>Yes</Button>
+                              }}/>
                            </DialogActions>
                         </Dialog>
                      </div>
@@ -1131,32 +1569,67 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                   <ExportMenuItem format="json" />
                </GridToolbarExportContainer>
             </div>
+
+            <div>
+               <MenuButton label="Selection" iconName={selectedIds.length == 0 ? "check_box_outline_blank" : "check_box"} disabled={totalRecords == 0} options={selectionMenuOptions} callback={selectionMenuCallback}/>
+               <SelectionSubsetDialog isOpen={selectionSubsetSizePromptOpen} initialValue={selectionSubsetSize} closeHandler={(value) =>
+               {
+                  setSelectionSubsetSizePromptOpen(false);
+
+                  if(value !== undefined)
+                  {
+                     if(typeof value === "number" && value > 0)
+                     {
+                        programmaticallySelectSomeOrAllRows(value);
+                        setSelectionSubsetSize(value);
+                        setSelectFullFilterState("filterSubset")
+                     }
+                     else
+                     {
+                        setAlertContent("Unexpected value: " + value);
+                     }
+                  }
+               }} />
+            </div>
+
             <div>
                {
                   selectFullFilterState === "checked" && (
                      <div className="selectionTool">
                         The
                         <strong>{` ${selectedIds.length.toLocaleString()} `}</strong>
-                        records on this page are selected.
+                        {joinIsMany ? " distinct " : ""}
-                        <Button onClick={() => setSelectFullFilterState("filter")}>
+                        record{selectedIds.length == 1 ? "" : "s"} on this page {selectedIds.length == 1 ? "is" : "are"} selected.
-                           Select all
-                           {` ${totalRecords ? totalRecords.toLocaleString() : ""} `}
-                           records matching this query
-                        </Button>
                      </div>
                   )
                }
                {
                   selectFullFilterState === "filter" && (
                      <div className="selectionTool">
-                        All
+                        {
-                        <strong>{` ${totalRecords ? totalRecords.toLocaleString() : ""} `}</strong>
+                           (joinIsMany
-                        records matching this query are selected.
+                              ? (
-                        <Button onClick={() => setSelectFullFilterState("checked")}>
+                                 distinctRecords == 1
-                           Select the
+                                    ? (<>The <strong>only 1</strong> distinct record matching this query is selected.</>)
-                           {` ${selectedIds.length.toLocaleString()} `}
+                                    : (<>All <strong>{(distinctRecords ? distinctRecords.toLocaleString() : "")}</strong> distinct records matching this query are selected.</>)
-                           records on this page
+                              )
-                        </Button>
+                              : (<>All <strong>{totalRecords ? totalRecords.toLocaleString() : ""}</strong> records matching this query are selected.</>)
+                           )
+                        }
+                     </div>
+                  )
+               }
+               {
+                  selectFullFilterState === "filterSubset" && (
+                     <div className="selectionTool">
+                        The <a onClick={() => setSelectionSubsetSizePromptOpen(true)} style={{cursor: "pointer"}}><strong>first {safeToLocaleString(selectionSubsetSize)}</strong></a> {joinIsMany ? "distinct" : ""} record{selectionSubsetSize == 1 ? "" : "s"} matching this query {selectionSubsetSize == 1 ? "is" : "are"} selected.
+                     </div>
+                  )
+               }
+               {
+                  (selectFullFilterState === "n/a" && selectedIds.length > 0) && (
+                     <div className="selectionTool">
+                        <strong>{safeToLocaleString(selectedIds.length)}</strong> {joinIsMany ? "distinct" : ""} {selectedIds.length == 1 ? "record is" : "records are"} selected.
                      </div>
                   )
                }
@@ -1174,20 +1647,20 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
       {
          menuItems.push(<Divider />);
       }
-   }
+   };
 
    const menuItems: JSX.Element[] = [];
    if (table.capabilities.has(Capability.TABLE_INSERT) && table.insertPermission)
    {
-      menuItems.push(<MenuItem onClick={bulkLoadClicked}><ListItemIcon><Icon>library_add</Icon></ListItemIcon>Bulk Load</MenuItem>)
+      menuItems.push(<MenuItem onClick={bulkLoadClicked}><ListItemIcon><Icon>library_add</Icon></ListItemIcon>Bulk Load</MenuItem>);
    }
    if (table.capabilities.has(Capability.TABLE_UPDATE) && table.editPermission)
    {
-      menuItems.push(<MenuItem onClick={bulkEditClicked}><ListItemIcon><Icon>edit</Icon></ListItemIcon>Bulk Edit</MenuItem>)
+      menuItems.push(<MenuItem onClick={bulkEditClicked}><ListItemIcon><Icon>edit</Icon></ListItemIcon>Bulk Edit</MenuItem>);
    }
    if (table.capabilities.has(Capability.TABLE_DELETE) && table.deletePermission)
    {
-      menuItems.push(<MenuItem onClick={bulkDeleteClicked}><ListItemIcon><Icon>delete</Icon></ListItemIcon>Bulk Delete</MenuItem>)
+      menuItems.push(<MenuItem onClick={bulkDeleteClicked}><ListItemIcon><Icon>delete</Icon></ListItemIcon>Bulk Delete</MenuItem>);
    }
 
    const runRecordScriptProcess = metaData?.processes.get("runRecordScript");
@@ -1212,7 +1685,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
 
    if (menuItems.length === 0)
    {
-      menuItems.push(<MenuItem disabled><ListItemIcon><Icon>block</Icon></ListItemIcon><i>No actions available</i></MenuItem>)
+      menuItems.push(<MenuItem disabled><ListItemIcon><Icon>block</Icon></ListItemIcon><i>No actions available</i></MenuItem>);
    }
 
    const renderActionsMenu = (
@@ -1256,8 +1729,9 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    useEffect(() =>
    {
       setTotalRecords(null);
+      setDistinctRecords(null);
       updateTable();
-   }, [ tableState, filterModel]);
+   }, [columnsModel, tableState, filterModel]);
 
    useEffect(() =>
    {
@@ -1343,7 +1817,7 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                <Card>
                   <Box height="100%">
                      <DataGridPro
-                        components={{Toolbar: CustomToolbar, Pagination: CustomPagination, LoadingOverlay: Loading, ColumnMenu: CustomColumnMenu}}
+                        components={{Toolbar: CustomToolbar, Pagination: CustomPagination, LoadingOverlay: Loading, ColumnMenu: CustomColumnMenu/*, ColumnsPanel: CustomColumnsPanel*/}}
                         pinnedColumns={pinnedColumns}
                         onPinnedColumnsChange={handlePinnedColumnsChange}
                         pagination
@@ -1374,6 +1848,9 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
                         sortingOrder={["asc", "desc"]}
                         sortModel={columnSortModel}
                         getRowClassName={(params) => (params.indexRelativeToCurrentPage % 2 === 0 ? "even" : "odd")}
+                        getRowId={(row) => row.__rowIndex}
+                        selectionModel={rowSelectionModel}
+                        hideFooterSelectedRowCount={true}
                      />
                   </Box>
                </Card>
@@ -1410,4 +1887,49 @@ function RecordQuery({table, launchProcess}: Props): JSX.Element
    );
 }
 
+//////////////////////////////////////////////////////////////////////////////////
+// mini-component that is the dialog for the user to enter the selection-subset //
+//////////////////////////////////////////////////////////////////////////////////
+function SelectionSubsetDialog(props: {isOpen: boolean; initialValue: number; closeHandler: (value?: number) => void})
+{
+   const [value, setValue] = useState(props.initialValue)
+
+   const handleChange = (newValue: string) =>
+   {
+      setValue(parseInt(newValue))
+   }
+
+   const keyPressed = (e: React.KeyboardEvent<HTMLDivElement>) =>
+   {
+      if(e.key == "Enter" && value)
+      {
+         props.closeHandler(value);
+      }
+   }
+
+   return (
+      <Dialog open={props.isOpen} onClose={() => props.closeHandler()} onKeyPress={(e) => keyPressed(e)}>
+         <DialogTitle>Subset of the Query Result</DialogTitle>
+         <DialogContent>
+            <DialogContentText>How many records do you want to select?</DialogContentText>
+            <TextField
+               autoFocus
+               name="selection-subset-size"
+               inputProps={{width: "100%", type: "number", min: 1}}
+               onChange={(e) => handleChange(e.target.value)}
+               value={value}
+               sx={{width: "100%"}}
+               onFocus={event => event.target.select()}
+            />
+         </DialogContent>
+         <DialogActions>
+            <QCancelButton disabled={false} onClickHandler={() => props.closeHandler()} />
+            <QSaveButton label="OK" iconName="check" disabled={value == undefined || isNaN(value)} onClickHandler={() => props.closeHandler(value)} />
+         </DialogActions>
+      </Dialog>
+   )
+}
+
+
+
 export default RecordQuery;

src/qqq/styles/qqq-override-styles.css

@@ -124,7 +124,6 @@
 
 .MuiDataGrid-toolbarContainer .selectionTool
 {
-   margin-left: 40px;
    font-size: 14px;
 }
 

src/qqq/utils/DataGridUtils.tsx

@@ -22,10 +22,12 @@
 import {AdornmentType} from "@kingsrook/qqq-frontend-core/lib/model/metaData/AdornmentType";
 import {QFieldMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QFieldMetaData";
 import {QFieldType} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QFieldType";
+import {QInstance} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QInstance";
 import {QTableMetaData} from "@kingsrook/qqq-frontend-core/lib/model/metaData/QTableMetaData";
 import {QRecord} from "@kingsrook/qqq-frontend-core/lib/model/QRecord";
 import {getGridDateOperators, GridColDef, GridRowsProp} from "@mui/x-data-grid-pro";
 import {GridFilterOperator} from "@mui/x-data-grid/models/gridFilterOperator";
+import React from "react";
 import {Link} from "react-router-dom";
 import {buildQGridPvsOperators, QGridBooleanOperators, QGridNumericOperators, QGridStringOperators} from "qqq/pages/records/query/GridFilterOperators";
 import ValueUtils from "qqq/utils/qqq/ValueUtils";
@@ -36,24 +38,39 @@ export default class DataGridUtils
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static makeRows = (results: QRecord[], tableMetaData: QTableMetaData): {rows: GridRowsProp[], columnsToRender: any} =>
+   public static makeRows = (results: QRecord[], tableMetaData: QTableMetaData): GridRowsProp[] =>
    {
       const fields = [ ...tableMetaData.fields.values() ];
       const rows = [] as any[];
-      const columnsToRender = {} as any;
+      let rowIndex = 0;
       results.forEach((record: QRecord) =>
       {
          const row: any = {};
+         row.__rowIndex = rowIndex++;
+
          fields.forEach((field) =>
          {
-            const value = ValueUtils.getDisplayValue(field, record, "query");
+            row[field.name] = ValueUtils.getDisplayValue(field, record, "query");
-            if (typeof value !== "string")
-            {
-               columnsToRender[field.name] = true;
-            }
-            row[field.name] = value;
          });
 
+         if(tableMetaData.exposedJoins)
+         {
+            for (let i = 0; i < tableMetaData.exposedJoins.length; i++)
+            {
+               const join = tableMetaData.exposedJoins[i];
+
+               if(join?.joinTable?.fields?.values())
+               {
+                  const fields = [...join.joinTable.fields.values()];
+                  fields.forEach((field) =>
+                  {
+                     let fieldName = join.joinTable.name + "." + field.name;
+                     row[fieldName] = ValueUtils.getDisplayValue(field, record, "query", fieldName);
+                  });
+               }
+            }
+         }
+
          if(!row["id"])
          {
             row["id"] = record.values.get(tableMetaData.primaryKeyField) ?? row[tableMetaData.primaryKeyField];
@@ -69,29 +86,53 @@ export default class DataGridUtils
          rows.push(row);
       });
 
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+      return (rows);
-      // do this secondary check for columnsToRender - in case we didn't have any rows above, and our check for string isn't enough. //
-      // ... shouldn't this be just based on the field definition anyway... ?  plus adornments?                                      //
-      /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-      fields.forEach((field) =>
-      {
-         if(field.possibleValueSourceName)
-         {
-            columnsToRender[field.name] = true;
-         }
-      });
-
-      return ({rows, columnsToRender});
    }
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static setupGridColumns = (tableMetaData: QTableMetaData, columnsToRender: any, linkBase: string = ""): GridColDef[] =>
+   public static setupGridColumns = (tableMetaData: QTableMetaData, linkBase: string = "", metaData?: QInstance, columnSort: "bySection" | "alphabetical" = "alphabetical"): GridColDef[] =>
    {
       const columns = [] as GridColDef[];
+      this.addColumnsForTable(tableMetaData, linkBase, columns, columnSort, null, null);
+
+      if(tableMetaData.exposedJoins)
+      {
+         for (let i = 0; i < tableMetaData.exposedJoins.length; i++)
+         {
+            const join = tableMetaData.exposedJoins[i];
+
+            let joinLinkBase = null;
+            if(metaData)
+            {
+               joinLinkBase = metaData.getTablePath(join.joinTable);
+               joinLinkBase += joinLinkBase.endsWith("/") ? "" : "/";
+            }
+
+            if(join?.joinTable?.fields?.values())
+            {
+               this.addColumnsForTable(join.joinTable, joinLinkBase, columns, columnSort, join.joinTable.name + ".", join.label + ": ");
+            }
+         }
+      }
+
+      return (columns);
+   };
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static addColumnsForTable(tableMetaData: QTableMetaData, linkBase: string, columns: GridColDef[], columnSort: "bySection" | "alphabetical" = "alphabetical", namePrefix?: string, labelPrefix?: string)
+   {
       const sortedKeys: string[] = [];
 
+      ////////////////////////////////////////////////////////////////////////
+      // this sorted by sections - e.g., manual sorting by the meta-data... //
+      ////////////////////////////////////////////////////////////////////////
+      if(columnSort === "bySection")
+      {
          for (let i = 0; i < tableMetaData.sections.length; i++)
          {
             const section = tableMetaData.sections[i];
@@ -105,34 +146,46 @@ export default class DataGridUtils
                sortedKeys.push(section.fieldNames[j]);
             }
          }
+      }
+      else // columnSort = "alphabetical"
+      {
+         ///////////////////////////
+         // sort by labels... mmm //
+         ///////////////////////////
+         sortedKeys.push(...tableMetaData.fields.keys())
+         sortedKeys.sort((a: string, b: string): number =>
+         {
+            return (tableMetaData.fields.get(a).label.localeCompare(tableMetaData.fields.get(b).label))
+         })
+      }
 
       sortedKeys.forEach((key) =>
       {
          const field = tableMetaData.fields.get(key);
-         const column = this.makeColumnFromField(field, tableMetaData, columnsToRender);
+         const column = this.makeColumnFromField(field, tableMetaData, namePrefix, labelPrefix);
 
-         if (key === tableMetaData.primaryKeyField && linkBase)
+         if(key === tableMetaData.primaryKeyField && linkBase && namePrefix == null)
          {
             columns.splice(0, 0, column);
-            column.renderCell = (cellValues: any) => (
-               <Link to={`${linkBase}${cellValues.value}`}>{cellValues.value}</Link>
-            );
          }
          else
          {
             columns.push(column);
          }
+
+         if (key === tableMetaData.primaryKeyField && linkBase)
+         {
+            column.renderCell = (cellValues: any) => (
+               <Link to={`${linkBase}${cellValues.value}`} onClick={(e) => e.stopPropagation()}>{cellValues.value}</Link>
+            );
+         }
       });
-
+   }
-      return (columns);
-   };
-
-
 
    /*******************************************************************************
     **
     *******************************************************************************/
-   public static makeColumnFromField = (field: QFieldMetaData, tableMetaData: QTableMetaData, columnsToRender: any): GridColDef =>
+   public static makeColumnFromField = (field: QFieldMetaData, tableMetaData: QTableMetaData, namePrefix?: string, labelPrefix?: string): GridColDef =>
    {
       let columnType = "string";
       let columnWidth = 200;
@@ -198,24 +251,21 @@ export default class DataGridUtils
          }
       }
 
+      let headerName = labelPrefix ? labelPrefix + field.label : field.label;
+      let fieldName = namePrefix ? namePrefix + field.name : field.name;
+
       const column = {
-         field: field.name,
+         field: fieldName,
          type: columnType,
-         headerName: field.label,
+         headerName: headerName,
          width: columnWidth,
          renderCell: null as any,
          filterOperators: filterOperators,
       };
 
-      /////////////////////////////////////////////////////////////////////////////////////////
-      // looks like, maybe we can just always render all columns, and remove this parameter? //
-      /////////////////////////////////////////////////////////////////////////////////////////
-      if (columnsToRender == null || columnsToRender[field.name])
-      {
       column.renderCell = (cellValues: any) => (
          (cellValues.value)
       );
-      }
 
       return (column);
    }

src/qqq/utils/qqq/FilterUtils.ts

@@ -514,7 +514,7 @@ class FilterUtils
    /*******************************************************************************
     ** build a qqq filter from a grid and column sort model
     *******************************************************************************/
-   public static buildQFilterFromGridFilter(tableMetaData: QTableMetaData, filterModel: GridFilterModel, columnSortModel: GridSortItem[]): QQueryFilter
+   public static buildQFilterFromGridFilter(tableMetaData: QTableMetaData, filterModel: GridFilterModel, columnSortModel: GridSortItem[], limit?: number): QQueryFilter
    {
       console.log("Building q filter with model:");
       console.log(filterModel);
@@ -528,6 +528,12 @@ class FilterUtils
          });
       }
 
+      if (limit)
+      {
+         console.log("Setting limit to: " + limit);
+         qFilter.limit = limit;
+      }
+
       if (filterModel)
       {
          let foundFilter = false;

src/qqq/utils/qqq/ValueUtils.tsx

@@ -70,10 +70,12 @@ class ValueUtils
     ** When you have a field, and a record - call this method to get a string or
     ** element back to display the field's value.
     *******************************************************************************/
-   public static getDisplayValue(field: QFieldMetaData, record: QRecord, usage: "view" | "query" = "view"): string | JSX.Element | JSX.Element[]
+   public static getDisplayValue(field: QFieldMetaData, record: QRecord, usage: "view" | "query" = "view", overrideFieldName?: string): string | JSX.Element | JSX.Element[]
    {
-      const displayValue = record.displayValues ? record.displayValues.get(field.name) : undefined;
+      const fieldName = overrideFieldName ?? field.name;
-      const rawValue = record.values ? record.values.get(field.name) : undefined;
+
+      const displayValue = record.displayValues ? record.displayValues.get(fieldName) : undefined;
+      const rawValue = record.values ? record.values.get(fieldName) : undefined;
 
       return ValueUtils.getValueForDisplay(field, rawValue, displayValue, usage);
    }

src/test/java/com/kingsrook/qqq/materialdashboard/tests/QueryScreenTest.java

@@ -128,7 +128,7 @@ public class QueryScreenTest extends QBaseSeleniumTest
       String expectedFilterContents1 = """
          {"fieldName":"firstName","operator":"CONTAINS","values":["Jam"]}""";
       String expectedFilterContents2 = """
-         "booleanOperator":"OR"}""";
+         "booleanOperator":"OR\"""";
 
       qSeleniumJavalin.waitForCapturedPathWithBodyContaining("/data/person/query", expectedFilterContents0);
       qSeleniumJavalin.waitForCapturedPathWithBodyContaining("/data/person/query", expectedFilterContents1);