UI table many rows (finos#1115)

* refacor scroll in prep for multi million row scrolling * prepare for vitrual scrolling in big datasets * ignore type issue with ArrayProxy * fix test errors caused by reversion key order in dataSOurce
ScottLogic · Jan 10, 2024 · 760263d · 760263d
1 parent 9e7e631
commit 760263d
Show file tree

Hide file tree

Showing 23 changed files with 3,019 additions and 361 deletions.
diff --git a/vuu-ui/packages/vuu-data-local/src/array-data-source/array-data-source.ts b/vuu-ui/packages/vuu-data-local/src/array-data-source/array-data-source.ts
@@ -439,9 +439,7 @@ export class ArrayDataSource
   }
 
   set range(range: VuuRange) {
-    if (range.from !== this.#range.from || range.to !== this.#range.to) {
-      this.setRange(range);
-    }
+    this.setRange(range);
   }
 
   protected delete(row: VuuRowDataItemType[]) {
@@ -492,9 +490,13 @@ export class ArrayDataSource
   };
 
   private setRange(range: VuuRange, forceFullRefresh = false) {
-    this.#range = range;
-    this.keys.reset(range);
-    this.sendRowsToClient(forceFullRefresh);
+    if (range.from !== this.#range.from || range.to !== this.#range.to) {
+      this.#range = range;
+      this.keys.reset(range);
+      this.sendRowsToClient(forceFullRefresh);
+    } else if (forceFullRefresh) {
+      this.sendRowsToClient(forceFullRefresh);
+    }
   }
 
   sendRowsToClient(forceFullRefresh = false, row?: DataSourceRow) {

diff --git a/vuu-ui/packages/vuu-data-remote/src/inlined-worker.js b/vuu-ui/packages/vuu-data-remote/src/inlined-worker.js
diff --git a/vuu-ui/packages/vuu-data-remote/test/server-proxy.test.ts b/vuu-ui/packages/vuu-data-remote/test/server-proxy.test.ts
@@ -18,8 +18,8 @@ import {
   updateTableRow,
   createSubscription,
 } from "./test-utils";
-import { DataSourceDataMessage, DataSourceEnabledMessage } from "../src";
 import { VuuRow } from "@finos/vuu-protocol-types";
+import { DataSourceDataMessage } from "@finos/vuu-data-types";
 
 const SERVER_MESSAGE_CONSTANTS = {
   module: "CORE",
@@ -270,8 +270,8 @@ describe("ServerProxy", () => {
           [7,7,true,false,0,0,"key-07",0,"key-07","name 07",1007,true],
           [8,8,true,false,0,0,"key-08",0,"key-08","name 08",1008,true],
           [9,9,true,false,0,0,"key-09",0,"key-09","name 09",1009,true],
-          [10,1,true,false,0,0,"key-10",0,"key-10","name 10",1010,true],
-          [11,0,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
+          [10,0,true,false,0,0,"key-10",0,"key-10","name 10",1010,true],
+          [11,1,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
         ],
       });
     });
@@ -323,16 +323,16 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [20,9,true,false,0,0,"key-20",0,"key-20","name 20",1020,true,],
-          [21,8,true,false,0,0,"key-21",0,"key-21","name 21",1021,true],
-          [22,7,true,false,0,0,"key-22",0,"key-22","name 22",1022,true],
-          [23,6,true,false,0,0,"key-23",0,"key-23","name 23",1023,true],
-          [24,5,true,false,0,0,"key-24",0,"key-24","name 24",1024,true],
-          [25,4,true,false,0,0,"key-25",0,"key-25","name 25",1025,true],
-          [26,3,true,false,0,0,"key-26",0,"key-26","name 26",1026,true],
-          [27,2,true,false,0,0,"key-27",0,"key-27","name 27",1027,true],
-          [28,1,true,false,0,0,"key-28",0,"key-28","name 28",1028,true],
-          [29,0,true,false,0,0,"key-29",0,"key-29","name 29",1029,true,],
+          [20,0,true,false,0,0,"key-20",0,"key-20","name 20",1020,true,],
+          [21,1,true,false,0,0,"key-21",0,"key-21","name 21",1021,true],
+          [22,2,true,false,0,0,"key-22",0,"key-22","name 22",1022,true],
+          [23,3,true,false,0,0,"key-23",0,"key-23","name 23",1023,true],
+          [24,4,true,false,0,0,"key-24",0,"key-24","name 24",1024,true],
+          [25,5,true,false,0,0,"key-25",0,"key-25","name 25",1025,true],
+          [26,6,true,false,0,0,"key-26",0,"key-26","name 26",1026,true],
+          [27,7,true,false,0,0,"key-27",0,"key-27","name 27",1027,true],
+          [28,8,true,false,0,0,"key-28",0,"key-28","name 28",1028,true],
+          [29,9,true,false,0,0,"key-29",0,"key-29","name 29",1029,true,],
         ],
       });
     });
@@ -499,15 +499,15 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [11,8,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
-          [12,7,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
-          [13,6,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
-          [14,5,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
+          [11,0,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
+          [12,1,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
+          [13,2,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
+          [14,3,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
           [15,4,true,false,0,0,"key-15",0,"key-15","name 15",1015,true,],
-          [16,3,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
-          [17,2,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
-          [18,1,true,false,0,0,"key-18",0,"key-18","name 18",1018,true],
-          [19,0,true,false,0,0,"key-19",0,"key-19","name 19",1019,true],
+          [16,5,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
+          [17,6,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
+          [18,7,true,false,0,0,"key-18",0,"key-18","name 18",1018,true],
+          [19,8,true,false,0,0,"key-19",0,"key-19","name 19",1019,true],
         ],
       });
     });
@@ -707,26 +707,26 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [4975,19,true,false,0,0,"key-75",0,"key-75","name 75",5975,true],
-          [4976,18,true,false,0,0,"key-76",0,"key-76","name 76",5976,true],
-          [4977,17,true,false,0,0,"key-77",0,"key-77","name 77",5977,true],
-          [4978,16,true,false,0,0,"key-78",0,"key-78","name 78",5978,true],
-          [4979,15,true,false,0,0,"key-79",0,"key-79","name 79",5979,true],
-          [4980,14,true,false,0,0,"key-80",0,"key-80","name 80",5980,true],
-          [4981,13,true,false,0,0,"key-81",0,"key-81","name 81",5981,true],
-          [4982,12,true,false,0,0,"key-82",0,"key-82","name 82",5982,true],
-          [4983,11,true,false,0,0,"key-83",0,"key-83","name 83",5983,true],
-          [4984,10,true,false,0,0,"key-84",0,"key-84","name 84",5984,true],
-          [4985,9,true,false,0,0,"key-85",0,"key-85","name 85",5985,true],
-          [4986,8,true,false,0,0,"key-86",0,"key-86","name 86",5986,true],
-          [4987,7,true,false,0,0,"key-87",0,"key-87","name 87",5987,true],
-          [4988,6,true,false,0,0,"key-88",0,"key-88","name 88",5988,true],
-          [4989,5,true,false,0,0,"key-89",0,"key-89","name 89",5989,true],
-          [4990,4,true,false,0,0,"key-90",0,"key-90","name 90",5990,true],
-          [4991,3,true,false,0,0,"key-91",0,"key-91","name 91",5991,true],
-          [4992,2,true,false,0,0,"key-92",0,"key-92","name 92",5992,true],
-          [4993,1,true,false,0,0,"key-93",0,"key-93","name 93",5993,true],
-          [4994,0,true,false,0,0,"key-94",0,"key-94","name 94",5994,true],
+          [4975,0,true,false,0,0,"key-75",0,"key-75","name 75",5975,true],
+          [4976,1,true,false,0,0,"key-76",0,"key-76","name 76",5976,true],
+          [4977,2,true,false,0,0,"key-77",0,"key-77","name 77",5977,true],
+          [4978,3,true,false,0,0,"key-78",0,"key-78","name 78",5978,true],
+          [4979,4,true,false,0,0,"key-79",0,"key-79","name 79",5979,true],
+          [4980,5,true,false,0,0,"key-80",0,"key-80","name 80",5980,true],
+          [4981,6,true,false,0,0,"key-81",0,"key-81","name 81",5981,true],
+          [4982,7,true,false,0,0,"key-82",0,"key-82","name 82",5982,true],
+          [4983,8,true,false,0,0,"key-83",0,"key-83","name 83",5983,true],
+          [4984,9,true,false,0,0,"key-84",0,"key-84","name 84",5984,true],
+          [4985,10,true,false,0,0,"key-85",0,"key-85","name 85",5985,true],
+          [4986,11,true,false,0,0,"key-86",0,"key-86","name 86",5986,true],
+          [4987,12,true,false,0,0,"key-87",0,"key-87","name 87",5987,true],
+          [4988,13,true,false,0,0,"key-88",0,"key-88","name 88",5988,true],
+          [4989,14,true,false,0,0,"key-89",0,"key-89","name 89",5989,true],
+          [4990,15,true,false,0,0,"key-90",0,"key-90","name 90",5990,true],
+          [4991,16,true,false,0,0,"key-91",0,"key-91","name 91",5991,true],
+          [4992,17,true,false,0,0,"key-92",0,"key-92","name 92",5992,true],
+          [4993,18,true,false,0,0,"key-93",0,"key-93","name 93",5993,true],
+          [4994,19,true,false,0,0,"key-94",0,"key-94","name 94",5994,true],
           [4995,20,true,false,0,0,"key-95",0,"key-95","name 95",5995,true],
           [4996,21,true,false,0,0,"key-96",0,"key-96","name 96",5996,true],
           [4997,22,true,false,0,0,"key-97",0,"key-97","name 97",5997,true],
@@ -770,8 +770,8 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [10,1,true,false,0,0,"key-10",0,"key-10","name 10",1010,true],
-          [11,0,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
+          [10,0,true,false,0,0,"key-10",0,"key-10","name 10",1010,true],
+          [11,1,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
         ],
       });
 
@@ -793,9 +793,9 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [12,4,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
+          [12,2,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
           [13,3,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
-          [14,2,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
+          [14,4,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
         ],
       });
 
@@ -818,9 +818,9 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [15,7,true,false,0,0,"key-15",0,"key-15","name 15",1015,true],
+          [15,5,true,false,0,0,"key-15",0,"key-15","name 15",1015,true],
           [16,6,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
-          [17,5,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
+          [17,7,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
         ],
       });
 
@@ -911,8 +911,8 @@ describe("ServerProxy", () => {
           [7,7,true,false,0,0,"key-07",0,"key-07","name 07",1007,true],
           [8,8,true,false,0,0,"key-08",0,"key-08","name 08",1008,true],
           [9,9,true,false,0,0,"key-09",0,"key-09","name 09",1009,true],
-          [10,1,true,false,0,0,"key-10",0,"key-10","name 10",1010,true],
-          [11,0,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
+          [10,0,true,false,0,0,"key-10",0,"key-10","name 10",1010,true],
+          [11,1,true,false,0,0,"key-11",0,"key-11","name 11",1011,true],
         ],
       });
     });
@@ -1023,16 +1023,16 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [12,0,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
-          [13,1,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
-          [14,2,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
-          [15,3,true,false,0,0,"key-15",0,"key-15","name 15",1015,true],
-          [16,4,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
-          [17,5,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
-          [18,6,true,false,0,0,"key-18",0,"key-18","name 18",1018,true],
-          [19,7,true,false,0,0,"key-19",0,"key-19","name 19",1019,true],
-          [20,9,true,false,0,0,"key-20",0,"key-20","name 20",1020,true],
-          [21,8,true,false,0,0,"key-21",0,"key-21","name 21",1021,true],
+          [12,2,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
+          [13,3,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
+          [14,4,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
+          [15,5,true,false,0,0,"key-15",0,"key-15","name 15",1015,true],
+          [16,6,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
+          [17,7,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
+          [18,8,true,false,0,0,"key-18",0,"key-18","name 18",1018,true],
+          [19,9,true,false,0,0,"key-19",0,"key-19","name 19",1019,true],
+          [20,0,true,false,0,0,"key-20",0,"key-20","name 20",1020,true],
+          [21,1,true,false,0,0,"key-21",0,"key-21","name 21",1021,true],
         ],
       });
     });
@@ -1086,16 +1086,16 @@ describe("ServerProxy", () => {
         type: "viewport-update",
         clientViewportId: "client-vp-1",
         rows: [
-          [12,9,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
-          [13,8,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
-          [14,7,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
-          [15,6,true,false,0,0,"key-15",0,"key-15","name 15",1015,true],
-          [16,5,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
-          [17,4,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
-          [18,3,true,false,0,0,"key-18",0,"key-18","name 18",1018,true],
-          [19,2,true,false,0,0,"key-19",0,"key-19","name 19",1019,true],
-          [20,1,true,false,0,0,"key-20",0,"key-20","name 20",1020,true],
-          [21,0,true,false,0,0,"key-21",0,"key-21","name 21",1021,true],
+          [12,0,true,false,0,0,"key-12",0,"key-12","name 12",1012,true],
+          [13,1,true,false,0,0,"key-13",0,"key-13","name 13",1013,true],
+          [14,2,true,false,0,0,"key-14",0,"key-14","name 14",1014,true],
+          [15,3,true,false,0,0,"key-15",0,"key-15","name 15",1015,true],
+          [16,4,true,false,0,0,"key-16",0,"key-16","name 16",1016,true],
+          [17,5,true,false,0,0,"key-17",0,"key-17","name 17",1017,true],
+          [18,6,true,false,0,0,"key-18",0,"key-18","name 18",1018,true],
+          [19,7,true,false,0,0,"key-19",0,"key-19","name 19",1019,true],
+          [20,8,true,false,0,0,"key-20",0,"key-20","name 20",1020,true],
+          [21,9,true,false,0,0,"key-21",0,"key-21","name 21",1021,true],
           [22,10,true,false,0,0,"key-22",0,"key-22","name 22",1022,true],
         ],
       });

diff --git a/...showcase/src/examples/utils/ArrayProxy.ts → .../packages/vuu-data-test/src/ArrayProxy.ts b/...showcase/src/examples/utils/ArrayProxy.ts → .../packages/vuu-data-test/src/ArrayProxy.ts
diff --git a/vuu-ui/packages/vuu-data-test/src/index.ts b/vuu-ui/packages/vuu-data-test/src/index.ts
@@ -1,3 +1,4 @@
+export * from "./ArrayProxy";
 export * from "./schemas";
 export * from "./TickingArrayDataSource";
 export * from "./vuu-row-generator";

diff --git a/vuu-ui/packages/vuu-table/src/Table.css b/vuu-ui/packages/vuu-table/src/Table.css
@@ -102,7 +102,8 @@
 }
 
 .vuuTable-body {
-  height: var(--content-height)
+  height: var(--content-height);
+  position: relative;
 }
 
 .vuuPinLeft, .vuuPinRight {

diff --git a/vuu-ui/packages/vuu-table/src/Table.tsx b/vuu-ui/packages/vuu-table/src/Table.tsx
@@ -36,6 +36,7 @@ import { TableHeader } from "./table-header/TableHeader";
 
 import "./Table.css";
 import type { DragDropState } from "@finos/vuu-ui-controls";
+import { ScrollingAPI } from "./useTableScroll";
 
 const classBase = "vuuTable";
 
@@ -98,6 +99,11 @@ export interface TableProps
   onSelectionChange?: SelectionChangeHandler;
   renderBufferSize?: number;
   rowHeight?: number;
+  /**
+   * imperative API for scrolling table
+   */
+  scrollingApiRef?: ForwardedRef<ScrollingAPI>;
+
   /**
    * Selection Bookends style the left and right edge of a selection block.
    * They are optional, value defaults to zero.
@@ -132,8 +138,9 @@ const TableCore = ({
   onRowClick: onRowClickProp,
   onSelect,
   onSelectionChange,
-  renderBufferSize = 0,
+  renderBufferSize = 5,
   rowHeight = 20,
+  scrollingApiRef,
   selectionModel = "extended",
   showColumnHeaders = true,
   headerHeight = showColumnHeaders ? 25 : 0,
@@ -148,6 +155,7 @@ const TableCore = ({
     columns,
     data,
     draggableRow,
+    getRowOffset,
     handleContextMenuAction,
     headings,
     highlightedIndex,
@@ -187,11 +195,12 @@ const TableCore = ({
     onSelectionChange,
     renderBufferSize,
     rowHeight,
+    scrollingApiRef,
     selectionModel,
     size,
   });
 
-  const className = cx(`${classBase}-contentContainer`, {
+  const contentContainerClassName = cx(`${classBase}-contentContainer`, {
     [`${classBase}-colLines`]: tableAttributes.columnSeparators,
     [`${classBase}-rowLines`]: tableAttributes.rowSeparators,
     // [`${classBase}-highlight`]: tableAttributes.showHighlightedRow,
@@ -225,7 +234,7 @@ const TableCore = ({
         <div className={`${classBase}-scrollbarContent`} />
       </div>
       <div
-        className={className}
+        className={contentContainerClassName}
         ref={scrollProps.contentContainerRef}
         style={cssVariables}
       >
@@ -258,9 +267,7 @@ const TableCore = ({
                 onClick={onRowClick}
                 onDataEdited={onDataEdited}
                 row={data}
-                offset={
-                  rowHeight * data[IDX] + viewportMeasurements.totalHeaderHeight
-                }
+                offset={getRowOffset(data)}
                 onToggleGroup={onToggleGroup}
                 zebraStripes={tableAttributes.zebraStripes}
               />
@@ -296,6 +303,7 @@ export const Table = forwardRef(function TableNext(
     onSelectionChange,
     renderBufferSize,
     rowHeight,
+    scrollingApiRef,
     selectionModel,
     showColumnHeaders,
     headerHeight,
@@ -349,6 +357,7 @@ export const Table = forwardRef(function TableNext(
           onSelectionChange={onSelectionChange}
           renderBufferSize={renderBufferSize}
           rowHeight={rowHeight}
+          scrollingApiRef={scrollingApiRef}
           selectionModel={selectionModel}
           showColumnHeaders={showColumnHeaders}
           size={size}

diff --git a/vuu-ui/packages/vuu-table/src/index.ts b/vuu-ui/packages/vuu-table/src/index.ts
@@ -7,4 +7,5 @@ export * from "./cell-renderers";
 export type { RowProps } from "./Row";
 export * from "./useControlledTableNavigation";
 export * from "./useTableModel";
+export * from "./useTableScroll";
 export * from "./useTableViewport";
diff --git a/vuu-ui/packages/vuu-table/src/table-dom-utils.ts b/vuu-ui/packages/vuu-table/src/table-dom-utils.ts
@@ -1,12 +1,15 @@
 import { RefObject } from "react";
 
+/**
+ * [rowIndex, colIndex
+ */
 export type CellPos = [number, number];
 
 export const headerCellQuery = (colIdx: number) =>
   `.vuuTable-col-headers .vuuTableHeaderCell:nth-child(${colIdx})`;
 
 export const dataCellQuery = (rowIdx: number, colIdx: number) =>
-  `.vuuTable-body > [aria-rowindex='${rowIdx}'] > [role='cell']:nth-child(${
+  `.vuuTable-body > [aria-rowindex='${rowIdx + 1}'] > [role='cell']:nth-child(${
     colIdx + 1
   })`;