Merge remote-tracking branch 'upstream/main'

# Conflicts:
#	vuu-ui/packages/vuu-data-test/src/UpdateGenerator.ts
#	vuu-ui/packages/vuu-data-test/src/basket/basket-schemas.ts
#	vuu-ui/packages/vuu-data-test/src/basket/data-generators/basket-generator.ts
#	vuu-ui/packages/vuu-data-test/src/basket/data-generators/basketConstituent-generator.ts
#	vuu-ui/packages/vuu-data-test/src/basket/data-generators/basketTrading-generator.ts
#	vuu-ui/packages/vuu-data-test/src/basket/data-generators/basketTradingConstituent-generator.ts
#	vuu-ui/packages/vuu-data-test/src/basket/data-generators/index.ts
#	vuu-ui/packages/vuu-data-test/src/basket/reference-data/basketTrading.ts
#	vuu-ui/packages/vuu-data-test/src/createArrayDataSource.ts
#	vuu-ui/packages/vuu-data-test/src/index.ts
#	vuu-ui/packages/vuu-data-test/src/rowUpdates.ts
#	vuu-ui/packages/vuu-data/src/array-data-source/array-data-source.ts
#	vuu-ui/packages/vuu-data/src/inlined-worker.js
#	vuu-ui/packages/vuu-data/src/remote-data-source.ts
#	vuu-ui/packages/vuu-datagrid/src/cell-renderers/progress-cell.tsx
#	vuu-ui/packages/vuu-icons/index.css
#	vuu-ui/packages/vuu-layout/src/layout-persistence/LayoutPersistenceManager.ts
#	vuu-ui/packages/vuu-layout/src/layout-persistence/LocalLayoutPersistenceManager.ts
#	vuu-ui/packages/vuu-layout/src/layout-persistence/index.ts
#	vuu-ui/packages/vuu-layout/src/layout-persistence/useLayoutContextMenuItems.tsx
#	vuu-ui/packages/vuu-layout/src/overflow-container/OverflowContainer.css
#	vuu-ui/packages/vuu-layout/src/toolbar/Toolbar.css
#	vuu-ui/packages/vuu-layout/test/layout-persistence/LocalLayoutPersistenceManager.test.ts
#	vuu-ui/packages/vuu-shell/src/layout-management/LayoutList.tsx
#	vuu-ui/packages/vuu-shell/src/layout-management/layoutTypes.ts
#	vuu-ui/packages/vuu-shell/src/layout-management/useLayoutManager.tsx
#	vuu-ui/packages/vuu-shell/src/left-nav/LeftNav.css
#	vuu-ui/packages/vuu-shell/src/shell.tsx
#	vuu-ui/packages/vuu-table-extras/src/cell-renderers-next/background-cell/BackgroundCell.tsx
#	vuu-ui/packages/vuu-table-extras/src/column-formatting-settings/NumericFormattingSettings.tsx
#	vuu-ui/packages/vuu-table-extras/src/column-settings/ColumnSettingsPanel.tsx
#	vuu-ui/packages/vuu-table-extras/src/column-settings/useColumnSettings.ts
#	vuu-ui/packages/vuu-table/src/table-next/header-cell/HeaderCell.css
#	vuu-ui/packages/vuu-table/src/table-next/table-cell/TableCell.tsx
#	vuu-ui/packages/vuu-table/src/table-next/useTableContextMenu.ts
#	vuu-ui/packages/vuu-table/src/table-next/useTableNext.ts
#	vuu-ui/packages/vuu-table/src/table/dataTableTypes.ts
#	vuu-ui/packages/vuu-table/src/table/useDataSource.ts
#	vuu-ui/packages/vuu-ui-controls/src/instrument-picker/useInstrumentPicker.ts
#	vuu-ui/packages/vuu-ui-controls/src/tabstrip/Tabstrip.css
#	vuu-ui/packages/vuu-utils/src/column-utils.ts
#	vuu-ui/sample-apps/app-vuu-basket-trader/index.tsx
#	vuu-ui/sample-apps/app-vuu-basket-trader/login.css
#	vuu-ui/sample-apps/app-vuu-basket-trader/public/demo.html
#	vuu-ui/sample-apps/app-vuu-basket-trader/public/index.html
#	vuu-ui/sample-apps/app-vuu-basket-trader/public/login.html
#	vuu-ui/sample-apps/app-vuu-basket-trader/src/App.css
#	vuu-ui/sample-apps/app-vuu-basket-trader/src/App.tsx
#	vuu-ui/sample-apps/app-vuu-example/index.tsx
#	vuu-ui/sample-apps/app-vuu-example/src/App.css
#	vuu-ui/sample-apps/app-vuu-example/src/App.tsx
#	vuu-ui/sample-apps/app-vuu-example/src/useRpcResponseHandler.tsx
#	vuu-ui/sample-apps/feature-basket-trading/package.json
#	vuu-ui/sample-apps/feature-basket-trading/src/VuuBasketTradingFeature.tsx
#	vuu-ui/sample-apps/feature-basket-trading/src/basket-selector/BasketSelector.tsx
#	vuu-ui/sample-apps/feature-basket-trading/src/basket-selector/useBasketSelector.ts
#	vuu-ui/sample-apps/feature-basket-trading/src/basket-toolbar/BasketToolbar.tsx
#	vuu-ui/sample-apps/feature-basket-trading/src/new-basket-panel/NewBasketPanel.tsx
#	vuu-ui/sample-apps/feature-basket-trading/src/new-basket-panel/useNewBasketPanel.ts
#	vuu-ui/sample-apps/feature-basket-trading/src/useBasketTrading.tsx
#	vuu-ui/sample-apps/feature-basket-trading/src/useBasketTradingDatasources.ts
#	vuu-ui/sample-apps/feature-filter-table/src/useFilterTable.tsx
#	vuu-ui/showcase/src/examples/Apps/NewTheme.examples.tsx
#	vuu-ui/showcase/src/examples/DataTable/FilterTable.examples.tsx
#	vuu-ui/showcase/src/examples/Filters/FilterBar/FilterBar.examples.tsx
#	vuu-ui/showcase/src/examples/Layout/OverflowContainer.examples.tsx
#	vuu-ui/showcase/src/examples/Shell/LeftNav.examples.tsx
#	vuu-ui/showcase/src/examples/Shell/Shell.examples.tsx
#	vuu-ui/showcase/src/examples/Shell/index.ts
#	vuu-ui/showcase/src/examples/Table/BasketTables.examples.tsx
#	vuu-ui/showcase/src/examples/Table/index.ts
#	vuu-ui/showcase/src/examples/TableExtras/ColumnSettings/ColumnSettings.examples.tsx
#	vuu-ui/showcase/src/examples/UiControls/InstrumentPicker.examples.tsx
#	vuu-ui/showcase/src/examples/VuuFeatures/BasketSelector.examples.tsx
#	vuu-ui/showcase/src/examples/VuuFeatures/BasketToolbar.examples.tsx
#	vuu-ui/showcase/src/examples/VuuFeatures/BasketTradingFeature.examples.tsx
#	vuu-ui/showcase/src/examples/VuuFeatures/FilterTableFeature.examples.tsx
#	vuu-ui/showcase/src/examples/VuuFeatures/NewBasketPanel.examples.tsx
#	vuu-ui/showcase/src/examples/html/html-table-components/div-element-keyed-with-translate-inline-scrollbars-css-variables/DivElementKeyedWithTranslateInlineScrollbarsCssVariables.tsx
#	vuu-ui/showcase/src/examples/html/html-table-components/vuu-table/VuuTable.tsx
#	vuu-ui/showcase/src/examples/utils/TickingArrayDataSource.ts
#	vuu-ui/showcase/src/examples/utils/useTableConfig.ts
#	vuu-ui/showcase/src/features/BasketTrading.feature.tsx
#	vuu-ui/showcase/src/features/BasketTradingNoBaskets.feature.tsx
#	vuu-ui/showcase/src/features/BasketTradingOneBasket.feature.tsx
#	vuu/src/main/scala/org/finos/vuu/core/module/ModuleFactory.scala
#	vuu/src/main/scala/org/finos/vuu/core/module/basket/BasketModule.scala
#	vuu/src/main/scala/org/finos/vuu/core/module/basket/service/BasketService.scala
#	vuu/src/main/scala/org/finos/vuu/core/module/basket/service/BasketTradingConstituentService.scala

.gitignore

@@ -34,3 +34,4 @@ website/package-lock.json
 
 # Generated docs
 docs/contributing.md
+vuu-ui/packages/vuu-data/src/inlined-worker.js

docs/rpc/Viewport_rpc.md

@@ -0,0 +1,11 @@
+import { SvgDottySeparator } from "@site/src/components/SvgDottySeparator";
+
+# RPC Calls
+
+<SvgDottySeparator style={{marginBottom: 32}}/>
+
+## Viewport RPC
+
+```scala
+In Progress
+```

docs/rpc/rpc.md

@@ -4,20 +4,34 @@ import { SvgDottySeparator } from "@site/src/components/SvgDottySeparator";
 
 <SvgDottySeparator style={{marginBottom: 32}}/>
 
-## Menus
+## Overview of RPC
+
+There are two scopes where rpc services can be defined:
+
+- Global Scope - these are services that can be called without a viewport being created.
+- Viewport Scope - these are services that are created when a user creates a viewport
+
+## Global Scope - RPC Services
+
+[RPC Services](service.md) allow us to expose server-side functionality to a Vuu client over a low-latency web-socket connection.
+
+The Vuu client framework can discover and programmatically call these services over the WebSocket. While there is no generic UI for invoking/inspecting REST services, many components (such as the Autocomplete Search) use services as an implementation mechanism.
+
+## Global Scope - REST Services
+
+REST Services allow us to expose server-side functionality to a Vuu client. Each service is modeled in REST-ful resource fashion, and can define the following standard verbs: `get_all`, `get`, `post`, `put`, `delete`
+
+## Viewport Scope - Menu Items
 
 [Menu Items](Menu_items.md) act upon a `table`, `selection`, `row` or `cell` (these are called `scope`).
 
 Once a `menu item` is registered by a server side [`provider`](../providers_tables_viewports/providers.md), it will be automatically displayed when user right-clicks on the corresponding Vuu Grid component.
 
 Menu items may have filter expressions (applied for each individual row) that determine for which rows they are visible. If a menu item is visible, it can be invoked. On invocation, depending on the `scope` the RPC handler will receive context information about what are we acting upon.
 
-## RPC Services
-
-[RPC Services](service.md) allow us to expose server-side functionality to a Vuu client over a low-latency connection.
-
-The Vuu client framework can discover and programmatically call these services over the WebSocket. While there is no generic UI for invoking/inspecting REST services, many components (such as the Autocomplete Search) use REST services as an implementation mechanism.
+## Viewport Scope - RPC Calls
 
-## REST Services
+[Viewport RPC](Viewport_rpc.md) calls are specific service methods that we want to call on a viewport we've created. They are specific i.e. the UI component needs
+to understand the type of call that is being called. In that way they should be used in functionally specific UI components.
 
-[REST Services](#) allow us to expose server-side functionality to a Vuu client. Each service is modeled in REST-ful resource fashion, and can define the following standard verbs: `get_all`, `get`, `post`, `put`, `delete`
+They implicitly have access to the viewport and its associated tables that they are being called on.

vuu-ui/package.json

@@ -19,12 +19,10 @@
     "format": "prettier --write './**/*.{js,,mjs,jsx,css,md,json}' --config ./.prettierrc",
     "build": "node ./scripts/build-all.mjs",
     "build:app": "cd sample-apps/app-vuu-example && node scripts/build.mjs",
-    "build:app:basket": "cd sample-apps/app-vuu-basket-trader && node scripts/build.mjs",
     "build:packages": "npm run build -- --cjs --license && npm run type-defs",
     "build:packages:debug": "npm run build -- --cjs --debug && npm run type-defs -- --debug",
     "build:salt": "node ./scripts/build-salt.mjs",
     "launch:demo": "node ./scripts/launch-app.mjs",
-    "launch:demo:basket": "node ./scripts/launch-app.mjs -- --basket",
     "launch:demo:electron": "cd tools/electron && node ./node_modules/.bin/electron .",
     "launch:showcase": " cd showcase && node scripts/launch.mjs",
     "deploy:websocket-test": "node ./tools/deploy-websocket-test.mjs",

vuu-ui/packages/vuu-data-ag-grid/src/ViewportRowModelDataSource.ts

@@ -1,10 +1,10 @@
 import {
   DataSource,
   DataSourceConfig,
+  isVuuFeatureAction,
   SubscribeCallback,
   VuuFeatureMessage,
 } from "@finos/vuu-data";
-import { isVuuFeatureAction } from "@finos/vuu-data-react";
 import { Filter } from "@finos/vuu-filter-types";
 import { VuuGroupBy, VuuSort } from "@finos/vuu-protocol-types";
 import {

vuu-ui/packages/vuu-data-ag-grid/src/useViewportRowModel.ts

@@ -1,5 +1,7 @@
 import {
   DataSourceConfig,
+  isViewportMenusAction,
+  isVisualLinksAction,
   MenuRpcResponse,
   RemoteDataSource,
   VuuFeatureMessage,
@@ -8,8 +10,6 @@ import {
 } from "@finos/vuu-data";
 
 import {
-  isViewportMenusAction,
-  isVisualLinksAction,
   MenuActionConfig,
   SuggestionFetcher,
   useTypeaheadSuggestions,

vuu-ui/packages/vuu-data-react/src/hooks/index.ts

@@ -1,4 +1,5 @@
 export * from "./useDataSource";
+export * from "./useLookupValues";
 export * from "./useServerConnectionStatus";
 export * from "./useServerConnectionQuality";
 export * from "./useTypeaheadSuggestions";

vuu-ui/packages/vuu-data-react/src/hooks/useLookupValues.ts

@@ -0,0 +1,119 @@
+import {
+  ColumnDescriptor,
+  ListOption,
+  LookupTableDetails,
+} from "@finos/vuu-datagrid-types";
+import { RemoteDataSource } from "@finos/vuu-data";
+import { useShellContext } from "@finos/vuu-shell";
+import {
+  buildColumnMap,
+  isLookupRenderer,
+  isTypeDescriptor,
+} from "@finos/vuu-utils";
+import { useMemo, useState } from "react";
+
+const NO_VALUES: ListOption[] = [];
+
+const lookupValueMap = new Map<string, Promise<ListOption[]>>();
+
+const loadLookupValues = ({
+  labelColumn,
+  table,
+  valueColumn,
+}: LookupTableDetails): Promise<ListOption[]> => {
+  const tableKey = `${table.module}:${table.table}`;
+  const lookupValues = lookupValueMap.get(tableKey);
+  if (lookupValues) {
+    return lookupValues;
+  } else {
+    const promise: Promise<ListOption[]> = new Promise((resolve) => {
+      const columns = [valueColumn, labelColumn];
+      const columnMap = buildColumnMap(columns);
+      const dataSource = new RemoteDataSource({
+        bufferSize: 0,
+        table,
+      });
+      dataSource.subscribe(
+        {
+          columns,
+          range: { from: 0, to: 100 },
+        },
+        (message) => {
+          if (message.type === "viewport-update") {
+            //TODO check we have full dataset
+            if (message.rows) {
+              const listOptions = message.rows.map<ListOption>((row) => ({
+                value: row[columnMap[valueColumn]] as string | number,
+                label: row[columnMap[labelColumn]] as string,
+              }));
+              resolve(listOptions);
+              dataSource.unsubscribe();
+            }
+          }
+        }
+      );
+    });
+    lookupValueMap.set(tableKey, promise);
+    return promise;
+  }
+};
+
+type LookupState = {
+  initialValue: ListOption | null;
+  values: ListOption[];
+};
+
+const getSelectedOption = (
+  values: ListOption[],
+  selectedValue: string | number | undefined
+) => {
+  if (selectedValue === undefined) {
+    return null;
+  }
+  return values.find((option) => option.value === selectedValue) ?? null;
+};
+
+const getLookupDetails = ({ name, type }: ColumnDescriptor) => {
+  if (isTypeDescriptor(type) && isLookupRenderer(type.renderer)) {
+    return type.renderer.lookup;
+  } else {
+    throw Error(
+      `useLookupValues column ${name} is not configured to use lookup values`
+    );
+  }
+};
+
+export const useLookupValues = (
+  column: ColumnDescriptor,
+  initialValueProp: number | string
+) => {
+  const lookupDetails = getLookupDetails(column);
+  const { getLookupValues } = useShellContext();
+
+  const initialState = useMemo<LookupState>(() => {
+    const values = getLookupValues?.(lookupDetails.table) ?? NO_VALUES;
+    return {
+      initialValue: getSelectedOption(values, initialValueProp),
+      values,
+    };
+  }, [getLookupValues, initialValueProp, lookupDetails.table]);
+
+  const [{ initialValue, values }, setLookupState] =
+    useState<LookupState>(initialState);
+
+  useMemo(() => {
+    if (values === NO_VALUES) {
+      loadLookupValues(lookupDetails).then((values) =>
+        setLookupState({
+          initialValue: getSelectedOption(values, initialValueProp),
+          values,
+        })
+      );
+    }
+  }, [values, lookupDetails, initialValueProp]);
+
+  return {
+    initialValue,
+    values,
+  };
+};

vuu-ui/packages/vuu-data-react/src/hooks/useVuuMenuActions.ts

@@ -53,34 +53,6 @@ const { KEY } = metadataKeys;
 
 const NO_CONFIG: MenuActionConfig = {};
 
-export const isVisualLinksAction = (
-  action: GridAction
-): action is DataSourceVisualLinksMessage => action.type === "vuu-links";
-
-export const isVisualLinkCreatedAction = (
-  action: GridAction
-): action is DataSourceVisualLinkCreatedMessage =>
-  action.type === "vuu-link-created";
-
-export const isVisualLinkRemovedAction = (
-  action: GridAction
-): action is DataSourceVisualLinkRemovedMessage =>
-  action.type === "vuu-link-removed";
-
-export const isViewportMenusAction = (
-  action: GridAction
-): action is DataSourceMenusMessage => action.type === "vuu-menu";
-
-export const isVuuFeatureAction = (
-  action: GridAction
-): action is VuuFeatureMessage =>
-  isViewportMenusAction(action) || isVisualLinksAction(action);
-
-export const isVuuFeatureInvocation = (
-  action: GridAction
-): action is VuuFeatureInvocationMessage =>
-  action.type === "vuu-link-created" || action.type === "vuu-link-removed";
-
 const isMenuItem = (menu: VuuMenuItem | VuuMenu): menu is VuuMenuItem =>
   "rpcName" in menu;
 
@@ -318,11 +290,12 @@ export const useVuuMenuActions = ({
 }: VuuMenuActionHookProps): ViewServerHookResult => {
   const buildViewserverMenuOptions: MenuBuilder = useCallback(
     (location, options) => {
-      const { visualLink, visualLinks, vuuMenu } = menuActionConfig;
+      const { links, menu } = dataSource;
+      const { visualLink } = menuActionConfig;
       const descriptors: ContextMenuItemDescriptor[] = [];
 
-      if (location === "grid" && visualLinks && !visualLink) {
-        visualLinks.forEach((linkDescriptor: LinkDescriptorWithLabel) => {
+      if (location === "grid" && links && !visualLink) {
+        links.forEach((linkDescriptor: LinkDescriptorWithLabel) => {
           const { link, label: linkLabel } = linkDescriptor;
           const label = linkLabel ? linkLabel : link.toTable;
           descriptors.push({
@@ -333,13 +306,13 @@ export const useVuuMenuActions = ({
         });
       }
 
-      if (vuuMenu && isTableLocation(location)) {
+      if (menu && isTableLocation(location)) {
         const menuDescriptor = buildMenuDescriptor(
-          vuuMenu,
+          menu,
           location,
           options as VuuServerMenuOptions
         );
-        if (isRoot(vuuMenu) && isGroupMenuItemDescriptor(menuDescriptor)) {
+        if (isRoot(menu) && isGroupMenuItemDescriptor(menuDescriptor)) {
           descriptors.push(...menuDescriptor.children);
         } else if (menuDescriptor) {
           descriptors.push(menuDescriptor);
@@ -348,7 +321,7 @@ export const useVuuMenuActions = ({
 
       return descriptors;
     },
-    [menuActionConfig]
+    [dataSource, menuActionConfig]
   );
 
   const handleMenuAction = useCallback(

vuu-ui/packages/vuu-data-react/src/hooks/useVuuTables.ts

@@ -18,9 +18,7 @@ export const useVuuTables = () => {
       const { tables } = await server.getTableList();
       const tableSchemas = buildTables(
         await Promise.all(
-          tables.map((tableDescriptor) =>
-            server.getTableSchema(tableDescriptor)
-          )
+          tables.map((vuuTable) => server.getTableSchema(vuuTable))
         )
       );
       setTables(tableSchemas);