hpcc-systems · GordonSmith · Jul 11, 2024 · Jul 8, 2024
diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json
diff --git a/esp/src/package.json b/esp/src/package.json
@@ -41,22 +41,22 @@
     "@fluentui/react-hooks": "8.7.0",
     "@fluentui/react-icons-mdl2": "1.3.59",
     "@fluentui/react-migration-v8-v9": "9.6.3",
-    "@hpcc-js/chart": "2.83.3",
-    "@hpcc-js/codemirror": "2.62.0",
-    "@hpcc-js/common": "2.71.17",
-    "@hpcc-js/comms": "2.92.3",
-    "@hpcc-js/dataflow": "8.1.6",
-    "@hpcc-js/eclwatch": "2.74.5",
-    "@hpcc-js/graph": "2.85.15",
-    "@hpcc-js/html": "2.42.20",
-    "@hpcc-js/layout": "2.49.22",
-    "@hpcc-js/map": "2.77.21",
-    "@hpcc-js/other": "2.15.22",
-    "@hpcc-js/phosphor": "2.18.8",
-    "@hpcc-js/react": "2.53.16",
-    "@hpcc-js/tree": "2.40.17",
-    "@hpcc-js/util": "2.51.0",
-    "@hpcc-js/wasm": "2.17.1",
+    "@hpcc-js/chart": "2.83.4",
+    "@hpcc-js/codemirror": "2.62.1",
+    "@hpcc-js/common": "2.71.18",
+    "@hpcc-js/comms": "2.93.0",
+    "@hpcc-js/dataflow": "8.1.7",
+    "@hpcc-js/eclwatch": "2.74.8",
+    "@hpcc-js/graph": "2.85.16",
+    "@hpcc-js/html": "2.42.21",
+    "@hpcc-js/layout": "2.49.23",
+    "@hpcc-js/map": "2.77.22",
+    "@hpcc-js/other": "2.15.23",
+    "@hpcc-js/phosphor": "2.18.9",
+    "@hpcc-js/react": "2.53.17",
+    "@hpcc-js/tree": "2.40.18",
+    "@hpcc-js/util": "2.51.1",
+    "@hpcc-js/wasm": "2.18.0",
     "@kubernetes/client-node": "0.20.0",
     "clipboard": "2.0.11",
     "d3-dsv": "3.0.1",

diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx
@@ -6,7 +6,7 @@ import { useConst } from "@fluentui/react-hooks";
 import { bundleIcon, Folder20Filled, Folder20Regular, FolderOpen20Filled, FolderOpen20Regular, TextCaseTitleRegular, TextCaseTitleFilled } from "@fluentui/react-icons";
 import { Database } from "@hpcc-js/common";
 import { WorkunitsServiceEx, IScope, splitMetric } from "@hpcc-js/comms";
-import { DBStore, Table } from "@hpcc-js/dgrid";
+import { CellFormatter, ColumnFormat, ColumnType, DBStore, RowType, Table } from "@hpcc-js/dgrid";
 import { compare, scopedLogger } from "@hpcc-js/util";
 import nlsHPCC from "src/nlsHPCC";
 import { WUTimelineNoFetch } from "src/Timings";
@@ -36,6 +36,16 @@ const defaultUIState = {
     hasSelection: false
 };
 
+class ColumnFormatEx extends ColumnFormat {
+    formatterFunc(): CellFormatter | undefined {
+        const colIdx = this._owner.columns().indexOf("__StdDevs");
+
+        return function (this: ColumnType, cell: any, row: RowType): string {
+            return row[colIdx];
+        };
+    }
+}
+
 class DBStoreEx extends DBStore {
 
     constructor(protected _table: TableEx, db: Database.Grid) {
@@ -82,7 +92,17 @@ class TableEx extends Table {
     metrics(metrics: any[], options: MetricsOptionsT, timelineFilter: string, scopeFilter: string, matchCase: boolean): this {
         this
             .columns(["##"])    //  Reset hash to force recalculation of default widths
-            .columns(["##", nlsHPCC.Type, nlsHPCC.Scope, ...options.properties])
+            .columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...options.properties, "__StdDevs"])
+            .columnFormats([
+                new ColumnFormatEx()
+                    .column("StdDevs")
+                    .paletteID("StdDevs")
+                    .min(0)
+                    .max(6),
+                new ColumnFormat()
+                    .column("__StdDevs")
+                    .width(0)
+            ])
             .data(metrics
                 .filter(m => this.scopeFilterFunc(m, scopeFilter, matchCase))
                 .filter(row => {
@@ -91,21 +111,21 @@ class TableEx extends Table {
                 }).map((row, idx) => {
                     if (idx === 0) {
                         this._rawDataMap = {
-                            0: "##", 1: "type", 2: "name"
+                            0: "##", 1: "type", 2: "__StdDevs", 3: "name"
                         };
                         options.properties.forEach((p, idx2) => {
-                            this._rawDataMap[3 + idx2] = p;
+                            this._rawDataMap[4 + idx2] = p;
                         });
                     }
                     row.__hpcc_id = row.name;
-                    return [idx, row.type, row.name, ...options.properties.map(p => {
+                    return [idx, row.type, row.__StdDevs === 0 ? undefined : row.__StdDevs, row.name, ...options.properties.map(p => {
                         return row.__groupedProps[p]?.Value ??
                             row.__groupedProps[p]?.Max ??
                             row.__groupedProps[p]?.Avg ??
                             row.__formattedProps[p] ??
                             row[p] ??
                             "";
-                    }), row];
+                    }), row.__StdDevsSource, row];
                 }))
             ;
         return this;

diff --git a/esp/src/src-react/components/MetricsPropertiesTables.tsx b/esp/src/src-react/components/MetricsPropertiesTables.tsx
@@ -1,11 +1,15 @@
 import * as React from "react";
 import { useConst } from "@fluentui/react-hooks";
+import { Palette } from "@hpcc-js/common";
 import { IScope } from "@hpcc-js/comms";
-import { Table } from "@hpcc-js/dgrid";
+import { ColumnFormat, Table } from "@hpcc-js/dgrid";
+import { formatDecimal } from "src/Utility";
 import nlsHPCC from "src/nlsHPCC";
 import { AutosizeHpccJSComponent } from "../layouts/HpccJSAdapter";
 
-interface MetricsPropertiesTablesProps {
+Palette.rainbow("StdDevs", ["white", "white", "#fff0f0", "#ffC0C0", "#ff8080", "#ff0000", "#ff0000"]);
+
+export interface MetricsPropertiesTablesProps {
     scopesTableColumns?: string[];
     scopes?: IScope[];
 }
@@ -21,7 +25,18 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT
 
     //  Props Table  ---
     const propsTable = useConst(() => new Table()
-        .columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax"])
+        .columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax", "StdDevs"])
+        .columnFormats([
+            new ColumnFormat()
+                .column(nlsHPCC.Property)
+                .paletteID("StdDevs")
+                .min(0)
+                .max(6)
+                .valueColumn("StdDevs"),
+            new ColumnFormat()
+                .column("StdDevs")
+                .width(0)
+        ])
         .sortable(true)
     );
 
@@ -31,7 +46,7 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT
             const scopeProps = [];
             for (const key in item.__groupedProps) {
                 const row = item.__groupedProps[key];
-                scopeProps.push([row.Key, row.Value, row.Avg, row.Min, row.Max, row.Delta, row.StdDev, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax]);
+                scopeProps.push([row.Key, row.Value, row.Avg, row.Min, row.Max, row.Delta, row.StdDev === undefined ? "" : `${row.StdDev} (${formatDecimal(row.StdDevs)}σ)`, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax, row.StdDevs]);
             }
             scopeProps.sort((l, r) => {
                 const lIdx = sortByColumns.indexOf(l[0]);
@@ -53,7 +68,7 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT
 
         propsTable
             .columns([])
-            .columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax"])
+            .columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax", "StdDevs"])
             .data(props)
             .lazyRender()
             ;

diff --git a/esp/src/src-react/components/MetricsSQL.tsx b/esp/src/src-react/components/MetricsSQL.tsx
@@ -26,15 +26,7 @@ export const MetricsSQL: React.FunctionComponent<MetricsDataProps> = ({
     onSelectionChanged
 }) => {
 
-    const cleanScopes = React.useMemo(() => {
-        return scopes.map(scope => {
-            const retVal = { ...scope };
-            delete retVal.__children;
-            return retVal;
-        });
-    }, [scopes]);
-
-    const connection = useDuckDBConnection(cleanScopes, "metrics");
+    const connection = useDuckDBConnection(scopes, "metrics");
     const [schema, setSchema] = React.useState<any[]>([]);
     const [sql, setSql] = React.useState<string>(defaultSql);
     const [sqlError, setSqlError] = React.useState<Error | undefined>();
@@ -91,7 +83,7 @@ export const MetricsSQL: React.FunctionComponent<MetricsDataProps> = ({
 
     //  Query  ---
     React.useEffect(() => {
-        if (cleanScopes.length === 0) {
+        if (scopes.length === 0) {
             setSchema([]);
             setData([]);
         } else if (connection) {
@@ -117,7 +109,7 @@ export const MetricsSQL: React.FunctionComponent<MetricsDataProps> = ({
                 scopesTable.noDataMessage(nlsHPCC.noDataMessage);
             });
         }
-    }, [cleanScopes.length, connection, scopesTable, sql]);
+    }, [connection, scopes.length, scopesTable, sql]);
 
     //  Selection  ---
     const onChange = React.useCallback((newSql: string) => {

diff --git a/esp/src/src-react/hooks/duckdb.ts b/esp/src/src-react/hooks/duckdb.ts
@@ -1,4 +1,5 @@
 import * as React from "react";
+import { IScope } from "@hpcc-js/comms";
 import { DuckDB } from "@hpcc-js/wasm/dist/duckdb";
 
 type AsyncDuckDB = any;
@@ -22,7 +23,7 @@ export function useDuckDB(): [AsyncDuckDB] {
     return [db];
 }
 
-export function useDuckDBConnection<T>(scopes: T, name: string): AsyncDuckDBConnection | undefined {
+export function useDuckDBConnection(scopes: IScope[], name: string): AsyncDuckDBConnection | undefined {
 
     const [db] = useDuckDB();
     const [connection, setConnection] = React.useState<AsyncDuckDBConnection | undefined>(undefined);
@@ -31,7 +32,16 @@ export function useDuckDBConnection<T>(scopes: T, name: string): AsyncDuckDBConn
         let c: AsyncDuckDBConnection | undefined;
         if (db) {
             db.connect().then(async connection => {
-                await db.registerFileText(`${name}.json`, JSON.stringify(scopes));
+                const scopesStr = JSON.stringify(scopes.map((scope, idx) => {
+                    const row = {};
+                    for (const key in scope) {
+                        if (key.indexOf("__") !== 0) {
+                            row[key] = scope[key];
+                        }
+                    }
+                    return row;
+                }));
+                await db.registerFileText(`${name}.json`, scopesStr);
                 await connection.insertJSONFromPath(`${name}.json`, { name });
                 await connection.close();
                 c = await db.connect();

diff --git a/esp/src/src-react/util/metricGraph.ts b/esp/src/src-react/util/metricGraph.ts
@@ -128,6 +128,8 @@ export class MetricGraph extends Graph2<IScope, IScopeEdge, IScope> {
                 parent = this.ensureLineage({
                     __formattedProps: {},
                     __groupedProps: {},
+                    __StdDevs: 0,
+                    __StdDevsSource: "",
                     id: this.scopeID(scope.__parentName),
                     name: scope.__parentName,
                     type: "unknown",