Skip to content

Commit

Permalink
Merge pull request #18867 from GordonSmith/HPCC-31683-SKEW_COLOR
Browse files Browse the repository at this point in the history 
HPCC-31683 Highlight metrics with outlier values
  • Loading branch information
@GordonSmith
GordonSmith authored Jul 11, 2024
2 parents 212ca79 + e9bf087 commit 54db65d
Show file tree
Hide file tree
Showing 7 changed files with 419 additions and 299 deletions.
599 changes: 340 additions & 259 deletions esp/src/package-lock.json
View file

Large diffs are not rendered by default.

32 changes: 16 additions & 16 deletions esp/src/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,22 @@
"@fluentui/react-hooks": "8.7.0",
"@fluentui/react-icons-mdl2": "1.3.59",
"@fluentui/react-migration-v8-v9": "9.6.3",
"@hpcc-js/chart": "2.83.3",
"@hpcc-js/codemirror": "2.62.0",
"@hpcc-js/common": "2.71.17",
"@hpcc-js/comms": "2.92.3",
"@hpcc-js/dataflow": "8.1.6",
"@hpcc-js/eclwatch": "2.74.5",
"@hpcc-js/graph": "2.85.15",
"@hpcc-js/html": "2.42.20",
"@hpcc-js/layout": "2.49.22",
"@hpcc-js/map": "2.77.21",
"@hpcc-js/other": "2.15.22",
"@hpcc-js/phosphor": "2.18.8",
"@hpcc-js/react": "2.53.16",
"@hpcc-js/tree": "2.40.17",
"@hpcc-js/util": "2.51.0",
"@hpcc-js/wasm": "2.17.1",
"@hpcc-js/chart": "2.83.4",
"@hpcc-js/codemirror": "2.62.1",
"@hpcc-js/common": "2.71.18",
"@hpcc-js/comms": "2.93.0",
"@hpcc-js/dataflow": "8.1.7",
"@hpcc-js/eclwatch": "2.74.8",
"@hpcc-js/graph": "2.85.16",
"@hpcc-js/html": "2.42.21",
"@hpcc-js/layout": "2.49.23",
"@hpcc-js/map": "2.77.22",
"@hpcc-js/other": "2.15.23",
"@hpcc-js/phosphor": "2.18.9",
"@hpcc-js/react": "2.53.17",
"@hpcc-js/tree": "2.40.18",
"@hpcc-js/util": "2.51.1",
"@hpcc-js/wasm": "2.18.0",
"@kubernetes/client-node": "0.20.0",
"clipboard": "2.0.11",
"d3-dsv": "3.0.1",
Expand Down
32 changes: 26 additions & 6 deletions esp/src/src-react/components/Metrics.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { useConst } from "@fluentui/react-hooks";
import { bundleIcon, Folder20Filled, Folder20Regular, FolderOpen20Filled, FolderOpen20Regular, TextCaseTitleRegular, TextCaseTitleFilled } from "@fluentui/react-icons";
import { Database } from "@hpcc-js/common";
import { WorkunitsServiceEx, IScope, splitMetric } from "@hpcc-js/comms";
import { DBStore, Table } from "@hpcc-js/dgrid";
import { CellFormatter, ColumnFormat, ColumnType, DBStore, RowType, Table } from "@hpcc-js/dgrid";
import { compare, scopedLogger } from "@hpcc-js/util";
import nlsHPCC from "src/nlsHPCC";
import { WUTimelineNoFetch } from "src/Timings";
Expand Down Expand Up @@ -36,6 +36,16 @@ const defaultUIState = {
hasSelection: false
};

class ColumnFormatEx extends ColumnFormat {
formatterFunc(): CellFormatter | undefined {
const colIdx = this._owner.columns().indexOf("__StdDevs");

return function (this: ColumnType, cell: any, row: RowType): string {
return row[colIdx];
};
}
}

class DBStoreEx extends DBStore {

constructor(protected _table: TableEx, db: Database.Grid) {
Expand Down Expand Up @@ -82,7 +92,17 @@ class TableEx extends Table {
metrics(metrics: any[], options: MetricsOptionsT, timelineFilter: string, scopeFilter: string, matchCase: boolean): this {
this
.columns(["##"]) // Reset hash to force recalculation of default widths
.columns(["##", nlsHPCC.Type, nlsHPCC.Scope, ...options.properties])
.columns(["##", nlsHPCC.Type, "StdDevs", nlsHPCC.Scope, ...options.properties, "__StdDevs"])
.columnFormats([
new ColumnFormatEx()
.column("StdDevs")
.paletteID("StdDevs")
.min(0)
.max(6),
new ColumnFormat()
.column("__StdDevs")
.width(0)
])
.data(metrics
.filter(m => this.scopeFilterFunc(m, scopeFilter, matchCase))
.filter(row => {
Expand All @@ -91,21 +111,21 @@ class TableEx extends Table {
}).map((row, idx) => {
if (idx === 0) {
this._rawDataMap = {
0: "##", 1: "type", 2: "name"
0: "##", 1: "type", 2: "__StdDevs", 3: "name"
};
options.properties.forEach((p, idx2) => {
this._rawDataMap[3 + idx2] = p;
this._rawDataMap[4 + idx2] = p;
});
}
row.__hpcc_id = row.name;
return [idx, row.type, row.name, ...options.properties.map(p => {
return [idx, row.type, row.__StdDevs === 0 ? undefined : row.__StdDevs, row.name, ...options.properties.map(p => {
return row.__groupedProps[p]?.Value ??
row.__groupedProps[p]?.Max ??
row.__groupedProps[p]?.Avg ??
row.__formattedProps[p] ??
row[p] ??
"";
}), row];
}), row.__StdDevsSource, row];
}))
;
return this;
Expand Down
25 changes: 20 additions & 5 deletions esp/src/src-react/components/MetricsPropertiesTables.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
import * as React from "react";
import { useConst } from "@fluentui/react-hooks";
import { Palette } from "@hpcc-js/common";
import { IScope } from "@hpcc-js/comms";
import { Table } from "@hpcc-js/dgrid";
import { ColumnFormat, Table } from "@hpcc-js/dgrid";
import { formatDecimal } from "src/Utility";
import nlsHPCC from "src/nlsHPCC";
import { AutosizeHpccJSComponent } from "../layouts/HpccJSAdapter";

interface MetricsPropertiesTablesProps {
Palette.rainbow("StdDevs", ["white", "white", "#fff0f0", "#ffC0C0", "#ff8080", "#ff0000", "#ff0000"]);

export interface MetricsPropertiesTablesProps {
scopesTableColumns?: string[];
scopes?: IScope[];
}
Expand All @@ -21,7 +25,18 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT

// Props Table ---
const propsTable = useConst(() => new Table()
.columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax"])
.columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax", "StdDevs"])
.columnFormats([
new ColumnFormat()
.column(nlsHPCC.Property)
.paletteID("StdDevs")
.min(0)
.max(6)
.valueColumn("StdDevs"),
new ColumnFormat()
.column("StdDevs")
.width(0)
])
.sortable(true)
);

Expand All @@ -31,7 +46,7 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT
const scopeProps = [];
for (const key in item.__groupedProps) {
const row = item.__groupedProps[key];
scopeProps.push([row.Key, row.Value, row.Avg, row.Min, row.Max, row.Delta, row.StdDev, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax]);
scopeProps.push([row.Key, row.Value, row.Avg, row.Min, row.Max, row.Delta, row.StdDev === undefined ? "" : `${row.StdDev} (${formatDecimal(row.StdDevs)}σ)`, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax, row.StdDevs]);
}
scopeProps.sort((l, r) => {
const lIdx = sortByColumns.indexOf(l[0]);
Expand All @@ -53,7 +68,7 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT

propsTable
.columns([])
.columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax"])
.columns([nlsHPCC.Property, nlsHPCC.Value, "Avg", "Min", "Max", "Delta", "StdDev", "SkewMin", "SkewMax", "NodeMin", "NodeMax", "StdDevs"])
.data(props)
.lazyRender()
;
Expand Down
14 changes: 3 additions & 11 deletions esp/src/src-react/components/MetricsSQL.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,7 @@ export const MetricsSQL: React.FunctionComponent<MetricsDataProps> = ({
onSelectionChanged
}) => {

const cleanScopes = React.useMemo(() => {
return scopes.map(scope => {
const retVal = { ...scope };
delete retVal.__children;
return retVal;
});
}, [scopes]);

const connection = useDuckDBConnection(cleanScopes, "metrics");
const connection = useDuckDBConnection(scopes, "metrics");
const [schema, setSchema] = React.useState<any[]>([]);
const [sql, setSql] = React.useState<string>(defaultSql);
const [sqlError, setSqlError] = React.useState<Error | undefined>();
Expand Down Expand Up @@ -91,7 +83,7 @@ export const MetricsSQL: React.FunctionComponent<MetricsDataProps> = ({

// Query ---
React.useEffect(() => {
if (cleanScopes.length === 0) {
if (scopes.length === 0) {
setSchema([]);
setData([]);
} else if (connection) {
Expand All @@ -117,7 +109,7 @@ export const MetricsSQL: React.FunctionComponent<MetricsDataProps> = ({
scopesTable.noDataMessage(nlsHPCC.noDataMessage);
});
}
}, [cleanScopes.length, connection, scopesTable, sql]);
}, [connection, scopes.length, scopesTable, sql]);

// Selection ---
const onChange = React.useCallback((newSql: string) => {
Expand Down
14 changes: 12 additions & 2 deletions esp/src/src-react/hooks/duckdb.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import * as React from "react";
import { IScope } from "@hpcc-js/comms";
import { DuckDB } from "@hpcc-js/wasm/dist/duckdb";

type AsyncDuckDB = any;
Expand All @@ -22,7 +23,7 @@ export function useDuckDB(): [AsyncDuckDB] {
return [db];
}

export function useDuckDBConnection<T>(scopes: T, name: string): AsyncDuckDBConnection | undefined {
export function useDuckDBConnection(scopes: IScope[], name: string): AsyncDuckDBConnection | undefined {

const [db] = useDuckDB();
const [connection, setConnection] = React.useState<AsyncDuckDBConnection | undefined>(undefined);
Expand All @@ -31,7 +32,16 @@ export function useDuckDBConnection<T>(scopes: T, name: string): AsyncDuckDBConn
let c: AsyncDuckDBConnection | undefined;
if (db) {
db.connect().then(async connection => {
await db.registerFileText(`${name}.json`, JSON.stringify(scopes));
const scopesStr = JSON.stringify(scopes.map((scope, idx) => {
const row = {};
for (const key in scope) {
if (key.indexOf("__") !== 0) {
row[key] = scope[key];
}
}
return row;
}));
await db.registerFileText(`${name}.json`, scopesStr);
await connection.insertJSONFromPath(`${name}.json`, { name });
await connection.close();
c = await db.connect();
Expand Down
2 changes: 2 additions & 0 deletions esp/src/src-react/util/metricGraph.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ export class MetricGraph extends Graph2<IScope, IScopeEdge, IScope> {
parent = this.ensureLineage({
__formattedProps: {},
__groupedProps: {},
__StdDevs: 0,
__StdDevsSource: "",
id: this.scopeID(scope.__parentName),
name: scope.__parentName,
type: "unknown",
Expand Down

0 comments on commit 54db65d

Please sign in to comment.