From be343509e1754462bd6d0d71284bfffb24a5efd0 Mon Sep 17 00:00:00 2001 From: Marian Ivanov Date: Wed, 5 Jun 2024 07:40:38 +0200 Subject: [PATCH 1/8] This should reduce memory consumption of ND histograms, also speed them up if most points are selected but slow down if almost no points selected --- .../InteractiveDrawing/bokeh/HistoNdCDS.ts | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts b/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts index 46dc16ba..87366ee5 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts @@ -250,22 +250,25 @@ export class HistoNdCDS extends ColumnarDataSource { } } } else { - const n_indices = view_indices.length + const n_indices = this.source.length if(weights != null){ const weights_array = this.source.get_column(weights) if (weights_array == null){ throw ReferenceError("Column "+ weights + " not found in " + this.source.name) } for(let i=0; i= 0 && bin < length){ - bincount[bin] += weights_array[j] + bincount[bin] += weights_array[i] } } } else { for(let i=0; i= 0 && bin < length){ bincount[bin] += 1 } @@ -424,12 +427,11 @@ export class HistoNdCDS extends ColumnarDataSource { let range_min = Infinity let range_max = -Infinity const view = this.view! - const l = this.view!.length + const l = view!.length for(let x=0; x= 0){ - view_sorted[working_indices[bin_idx]] = view[i] + if(bin_idx >= 0 && view[i]){ + view_sorted[working_indices[bin_idx]] = i working_indices[bin_idx] ++ } } From 2abc12062951d1dc95aebe8ff69e9808abbc0cc3 Mon Sep 17 00:00:00 2001 From: pl0xz0rz Date: Wed, 5 Jun 2024 12:06:45 +0200 Subject: [PATCH 2/8] Reduced memory consumption of interection filter --- .../bokeh/LazyIntersectionFilter.ts | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts b/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts index 07765035..9e2abea2 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts @@ -29,7 +29,7 @@ export class LazyIntersectionFilter extends RIFilter { changed: Set counts: number[] cached_vector: boolean[] - old_values: boolean[][] + old_values: number[][] cached_indices: number[] changed_indices: boolean _changed_values: boolean @@ -77,8 +77,8 @@ export class LazyIntersectionFilter extends RIFilter { this.old_values[x] = [] } const values = filters[x].v_compute() - while(this.old_values[x].length < values.length){ - this.old_values[x].push(true) + while(this.old_values[x].length < values.length / 32 + 1){ + this.old_values[x].push(-1) } if(this.counts == null){ this.counts = Array(values.length).fill(0) @@ -86,22 +86,32 @@ export class LazyIntersectionFilter extends RIFilter { this.counts.length = values.length const invert = filters[x].invert const old_values = this.old_values[x] + let mask = 1 if(filters[x].active){ + debugger for(let i=0; i < values.length; i++){ const new_value = values[i]!==invert let old_count = this.counts[i] this.counts[i] += new_value ? 1 : 0 - this.counts[i] -= old_values[i] ? 1 : 0 + this.counts[i] -= old_values[i >> 5] & mask ? 1 : 0 this._changed_values ||= (!!old_count !== !!this.counts[i]) - old_values[i] = new_value + if (new_value){ + old_values[i >> 5] |= mask + } else { + old_values[i >> 5] &= ~mask + } + mask = mask << 1 + if (mask === 0) mask = 1 } } else { for(let i=0; i < values.length; i++){ let old_count = this.counts[i] this.counts[i] += 1 - this.counts[i] -= old_values[i] ? 1 : 0 + this.counts[i] -= old_values[i >> 5] & mask ? 1 : 0 this._changed_values ||= (!!old_count !== !!this.counts[i]) - old_values[i] = true + old_values[i >> 5] |= mask + mask = mask << 1 + if (mask === 0) mask = 1 } } } From 375ae844abac46a58b3816000141346926dab85f Mon Sep 17 00:00:00 2001 From: Marian Ivanov Date: Tue, 11 Jun 2024 11:09:07 +0200 Subject: [PATCH 3/8] Removed a breakpoint --- .../InteractiveDrawing/bokeh/LazyIntersectionFilter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts b/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts index 9e2abea2..cf60e245 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts @@ -88,7 +88,6 @@ export class LazyIntersectionFilter extends RIFilter { const old_values = this.old_values[x] let mask = 1 if(filters[x].active){ - debugger for(let i=0; i < values.length; i++){ const new_value = values[i]!==invert let old_count = this.counts[i] From c5b552b91a83c7c69901b450a9103b7af0a8ad95 Mon Sep 17 00:00:00 2001 From: Marian Ivanov Date: Tue, 11 Jun 2024 11:20:24 +0200 Subject: [PATCH 4/8] bugfix --- RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts b/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts index 87366ee5..9bc6733b 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts @@ -535,7 +535,7 @@ export class HistoNdCDS extends ColumnarDataSource { } } else { for(let i=0; i= 0 && view[i]){ view_sorted[working_indices[bin_idx]] = i working_indices[bin_idx] ++ From c44a44b93613f733846aa93b024fe14e4196dbff Mon Sep 17 00:00:00 2001 From: Marian Ivanov Date: Thu, 13 Jun 2024 15:43:54 +0200 Subject: [PATCH 5/8] Changed funCustom to only use required columns, reducing memory consumption --- .../InteractiveDrawing/bokeh/CDSAlias.ts | 10 ++++-- .../bokeh/CustomJSNAryFunction.py | 3 +- .../bokeh/CustomJSNAryFunction.ts | 31 ++++++++++++++++--- .../InteractiveDrawing/bokeh/bokehTools.py | 3 +- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/CDSAlias.ts b/RootInteractive/InteractiveDrawing/bokeh/CDSAlias.ts index 3174c8b4..e0f7b12d 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/CDSAlias.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/CDSAlias.ts @@ -106,7 +106,13 @@ export class CDSAlias extends ColumnarDataSource { return } _locked_columns.add(key) - const fields = column.fields.map((x: string) => isNaN(Number(x)) ? this.get_column(x)! : Array(len).fill(Number(x))) + let field_names + if (column.fields === "auto"){ + field_names = column.transform.get_fields() + } else { + field_names = column.fields + } + const fields = field_names.map((x: string) => isNaN(Number(x)) ? this.get_column(x)! : Array(len).fill(Number(x))) let new_column = column.transform.v_compute(fields, this.source, data[key]) if(new_column){ data[key] = new_column @@ -123,7 +129,7 @@ export class CDSAlias extends ColumnarDataSource { new_column[i] = column.transform.compute(row) } } catch (error) { - console.error(error) + console.error(error) } } else{ new_column = new Array(len).fill(.0) diff --git a/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.py b/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.py index 90bb630a..293dea04 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.py +++ b/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.py @@ -1,5 +1,5 @@ from bokeh.model import Model -from bokeh.core.properties import String, Dict, List, Any +from bokeh.core.properties import String, Dict, List, Any, Bool class CustomJSNAryFunction(Model): __implementation__ = "CustomJSNAryFunction.ts" @@ -8,4 +8,5 @@ class CustomJSNAryFunction(Model): fields = List(String, default=[], help="List of positional arguments - might be made optional in the future") func = String(help="Code to be computed on the client - scalar case") v_func = String(help="Code to be computed on the client - vector case") + auto_fields = Bool(default=False, help="Automatically try to figure out used variables using regular expression - only used for text widget, not general") diff --git a/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.ts b/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.ts index cdb3c2e7..24475223 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.ts @@ -9,6 +9,7 @@ export namespace CustomJSNAryFunction { fields: p.Property> func: p.Property v_func: p.Property + auto_fields:p.Property } } @@ -24,11 +25,12 @@ export class CustomJSNAryFunction extends Model { static __name__ = "CustomJSNAryFunction" static { - this.define(({Array, String, Any, Nullable})=>({ + this.define(({Array, String, Any, Nullable, Boolean})=>({ parameters: [Any, {}], fields: [Array(String), []], func: [ Nullable(String), null ], - v_func: [Nullable(String), null] + v_func: [Nullable(String), null], + auto_fields: [Boolean, false] })) } @@ -41,6 +43,8 @@ export class CustomJSNAryFunction extends Model { args_keys: Array args_values: Array + effective_fields: Array + scalar_func: Function | null vector_func: Function | null @@ -53,9 +57,10 @@ export class CustomJSNAryFunction extends Model { this.scalar_func = null return } + this.compute_effective_fields(this.func) this.args_keys = Object.keys(this.parameters) this.args_values = Object.values(this.parameters) - this.scalar_func = new Function(...this.args_keys, ...this.fields, '"use strict";\n'+this.func) + this.scalar_func = new Function(...this.args_keys, ...this.effective_fields, '"use strict";\n'+this.func) this.change.emit() } @@ -68,9 +73,10 @@ export class CustomJSNAryFunction extends Model { this.vector_func = null return } + this.compute_effective_fields(this.v_func) this.args_keys = Object.keys(this.parameters) this.args_values = Object.values(this.parameters) - this.vector_func = new Function(...this.args_keys, ...this.fields, "data_source", "$output",'"use strict";\n'+this.v_func) + this.vector_func = new Function(...this.args_keys, ...this.effective_fields, "data_source", "$output",'"use strict";\n'+this.v_func) this.change.emit() } @@ -88,4 +94,21 @@ export class CustomJSNAryFunction extends Model { } } + compute_effective_fields(search:string){ + if(!this.auto_fields){ + this.effective_fields = this.fields + return + } + this.effective_fields = [] + for (const field of this.fields) { + if(search.includes(field)){ + this.effective_fields.push(field) + } + } + } + + get_fields():string[]{ + return this.effective_fields + } + } diff --git a/RootInteractive/InteractiveDrawing/bokeh/bokehTools.py b/RootInteractive/InteractiveDrawing/bokeh/bokehTools.py index fb2cc9a2..dc6a3058 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/bokehTools.py +++ b/RootInteractive/InteractiveDrawing/bokeh/bokehTools.py @@ -1216,8 +1216,9 @@ def bokehDrawArray(dataFrame, query, figureArray, histogramArray=[], parameterAr if "transform" in aliasDict[cdsKey][columnKey]: aliasArrayLocal.add(aliasDict[cdsKey][columnKey]["transform"]) if "fields" in aliasDict[cdsKey][columnKey] and aliasDict[cdsKey][columnKey]["fields"] is None: - aliasDict[cdsKey][columnKey]["fields"] = weakAll + aliasDict[cdsKey][columnKey]["fields"] = "auto" aliasDict[cdsKey][columnKey]["transform"].fields = weakAll + aliasDict[cdsKey][columnKey]["transform"].auto_fields = True # Columns directly controlled by parameter elif value["type"] == "parameter": cdsFull.mapping[columnKey] = paramDict[value["name"]]["value"] From ebde917677c0df8ca68de8dd6b5d8c8d66ce2653 Mon Sep 17 00:00:00 2001 From: pl0xz0rz Date: Mon, 17 Jun 2024 10:50:12 +0200 Subject: [PATCH 6/8] Export LU decomposition for unit tests --- RootInteractive/InteractiveDrawing/bokeh/MathUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/MathUtils.ts b/RootInteractive/InteractiveDrawing/bokeh/MathUtils.ts index b2aaa5ad..a45aebce 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/MathUtils.ts +++ b/RootInteractive/InteractiveDrawing/bokeh/MathUtils.ts @@ -116,7 +116,7 @@ export function weighted_kth_value(sample:number[], weights: number[], k:number, // Cholesky decomposition without pivoting - inplace // TODO: Perhaps add a version with pivoting too? -function chol(X: number[], nRows: number){ +export function chol(X: number[], nRows: number){ let iRow = 0 let jRow, kRow for(let i=0; i < nRows; ++i){ From bf5b45eae0d699b1f73c14dbecc2bc4e929a4345 Mon Sep 17 00:00:00 2001 From: pl0xz0rz Date: Mon, 17 Jun 2024 10:51:17 +0200 Subject: [PATCH 7/8] Added unit test, can be run in nodejs --- .../bokeh/test_mathutils.mjs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 RootInteractive/InteractiveDrawing/bokeh/test_mathutils.mjs diff --git a/RootInteractive/InteractiveDrawing/bokeh/test_mathutils.mjs b/RootInteractive/InteractiveDrawing/bokeh/test_mathutils.mjs new file mode 100644 index 00000000..1c29d1a2 --- /dev/null +++ b/RootInteractive/InteractiveDrawing/bokeh/test_mathutils.mjs @@ -0,0 +1,18 @@ +import MathUtils from "./MathUtils.js" + +function shallow_compare_absolute(A,B,delta){ + if(A === B) return true + if(A.length !== B.length) return false + return A.reduce((acc,cur,idx)=>(acc*idx+Math.abs(cur-B[idx]))/(idx+1), 0) <= delta +} + +function test_chol(){ + let A = [1,0,1,0,0,1] + let A_llt = [1,0,1,0,0,1] + MathUtils.chol(A,3) + console.assert(shallow_compare_absolute(A,A_llt,1e-6), "expected "+A+" got "+A_llt) + +} + + +test_chol() From 560f1b8c71eb5f43f4745ee868c5ef5e2836d9ff Mon Sep 17 00:00:00 2001 From: Marian Ivanov Date: Tue, 18 Jun 2024 18:03:07 +0200 Subject: [PATCH 8/8] Fixed bug with funCustom if multiselect Y axis not used --- .../InteractiveDrawing/bokeh/bokehInteractiveTemplate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/RootInteractive/InteractiveDrawing/bokeh/bokehInteractiveTemplate.py b/RootInteractive/InteractiveDrawing/bokeh/bokehInteractiveTemplate.py index e20c7b70..4ca9ead0 100644 --- a/RootInteractive/InteractiveDrawing/bokeh/bokehInteractiveTemplate.py +++ b/RootInteractive/InteractiveDrawing/bokeh/bokehInteractiveTemplate.py @@ -600,9 +600,9 @@ def getDefaultVarsRefWeights(variables=None, defaultVariables={}, weights=None, variables = [] variablesCopy = [i for i in variables if re.match(RE_VALID_JS_NAME, i)] aliasArray=[ - {"name": "funCustom0", "func":"funCustomForm0", "variables":variablesCopy}, - {"name": "funCustom1", "func":"funCustomForm1", "variables":variablesCopy}, - {"name": "funCustom2", "func":"funCustomForm2", "variables":variablesCopy}, + {"name": "funCustom0", "func":"funCustomForm0"}, + {"name": "funCustom1", "func":"funCustomForm1"}, + {"name": "funCustom2", "func":"funCustomForm2"}, ] variables.extend(["funCustom0","funCustom1","funCustom2"])