Memory consumption optimizations

RootInteractive/InteractiveDrawing/bokeh/CDSAlias.ts

@@ -106,7 +106,13 @@ export class CDSAlias extends ColumnarDataSource {
           return
         }
         _locked_columns.add(key)
-        const fields = column.fields.map((x: string) => isNaN(Number(x)) ? this.get_column(x)! : Array(len).fill(Number(x)))
+        let field_names
+        if (column.fields === "auto"){
+          field_names = column.transform.get_fields()
+        } else {
+          field_names = column.fields
+        }
+        const fields = field_names.map((x: string) => isNaN(Number(x)) ? this.get_column(x)! : Array(len).fill(Number(x)))
         let new_column = column.transform.v_compute(fields, this.source, data[key])
         if(new_column){
             data[key] = new_column
@@ -123,7 +129,7 @@ export class CDSAlias extends ColumnarDataSource {
               new_column[i] = column.transform.compute(row)
             }  
   	    } catch (error) {
-		console.error(error)
+		      console.error(error)
 	    }	    
         } else{
             new_column = new Array(len).fill(.0)

RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.py

@@ -1,5 +1,5 @@
 from bokeh.model import Model
-from bokeh.core.properties import String, Dict, List, Any
+from bokeh.core.properties import String, Dict, List, Any, Bool
 
 class CustomJSNAryFunction(Model):
     __implementation__ = "CustomJSNAryFunction.ts"
@@ -8,4 +8,5 @@ class CustomJSNAryFunction(Model):
     fields = List(String, default=[], help="List of positional arguments - might be made optional in the future")
     func = String(help="Code to be computed on the client - scalar case")
     v_func = String(help="Code to be computed on the client - vector case")
+    auto_fields = Bool(default=False, help="Automatically try to figure out used variables using regular expression - only used for text widget, not general")
 

RootInteractive/InteractiveDrawing/bokeh/CustomJSNAryFunction.ts

@@ -9,6 +9,7 @@ export namespace CustomJSNAryFunction {
     fields: p.Property<Array<string>>
     func: p.Property<string | null>
     v_func: p.Property<string | null>
+    auto_fields:p.Property<boolean>
   }
 }
 
@@ -24,11 +25,12 @@ export class CustomJSNAryFunction extends Model {
   static __name__ = "CustomJSNAryFunction"
 
   static {
-    this.define<CustomJSNAryFunction.Props>(({Array, String, Any, Nullable})=>({
+    this.define<CustomJSNAryFunction.Props>(({Array, String, Any, Nullable, Boolean})=>({
       parameters:  [Any, {}],
       fields: [Array(String), []],
       func:    [ Nullable(String), null ],
-      v_func:  [Nullable(String), null]
+      v_func:  [Nullable(String), null],
+      auto_fields: [Boolean, false]
     }))
   }
 
@@ -41,6 +43,8 @@ export class CustomJSNAryFunction extends Model {
   args_keys: Array<string>
   args_values: Array<any>
 
+  effective_fields: Array<string>
+
   scalar_func: Function | null
   vector_func: Function | null
 
@@ -53,9 +57,10 @@ export class CustomJSNAryFunction extends Model {
 	    this.scalar_func = null
 	    return
     }
+    this.compute_effective_fields(this.func)
     this.args_keys = Object.keys(this.parameters)
     this.args_values = Object.values(this.parameters)
-    this.scalar_func = new Function(...this.args_keys, ...this.fields, '"use strict";\n'+this.func)
+    this.scalar_func = new Function(...this.args_keys, ...this.effective_fields, '"use strict";\n'+this.func)
     this.change.emit()
   }
 
@@ -68,9 +73,10 @@ export class CustomJSNAryFunction extends Model {
 	    this.vector_func = null
 	    return
     }
+    this.compute_effective_fields(this.v_func)
     this.args_keys = Object.keys(this.parameters)
     this.args_values = Object.values(this.parameters)
-    this.vector_func = new Function(...this.args_keys, ...this.fields, "data_source", "$output",'"use strict";\n'+this.v_func)
+    this.vector_func = new Function(...this.args_keys, ...this.effective_fields, "data_source", "$output",'"use strict";\n'+this.v_func)
     this.change.emit()
   }
 
@@ -88,4 +94,21 @@ export class CustomJSNAryFunction extends Model {
       }
   }
 
+  compute_effective_fields(search:string){
+    if(!this.auto_fields){
+      this.effective_fields = this.fields
+      return
+    }
+    this.effective_fields = []
+    for (const field of this.fields) {
+      if(search.includes(field)){
+        this.effective_fields.push(field)
+      }
+    }
+  }
+
+  get_fields():string[]{
+    return this.effective_fields
+  }
+
 }

RootInteractive/InteractiveDrawing/bokeh/HistoNdCDS.ts

@@ -250,22 +250,25 @@ export class HistoNdCDS extends ColumnarDataSource {
         }
       }
     } else {
-      const n_indices = view_indices.length
+      const n_indices = this.source.length
       if(weights != null){
         const weights_array = this.source.get_column(weights)
         if (weights_array == null){
           throw ReferenceError("Column "+ weights + " not found in " + this.source.name)
         }
         for(let i=0; i<n_indices; i++){
-          let j = view_indices[i]
-          const bin = this.getbin(j, sample_array)
+          if (!view_indices[i])
+            continue
+          const bin = this.getbin(i, sample_array)
           if(bin >= 0 && bin < length){
-            bincount[bin] += weights_array[j]
+            bincount[bin] += weights_array[i]
           }
         }
       } else {
         for(let i=0; i<n_indices; i++){
-          const bin = this.getbin(view_indices[i], sample_array)
+          if (!view_indices[i])
+            continue
+          const bin = this.getbin(i, sample_array)
           if(bin >= 0 && bin < length){
             bincount[bin] += 1
           }
@@ -424,12 +427,11 @@ export class HistoNdCDS extends ColumnarDataSource {
     let range_min = Infinity
     let range_max = -Infinity
     const view = this.view!
-    const l = this.view!.length
+    const l = view!.length
     for(let x=0; x<l; x++){
-      const y = view[x]
-      if(isFinite(column[y])){
-        range_min = Math.min(range_min, column[y])
-        range_max = Math.max(range_max, column[y])
+      if(view[x] && isFinite(column[x])){
+        range_min = Math.min(range_min, column[x])
+        range_max = Math.max(range_max, column[x])
       }
     }
     if(range_min == range_max){
@@ -451,7 +453,7 @@ export class HistoNdCDS extends ColumnarDataSource {
     this._unweighted_histogram = null
     this.cached_columns.clear()
     if(this.filter != null && this.filter.active){
-      this.view = this.filter.get_indices()
+      this.view = this.filter.v_compute()
     } else {
       this.view = null
     }
@@ -522,7 +524,7 @@ export class HistoNdCDS extends ColumnarDataSource {
       working_indices[i] -= histogram[i]
     }
     const view_sorted: number[] = Array(n_entries).fill(0)
-    const l = view != null ? view.length : source.get_length()!
+    const l = source.get_length()!
     if(view == null){
       for(let i=0; i<l; i++){
         let bin_idx = this.getbin(i, sample_array)
@@ -533,9 +535,9 @@ export class HistoNdCDS extends ColumnarDataSource {
       }
     } else {
       for(let i=0; i<l; i++){
-        let bin_idx = this.getbin(view[i], sample_array)
-        if(bin_idx >= 0){
-          view_sorted[working_indices[bin_idx]] = view[i]
+        let bin_idx = this.getbin(i, sample_array)
+        if(bin_idx >= 0 && view[i]){
+          view_sorted[working_indices[bin_idx]] = i
           working_indices[bin_idx] ++
         }
       }

RootInteractive/InteractiveDrawing/bokeh/LazyIntersectionFilter.ts

@@ -29,7 +29,7 @@ export class LazyIntersectionFilter extends RIFilter {
   changed: Set<number>
   counts: number[]
   cached_vector: boolean[]
-  old_values: boolean[][]
+  old_values: number[][]
   cached_indices: number[]
   changed_indices: boolean
   _changed_values: boolean
@@ -77,31 +77,40 @@ export class LazyIntersectionFilter extends RIFilter {
             this.old_values[x] = []
         }
         const values = filters[x].v_compute()
-        while(this.old_values[x].length < values.length){
-            this.old_values[x].push(true)
+        while(this.old_values[x].length < values.length / 32 + 1){
+            this.old_values[x].push(-1)
         }
         if(this.counts == null){
             this.counts = Array(values.length).fill(0)
         }
         this.counts.length = values.length
         const invert = filters[x].invert
         const old_values = this.old_values[x]
+	let mask = 1
         if(filters[x].active){
           for(let i=0; i < values.length; i++){
             const new_value = values[i]!==invert
 	    let old_count = this.counts[i]
             this.counts[i] += new_value ? 1 : 0
-            this.counts[i] -= old_values[i] ? 1 : 0
+            this.counts[i] -= old_values[i >> 5] & mask ? 1 : 0
 	    this._changed_values ||= (!!old_count !== !!this.counts[i])
-            old_values[i] = new_value
+	    if (new_value){
+		    old_values[i >> 5] |= mask
+	    } else {
+		    old_values[i >> 5] &= ~mask
+	    }
+	    mask = mask << 1
+	    if (mask === 0) mask = 1
           }
         } else {
           for(let i=0; i < values.length; i++){
             let old_count = this.counts[i]
             this.counts[i] += 1
-            this.counts[i] -= old_values[i] ? 1 : 0
+            this.counts[i] -= old_values[i >> 5] & mask ? 1 : 0
 	    this._changed_values ||= (!!old_count !== !!this.counts[i])
-            old_values[i] = true
+            old_values[i >> 5] |= mask
+	    mask = mask << 1
+	    if (mask === 0) mask = 1
           }            
         }
     }

RootInteractive/InteractiveDrawing/bokeh/MathUtils.ts

@@ -116,7 +116,7 @@ export function weighted_kth_value(sample:number[], weights: number[], k:number,
 
   // Cholesky decomposition without pivoting - inplace
 // TODO: Perhaps add a version with pivoting too?
-function chol(X: number[], nRows: number){
+export function chol(X: number[], nRows: number){
   let iRow = 0
   let jRow, kRow
   for(let i=0; i < nRows; ++i){

RootInteractive/InteractiveDrawing/bokeh/bokehInteractiveTemplate.py

@@ -600,9 +600,9 @@ def getDefaultVarsRefWeights(variables=None, defaultVariables={}, weights=None,
         variables = []
     variablesCopy = [i for i in variables if re.match(RE_VALID_JS_NAME, i)]
     aliasArray=[
-        {"name": "funCustom0",  "func":"funCustomForm0", "variables":variablesCopy},
-        {"name": "funCustom1",  "func":"funCustomForm1", "variables":variablesCopy},
-        {"name": "funCustom2",  "func":"funCustomForm2", "variables":variablesCopy},
+        {"name": "funCustom0",  "func":"funCustomForm0"},
+        {"name": "funCustom1",  "func":"funCustomForm1"},
+        {"name": "funCustom2",  "func":"funCustomForm2"},
     ]
     variables.extend(["funCustom0","funCustom1","funCustom2"])
 

RootInteractive/InteractiveDrawing/bokeh/bokehTools.py

@@ -1216,8 +1216,9 @@ def bokehDrawArray(dataFrame, query, figureArray, histogramArray=[], parameterAr
                     if "transform" in aliasDict[cdsKey][columnKey]:
                         aliasArrayLocal.add(aliasDict[cdsKey][columnKey]["transform"])
                     if "fields" in aliasDict[cdsKey][columnKey] and aliasDict[cdsKey][columnKey]["fields"] is None:
-                        aliasDict[cdsKey][columnKey]["fields"] = weakAll
+                        aliasDict[cdsKey][columnKey]["fields"] = "auto"
                         aliasDict[cdsKey][columnKey]["transform"].fields = weakAll
+                        aliasDict[cdsKey][columnKey]["transform"].auto_fields = True
                 # Columns directly controlled by parameter
                 elif value["type"] == "parameter":
                     cdsFull.mapping[columnKey] = paramDict[value["name"]]["value"]

RootInteractive/InteractiveDrawing/bokeh/test_mathutils.mjs

@@ -0,0 +1,18 @@
+import MathUtils from "./MathUtils.js"
+
+function shallow_compare_absolute(A,B,delta){
+	if(A === B) return true
+	if(A.length !== B.length) return false
+	return A.reduce((acc,cur,idx)=>(acc*idx+Math.abs(cur-B[idx]))/(idx+1), 0) <= delta
+}
+
+function test_chol(){
+	let A = [1,0,1,0,0,1]
+	let A_llt = [1,0,1,0,0,1]
+	MathUtils.chol(A,3)
+	console.assert(shallow_compare_absolute(A,A_llt,1e-6), "expected "+A+" got "+A_llt)	
+
+}
+
+
+test_chol()