chore(swingset): add misc analysis tools

This is a grab-bag of tools I've assembled over the last few years, to
analyze slogfiles for various things (mostly timing and leaks). Some
in JS, some in Python. I wanted to get them off my local hard drive
and into the tree where other folks could benefit from them.

* monitor-slog-block-time.py : I use this on a follower node to print
  a one-line summary of each block, from the slogfile

* prune-transcripts.js : I run this on a copy of the swingstore DB to
  delete all the old transcript spans, since most of my analysis doesn't
  need them, and it reduces the size by about 90%.

* follower-run-tools/ : I use these to measure size data about the
  original and pruned DB.

These tools are not TSC or eslint -clean: they use @ ts-nocheck and an
.eslintignore to suppress complaints.

.eslintignore

@@ -1,3 +1,4 @@
 # also ignored in packages/cosmic-proto/.eslintignore, but IDE's pick up the root config
 packages/cosmic-proto/dist
 packages/cosmic-proto/src/codegen/
+packages/SwingSet/misc-tools/

packages/SwingSet/.eslintignore

@@ -0,0 +1 @@
+misc-tools/

packages/SwingSet/misc-tools/all-delivery-time-computrons.py

@@ -0,0 +1,34 @@
+#
+# Given a slogfile on stdin and a vatID, this emits a CSV of every
+# delivery (including BOYD and GC actions), with their wallclock time
+# and computrons (if any). This can be turned into a scatter chart
+# which might show trends like organic GC taking longer over time. It
+# ignores replays.
+
+import sys, json
+from collections import defaultdict
+from itertools import count
+
+cranks = [] # (deliveryNum, wallclock, computrons)
+deliveryNum = None
+start = None
+summary = None
+
+vatID = sys.argv[1]
+
+print("crankNum,deliveryNum,elapsed,computrons")
+for line in sys.stdin:
+ d = json.loads(line)
+ time = d["time"]
+ stype = d["type"]
+ if d.get("vatID") != vatID:
+ continue
+ if stype == "deliver" and not d["replay"]:
+ crankNum = d["crankNum"]
+ deliveryNum = d["deliveryNum"]
+ start = time
+ if stype and deliveryNum is not None and stype == "deliver-result" and not d["replay"]:
+ elapsed = time - start
+ computrons = d["dr"][2]["compute"]
+ print("%s,%s,%s,%s" % (crankNum, deliveryNum, elapsed, computrons))
+ deliveryNum = None

packages/SwingSet/misc-tools/audit-refgraph.py

@@ -0,0 +1,115 @@
+import sys, json, re
+from collections import defaultdict
+
+# first build an underscore-separated list of all kvStore values
+# sqlite3 -separator _ ss.sqlite 'SELECT * FROM kvStore' |sort >all-kv.txt
+
+# then feed that into stdin
+
+vatRE = re.compile(r'^(v\d+)\.(.*)')
+vcRE = re.compile(r'^vc\.(\d+)\.(.*)')
+rcRE = re.compile(r'^vom\.rc\.(.*)')
+
+class Collection:
+ def __init__(self, vatID, collectionID):
+ self.vatID = vatID
+ self.collectionID = collectionID
+ self.data = {}
+ self.meta = {}
+ self.ordinals = {}
+
+ def __str__(self):
+ return "Collection(%s.c%d)" % (self.vatID, self.collectionID)
+
+ def add(self, c_key, value):
+ reachable_vrefs = []
+ if c_key in ["|entryCount", "|schemata", "|nextOrdinal"]:
+ self.meta[c_key] = value
+ elif c_key.startswith("|"):
+ # ordinal assignment record
+ # 'o+d31/35:1' -> 3
+ self.ordinals[c_key[1:]] = int(value)
+ else:
+ self.data[c_key] = value
+ print(c_key, value)
+ data = json.loads(value)
+ reachable_vrefs.extend(data["slots"])
+ return reachable_vrefs
+
+ def audit(self):
+ assert(int(self.meta["|entryCount"]) == len(self.data))
+
+ def audit_ordinals(self):
+ for c_key in self.data:
+ if c_key.startswith("r"):
+ # data record, where the key is an ordinal
+ # 'r0000000003:o+d31/35:1' -> '{"body":"#null","slots":[]}'
+ pieces = c_key.split(":", maxsplit=1)
+ dr_ordinal_s = pieces[0][1:]
+ dr_ordinal = int(dr_ordinal_s)
+ dr_ordinal_vref = pieces[1]
+ if dr_ordinal_vref not in self.ordinals:
+ raise ValueError("%s.c%s vref=%s ordinal=%s" % (self.vatID, self.collectionID, dr_ordinal_vref, dr_ordinal))
+ oa_ordinal = self.ordinals[dr_ordinal_vref]
+ if dr_ordinal != oa_ordinal:
+ raise ValueError("%s.c%s vref=%s ordinal=%s/%s" % (self.vatID, self.collectionID, dr_ordinal_vref, dr_ordinal, oa_ordinal))
+ for oa_vref, oa_ordinal in self.ordinals.items():
+ dr_key = "r%010d:%s" % (oa_ordinal, oa_vref)
+ if dr_key not in self.data:
+ raise ValueError("%s.c%s vref=%s ordinal=%s" % (self.vatID, self.collectionID, oa_vref, oa_ordinal))
+
+
+def make_vatID():
+ return defaultdict(Collection)
+
+class Vat:
+ def __init__(self, vatID):
+ self.vatID = vatID
+ self.collections = {}
+ self.refcounts = {} # vom.rc.$vref: $vref -> reachable_count
+ self.refs = defaultdict(set) # $vref -> inbound vrefs
+
+ def add_line(self, v_line, value):
+ if v_line.startswith("vs."):
+ vs_key = v_line[len("vs."):]
+ self.add_vatstore(vs_key, value)
+ def add_vatstore(self, vs_key, value): # vom.rc. or vc.$cid. , etc
+ #print(vatID, vs_key, value)
+ mo = vcRE.match(vs_key) # vc.$cid.
+ if mo:
+ (collectionID, c_key) = mo.groups()
+ collectionID = int(collectionID)
+ if collectionID not in self.collections:
+ self.collections[collectionID] = Collection(self.vatID, collectionID)
+ c = self.collections[collectionID]
+ reachable_vrefs = c.add(c_key, value)
+ for vref in reachable_vrefs:
+ self.refs[vref].add("collection-%d" % collectionID)
+ mo = rcRE.search(vs_key) # vom.rc.
+ if mo:
+ vref = mo.group(1)
+ reachable_s = value
+ self.refcounts[vref] = int(reachable_s)
+
+
+
+collections_by_vatID = {}
+
+
+for row in sys.stdin:
+ key,value = row.strip().split("_", maxsplit=1)
+ mo = vatRE.search(key)
+ if mo:
+ (vatID, rest) = mo.groups()
+ if vatID not in collections_by_vatID:
+ collections_by_vatID[vatID] = Vat(vatID)
+ v = collections_by_vatID[vatID]
+ v.add_line(rest, value)
+
+c = collections_by_vatID["v9"].collections[94]
+#print(c.data)
+#print(c.ordinals)
+for v in collections_by_vatID.values():
+ for c in v.collections.values():
+ c.audit_ordinals()
+ print("%s ok" % c)

packages/SwingSet/misc-tools/block-times.py

@@ -0,0 +1,54 @@
+#
+# given a slogfile on stdin, emit a CSV of (blockHeight, start,
+# elapsed) that measures the swingset time for each block (from
+# cosmic-swingset-end-block-start to
+# cosmic-swingset-end-block-finish), suitable for pasting into a
+# spreadsheet to make a graph
+
+import sys, json, statistics
+
+print("height,start,lag,swingset,deliveries,computrons,bridge_inbounds,consensus_time,block_time")
+# consensus_time is from cosmic-swingset-after-commit-block to cosmic-swingset-begin-block
+# block_time is from cosmic-swingset-end-block-start to cosmic-swingset-end-block-start
+
+lag = None # time - blockTime, how far was the node behind
+start = None # latest cosmic-swingset-end-block-start time
+last_after_commit_block = None
+last_begin_block = None
+deliveries = 0
+computrons = 0
+bridge_inbounds = 0
+consensus_time = 0
+block_time = 0
+
+for line in sys.stdin:
+ d = json.loads(line)
+ time = d["time"]
+ dtype = d["type"]
+ if dtype == "cosmic-swingset-begin-block":
+ if last_begin_block is not None:
+ block_time = time - last_begin_block
+ last_begin_block = time
+ lag = d["time"] - d["blockTime"]
+ if dtype == "cosmic-swingset-bridge-inbound":
+ bridge_inbounds += 1
+ if dtype == "cosmic-swingset-end-block-start":
+ start = time
+ if dtype == "cosmic-swingset-after-commit-block":
+ last_after_commit_block = time
+ if dtype == "cosmic-swingset-begin-block" and last_after_commit_block is not None:
+ consensus_time = time - last_after_commit_block
+ if start is not None and dtype == "deliver-result":
+ deliveries += 1
+ if d["dr"][2] and "compute" in d["dr"][2]:
+ computrons += d["dr"][2]["compute"]
+ if start is not None and dtype == "cosmic-swingset-end-block-finish":
+ height = d["blockHeight"]
+ swingset = time - start # end-block-start to end-block-finish
+ print("%d,%f,%f,%f,%d,%d,%d,%f,%f" % (height, start, lag, swingset, deliveries,
+ computrons, bridge_inbounds,
+ consensus_time, block_time))
+ start = None
+ deliveries = 0
+ computrons = 0
+ bridge_inbounds = 0

packages/SwingSet/misc-tools/bootstrap-exhaust-keyids.js

@@ -0,0 +1,32 @@
+// @ts-nocheck
+/* eslint-disable */
+import { E } from '@endo/eventual-send';
+import { Far } from '@endo/marshal';
+import { defineKind } from '@agoric/vat-data';
+
+export function buildRootObject() {
+ let count = 0;
+ const makeHolder = defineKind('holder', () => ({ held: 0 }), {
+ hold: ({ state }, value) => {
+ count += 1;
+ state.held = value;
+ },
+ });
+ const holder = makeHolder();
+ const makeHeld = defineKind('held', () => ({}), {});
+
+ return Far('root', {
+ async bootstrap(vats, devices) {
+ holder.hold(makeHeld());
+ holder.hold(makeHeld());
+ console.log(`count: ${count}`);
+ },
+
+ async more() {
+ for (let i = 0; i < 100; i++) {
+ holder.hold(makeHeld());
+ }
+ console.log(`count: ${count}`);
+ },
+ });
+}