fix(err): include raw js frame on frame content in pg (#26326)

rust/cymbal/src/bin/test.json

@@ -0,0 +1,28 @@
+[
+    {
+        "uuid": "01932ad3-bd2a-796c-9f4c-f65ab1ff27ac",
+        "event": "$exception",
+        "properties": "{\"$exception_list\":[{\"type\":\"SyntaxError\",\"value\":\"Unexpected non-whitespace character after JSON at position 3708 (line 2 column 1)\",\"stacktrace\":{\"frames\":[{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"drainQueue\",\"in_app\":true,\"lineno\":219,\"colno\":42},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"Item.run\",\"in_app\":true,\"lineno\":249,\"colno\":14},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"afterTickTwo\",\"in_app\":true,\"lineno\":2392,\"colno\":10},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"resume_\",\"in_app\":true,\"lineno\":4950,\"colno\":3},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"flow\",\"in_app\":true,\"lineno\":4967,\"colno\":34},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"Readable.read\",\"in_app\":true,\"lineno\":4623,\"colno\":26},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"exports.IncomingMessage.emit\",\"in_app\":true,\"lineno\":3377,\"colno\":5},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"exports.IncomingMessage.<anonymous>\",\"in_app\":true,\"lineno\":6756,\"colno\":11},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"parseEventStreamLine\",\"in_app\":true,\"lineno\":6799,\"colno\":9},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"_emit\",\"in_app\":true,\"lineno\":6784,\"colno\":17},{\"platform\":\"javascript\",\"filename\":\"https://unpkg.com/[email protected]/example/eventsource-polyfill.js\",\"function\":\"EventSource.emit\",\"in_app\":true,\"lineno\":3377,\"colno\":5},{\"platform\":\"javascript\",\"filename\":\"https://app-static-prod.posthog.com/static/chunk-B4MXHXII.js\",\"function\":\"a.onmessage\",\"in_app\":true,\"lineno\":290,\"colno\":97793},{\"platform\":\"javascript\",\"filename\":\"<anonymous>\",\"function\":\"JSON.parse\",\"in_app\":true}],\"type\":\"raw\"},\"mechanism\":{\"handled\":true,\"synthetic\":false}}],\"$exception_level\":\"error\"}",
+        "timestamp": "2024-11-14 13:19:00.117",
+        "team_id": 2,
+        "project_id": 2,
+        "distinct_id": "cu5skLyhm2En4xb5ERxtMkmMtC99pdbZDcqO3VDaDqt",
+        "elements_chain": "",
+        "created_at": "2024-11-14 13:20:30.828",
+        "person_created_at": "2024-07-02 13:11:05",
+        "person_mode": "full"
+    },
+    {
+        "uuid": "01933c1f-f831-7f29-8b18-2bee05ce0125",
+        "event": "$exception",
+        "properties": "{\"$exception_list\":[{\"type\":\"Error\",\"value\":\"Unexpected usage\\n\\nError: Unexpected usage\\n    at n.loadForeignModule (https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:15003)\\n    at https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:25112\",\"stacktrace\":{\"frames\":[{\"filename\":\"https://app-static-prod.posthog.com/static/chunk-C6S33U6V.js\",\"function\":\"r\",\"in_app\":true,\"lineno\":19,\"colno\":2044},{\"filename\":\"https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js\",\"function\":\"?\",\"in_app\":true,\"lineno\":3,\"colno\":12},{\"filename\":\"https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js\",\"function\":\"?\",\"in_app\":true,\"lineno\":64,\"colno\":25112},{\"filename\":\"https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js\",\"function\":\"n.loadForeignModule\",\"in_app\":true,\"lineno\":64,\"colno\":15003}],\"type\":\"raw\"},\"mechanism\":{\"type\":\"generic\",\"handled\":true}}],\"$sentry_event_id\":\"c2954a0d25c643e4ac6b9aef7c3b39e4\",\"$sentry_exception\":{},\"$sentry_exception_message\":\"Unexpected usage\\n\\nError: Unexpected usage\\n    at n.loadForeignModule (https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:15003)\\n    at https://app-static-prod.posthog.com/static/chunk-PGUQKT6S.js:64:25112\"}",
+        "timestamp": "2024-11-17 21:55:48.475",
+        "team_id": 2,
+        "project_id": 2,
+        "distinct_id": "KCVn0RwlQkwrWhSWU5j2CzvN9QTbwsZPdqz6cfSPcXH",
+        "elements_chain": "",
+        "created_at": "2024-11-17 21:55:51.928",
+        "person_created_at": "2023-03-21 17:54:03",
+        "person_mode": "full"
+    }
+]

rust/cymbal/src/bin/test_js_resolution.rs

@@ -0,0 +1,86 @@
+use std::sync::Arc;
+
+use cymbal::{
+    config::Config,
+    langs::js::RawJSFrame,
+    symbol_store::{
+        caching::{Caching, SymbolSetCache},
+        sourcemap::SourcemapProvider,
+        Catalog,
+    },
+};
+use serde_json::Value;
+use tokio::sync::Mutex;
+
+/**
+Input data gathered by running the following, then converting to json:
+SELECT
+    symbol_set.ref as filename,
+    contents::json->>'mangled_name' as "function",
+    (contents::json->>'in_app')::boolean as in_app,
+    CASE
+        WHEN contents::json->>'line' IS NOT NULL
+        THEN (contents::json->>'line')::int
+    END as lineno,
+    CASE
+        WHEN contents::json->>'column' IS NOT NULL
+        THEN (contents::json->>'column')::int
+    END as colno
+FROM posthog_errortrackingstackframe frame
+LEFT JOIN posthog_errortrackingsymbolset symbol_set
+    ON frame.symbol_set_id = symbol_set.id
+WHERE (contents::json->>'resolved_name') is null
+    AND contents::json->>'lang' = 'javascript'
+    AND symbol_set.storage_ptr IS NOT NULL;
+
+This doesn't actually work - we don't have the original line and column number, and
+so can't repeat the original resolution. I couldn't find a way to reverse that mapping
+with sourcemaps, so instead I'm going to temporarily add the raw frame to the resolve
+Frame.
+*/
+const NAMELESS_FRAMES_IN_RAW_FMT: &str = include_str!("./nameless_frames_in_raw_format.json");
+
+#[tokio::main]
+async fn main() {
+    let config = Config::init_with_defaults().unwrap();
+    let provider = SourcemapProvider::new(&config);
+    let cache = Arc::new(Mutex::new(SymbolSetCache::new(1_000_000_000)));
+    let provider = Caching::new(provider, cache);
+
+    let catalog = Catalog::new(provider);
+
+    let frames: Vec<Value> = serde_json::from_str(NAMELESS_FRAMES_IN_RAW_FMT).unwrap();
+
+    // Deal with metabase giving me string-only values
+    let frames: Vec<RawJSFrame> = frames
+        .into_iter()
+        .map(|f| {
+            let mut f = f;
+            let in_app = f["in_app"].as_str().unwrap() == "true";
+            f["in_app"] = Value::Bool(in_app);
+            let lineno: u32 = f["lineno"]
+                .as_str()
+                .unwrap()
+                .replace(",", "")
+                .parse()
+                .unwrap();
+            let colno: u32 = f["colno"]
+                .as_str()
+                .unwrap()
+                .replace(",", "")
+                .parse()
+                .unwrap();
+            f["lineno"] = Value::Number(lineno.into());
+            f["colno"] = Value::Number(colno.into());
+            serde_json::from_value(f).unwrap()
+        })
+        .collect();
+
+    for frame in frames {
+        let res = frame.resolve(0, &catalog).await.unwrap();
+
+        if res.resolved_name.is_none() {
+            panic!("Frame name not resolved: {:?}", frame);
+        }
+    }
+}

rust/cymbal/src/fingerprinting.rs

@@ -44,6 +44,7 @@ mod test {
                 resolved: true,
                 resolve_failure: None,
                 lang: "javascript".to_string(),
+                junk_drawer: None,
                 context: None,
             },
             Frame {
@@ -57,6 +58,7 @@ mod test {
                 resolved: true,
                 resolve_failure: None,
                 lang: "javascript".to_string(),
+                junk_drawer: None,
                 context: None,
             },
         ];
@@ -72,6 +74,7 @@ mod test {
             resolved: false,
             resolve_failure: None,
             lang: "javascript".to_string(),
+            junk_drawer: None,
             context: None,
         };
 
@@ -116,6 +119,7 @@ mod test {
                 resolved: false,
                 resolve_failure: None,
                 lang: "javascript".to_string(),
+                junk_drawer: None,
                 context: None,
             },
             Frame {
@@ -129,6 +133,7 @@ mod test {
                 resolved: false,
                 resolve_failure: None,
                 lang: "javascript".to_string(),
+                junk_drawer: None,
                 context: None,
             },
             Frame {
@@ -142,6 +147,7 @@ mod test {
                 resolved: false,
                 resolve_failure: None,
                 lang: "javascript".to_string(),
+                junk_drawer: None,
                 context: None,
             },
         ];
@@ -180,6 +186,7 @@ mod test {
             resolved: false,
             resolve_failure: None,
             lang: "javascript".to_string(),
+            junk_drawer: None,
             context: None,
         }];
 
@@ -194,6 +201,7 @@ mod test {
             resolved: false,
             resolve_failure: None,
             lang: "javascript".to_string(),
+            junk_drawer: None,
             context: None,
         };
 

rust/cymbal/src/frames/mod.rs

@@ -1,4 +1,7 @@
+use std::collections::HashMap;
+
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use sha2::{Digest, Sha512};
 
 use crate::{
@@ -54,6 +57,7 @@ impl RawFrame {
 // We emit a single, unified representation of a frame, which is what we pass on to users.
 #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
 pub struct Frame {
+    // Properties used in processing
     pub raw_id: String,                  // The raw frame id this was resolved from
     pub mangled_name: String,            // Mangled name of the function
     pub line: Option<u32>,               // Line the function is define on, if known
@@ -64,6 +68,11 @@ pub struct Frame {
     pub lang: String,                    // The language of the frame. Always known (I guess?)
     pub resolved: bool,                  // Did we manage to resolve the frame?
     pub resolve_failure: Option<String>, // If we failed to resolve the frame, why?
+
+    // Random extra/internal data we want to tag onto frames, e.g. the raw input. For debugging
+    // purposes, all production code should assume this is None
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub junk_drawer: Option<HashMap<String, Value>>,
     // The lines of code surrounding the frame ptr, if known. We skip serialising this because
     // it should never go in clickhouse / be queried over, but we do store it in PG for
     // use in the frontend
@@ -110,6 +119,19 @@ impl Frame {
 
         h.update(self.lang.as_bytes());
     }
+
+    pub fn add_junk<T>(&mut self, key: impl ToString, val: T) -> Result<(), serde_json::Error>
+    where
+        T: Serialize,
+    {
+        let key = key.to_string();
+        let val = serde_json::to_value(val)?;
+        self.junk_drawer
+            .get_or_insert_with(HashMap::new)
+            .insert(key, val);
+
+        Ok(())
+    }
 }
 
 impl ContextLine {
@@ -120,3 +142,69 @@ impl ContextLine {
         }
     }
 }
+
+impl std::fmt::Display for Frame {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        writeln!(f, "Frame {}:", self.raw_id)?;
+
+        // Function name and location
+        write!(
+            f,
+            "  {} (from {}) ",
+            self.resolved_name.as_deref().unwrap_or("unknown"),
+            self.mangled_name
+        )?;
+
+        if let Some(source) = &self.source {
+            write!(f, "in {}", source)?;
+            match (self.line, self.column) {
+                (Some(line), Some(column)) => writeln!(f, ":{line}:{column}"),
+                (Some(line), None) => writeln!(f, ":{line}"),
+                (None, Some(column)) => writeln!(f, ":?:{column}"),
+                (None, None) => writeln!(f),
+            }?;
+        } else {
+            writeln!(f, "in unknown location")?;
+        }
+
+        // Metadata
+        writeln!(f, "  in_app: {}", self.in_app)?;
+        writeln!(f, "  lang: {}", self.lang)?;
+        writeln!(f, "  resolved: {}", self.resolved)?;
+        writeln!(
+            f,
+            "  resolve_failure: {}",
+            self.resolve_failure.as_deref().unwrap_or("no failure")
+        )?;
+
+        // Context
+        writeln!(f, "  context:")?;
+        if let Some(context) = &self.context {
+            for line in &context.before {
+                writeln!(f, "    {}: {}", line.number, line.line)?;
+            }
+            writeln!(f, "  > {}: {}", context.line.number, context.line.line)?;
+            for line in &context.after {
+                writeln!(f, "    {}: {}", line.number, line.line)?;
+            }
+        } else {
+            writeln!(f, "    no context")?;
+        }
+
+        // Junk drawer
+        writeln!(f, "  junk drawer:")?;
+        if let Some(junk) = &self.junk_drawer {
+            if junk.is_empty() {
+                writeln!(f, "    no junk")?;
+            } else {
+                for (key, value) in junk {
+                    writeln!(f, "    {}: {}", key, value)?;
+                }
+            }
+        } else {
+            writeln!(f, "    no junk")?;
+        }
+
+        Ok(())
+    }
+}