Merge branch 'master' into native-all

.github/workflows/build.yml

@@ -118,46 +118,49 @@ jobs:
         path: dist
 
   build-freebsd:
-    runs-on: macos-latest
+    runs-on: ubuntu-22.04
     needs: [lint]
     timeout-minutes: 30
     strategy:
       matrix:
-        include:
-          - box: fbsd_13_1
-            release: FreeBSD-13.1-STABLE
-            url: https://github.com/rbspy/freebsd-vagrant-box/releases/download/20221112/fbsd_13_1.box
+        box:
+          - freebsd-14
     steps:
       - uses: actions/checkout@v3
       - name: Cache Vagrant box
         uses: actions/[email protected]
         with:
-          path: ~/.vagrant.d
-          key: ${{ matrix.box }}-vagrant-boxes-20221112-${{ hashFiles('ci/Vagrantfile') }}
+          path: .vagrant.d
+          key: ${{ matrix.box }}-vagrant-boxes-20231115-${{ hashFiles('ci/Vagrantfile') }}
           restore-keys: |
-            ${{ matrix.box }}-vagrant-boxes-20221112-
+            ${{ matrix.box }}-vagrant-boxes-20231115-
       - name: Cache Cargo and build artifacts
         uses: actions/[email protected]
         with:
           path: build-artifacts.tar
-          key: ${{ matrix.box }}-cargo-20221112-${{ hashFiles('**/Cargo.lock') }}
+          key: ${{ matrix.box }}-cargo-20231115-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: |
-            ${{ matrix.box }}-cargo-20221112-
+            ${{ matrix.box }}-cargo-20231115-
+      - name: Display CPU info
+        run: lscpu
+      - name: Install VM tools
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -qq -o=Dpkg::Use-Pty=0 moreutils
+          sudo chronic apt-get install -qq -o=Dpkg::Use-Pty=0 vagrant virtualbox qemu libvirt-daemon-system
       - name: Set up VM
+        shell: sudo bash {0}
         run: |
-          brew install vagrant
-          vagrant plugin install vagrant-vbguest
+          vagrant plugin install vagrant-libvirt
           vagrant plugin install vagrant-scp
-
           ln -sf ci/Vagrantfile Vagrantfile
-
-          if [ ! -d ~/.vagrant.d/boxes/rbspy-VAGRANTSLASH-${{ matrix.release }} ]; then
-            vagrant box add --no-tty rbspy/${{ matrix.release }} ${{ matrix.url }}
-          fi
-          vagrant up ${{ matrix.box }}
+          vagrant status
+          vagrant up --no-tty --provider libvirt ${{ matrix.box }}
       - name: Build and test
+        shell: sudo bash {0}
         run: vagrant ssh ${{ matrix.box }} -- bash /vagrant/ci/test_freebsd.sh
       - name: Retrieve build artifacts for caching purposes
+        shell: sudo bash {0}
         run: |
           vagrant scp ${{ matrix.box }}:/vagrant/build-artifacts.tar build-artifacts.tar
           ls -ahl build-artifacts.tar

ci/Vagrantfile

@@ -1,20 +1,27 @@
 Vagrant.configure("2") do |config|
-  config.vm.define "fbsd_12_2" do |fbsd_12_2|
-    fbsd_12_2.vm.box = "rbspy/FreeBSD-12.2-STABLE"
+  config.vm.define "freebsd-14" do |c|
+    c.vm.box = "roboxes/freebsd14"
   end
-  config.vm.define "fbsd_13_1" do |c|
-    c.vm.box = "rbspy/FreeBSD-13.1-STABLE"
+
+  config.vm.boot_timeout = 600
+
+  config.vm.provider "libvirt" do |qe|
+    # https://vagrant-libvirt.github.io/vagrant-libvirt/configuration.html
+    qe.driver = "kvm"
+    qe.cpus = 3
+    qe.memory = 8192
   end
 
   config.vm.synced_folder ".", "/vagrant", type: "rsync",
     rsync__exclude: [".git", ".vagrant.d"]
 
   config.vm.provision "shell", inline: <<~SHELL
-    pkg install -y python devel/llvm11
+    set -e
+    pkg install -y curl bash python llvm
+    chsh -s /usr/local/bin/bash vagrant
+    pw groupmod wheel -m vagrant
+    su -l vagrant <<'EOF'
+    curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain 1.73.0
+    EOF
   SHELL
-
-  config.vm.provider "virtualbox" do |v|
-    v.memory = 4096
-    v.cpus = 2
-  end
 end

ci/test_freebsd.sh

@@ -1,23 +1,26 @@
 #!/usr/bin/env bash
 
-source ~/.bash_profile
+source "$HOME/.cargo/env"
 
 set -e
 
 python --version
 cargo --version
 
-export CARGO_HOME="/vagrant/.cargo"
-mkdir -p $CARGO_HOME
-
 cd /vagrant
 
 if [ -f build-artifacts.tar ]; then
+  echo "Unpacking cached build artifacts..."
   tar xf build-artifacts.tar
   rm -f build-artifacts.tar
 fi
 
 cargo build --release --workspace --all-targets
 cargo test --release
 
+set +e
 tar cf build-artifacts.tar target
+tar rf build-artifacts.tar "$HOME/.cargo/git"
+tar rf build-artifacts.tar "$HOME/.cargo/registry"
+
+exit 0

src/main.rs

@@ -284,6 +284,7 @@ fn record_samples(pid: remoteprocess::Pid, config: &Config) -> Result<(), Error>
                     short_filename: None,
                     line: 0,
                     locals: None,
+                    is_entry: true,
                 });
             }
 

src/native_stack_trace.rs

@@ -93,8 +93,16 @@ impl NativeStack {
                         // if we have a corresponding python frame for the evalframe
                         // merge it into the stack. (if we're out of bounds a later
                         // check will pick up - and report overall totals mismatch)
-                        if python_frame_index < frames.len() {
+
+                        // Merge all python frames until we hit one with `is_entry`.
+                        while python_frame_index < frames.len() {
                             merged.push(frames[python_frame_index].clone());
+
+                            if frames[python_frame_index].is_entry {
+                                break;
+                            }
+
+                            python_frame_index += 1;
                         }
                         python_frame_index += 1;
                     }
@@ -141,6 +149,7 @@ impl NativeStack {
                         short_filename: None,
                         module: None,
                         locals: None,
+                        is_entry: true,
                     });
                 });
 
@@ -319,6 +328,7 @@ impl NativeStack {
                     short_filename: None,
                     module: Some(frame.module.clone()),
                     locals: None,
+                    is_entry: true,
                 })
             }
             None => Some(Frame {
@@ -328,6 +338,7 @@ impl NativeStack {
                 line: 0,
                 short_filename: None,
                 module: Some(frame.module.clone()),
+                is_entry: true,
             }),
         }
     }

src/python_data_access.rs

@@ -33,10 +33,11 @@ pub fn copy_string<T: StringObject, P: ProcessMemory>(
             Ok(chars.iter().collect())
         }
         (2, _) => {
-            // UCS2 strings aren't used internally after v3.3: https://www.python.org/dev/peps/pep-0393/
-            // TODO: however with python 2.7 they could be added with --enable-unicode=ucs2 configure flag.
-            //            or with python 3.2 --with-wide-unicode=ucs2
-            Err(format_err!("ucs2 strings aren't supported yet!"))
+            #[allow(clippy::cast_ptr_alignment)]
+            let chars = unsafe {
+                std::slice::from_raw_parts(bytes.as_ptr() as *const u16, bytes.len() / 2)
+            };
+            Ok(String::from_utf16(chars)?)
         }
         (1, true) => Ok(String::from_utf8(bytes)?),
         (1, false) => Ok(bytes.iter().map(|&b| b as char).collect()),

src/python_interpreters.rs

@@ -47,6 +47,7 @@ pub trait FrameObject {
     fn code(&self) -> *mut Self::CodeObject;
     fn lasti(&self) -> i32;
     fn back(&self) -> *mut Self;
+    fn is_entry(&self) -> bool;
 }
 
 pub trait CodeObject {
@@ -157,6 +158,9 @@ macro_rules! PythonCommonImpl {
             fn back(&self) -> *mut Self {
                 self.f_back
             }
+            fn is_entry(&self) -> bool {
+                true
+            }
         }
 
         impl Object for $py::PyObject {
@@ -357,6 +361,9 @@ impl FrameObject for v3_11_0::_PyInterpreterFrame {
     fn back(&self) -> *mut Self {
         self.previous
     }
+    fn is_entry(&self) -> bool {
+        self.is_entry
+    }
 }
 
 impl Object for v3_11_0::PyObject {

src/sampler.rs

@@ -56,7 +56,7 @@ impl Sampler {
                     spy
                 }
                 Err(e) => {
-                    initialized_tx.send(Err(e)).unwrap_err();
+                    initialized_tx.send(Err(e)).unwrap();
                     return;
                 }
             };
@@ -308,7 +308,7 @@ impl PythonSpyThread {
                 }
                 Err(e) => {
                     warn!("Failed to profile python from process {}: {}", pid, e);
-                    initialized_tx.send(Err(e)).unwrap_err();
+                    initialized_tx.send(Err(e)).unwrap();
                     return;
                 }
             };

src/speedscope.rs

@@ -288,6 +288,7 @@ mod tests {
             short_filename: None,
             line: 0,
             locals: None,
+            is_entry: true,
         };
 
         let trace = stack_trace::StackTrace {

src/stack_trace.rs

@@ -47,6 +47,8 @@ pub struct Frame {
     pub line: i32,
     /// Local Variables associated with the frame
     pub locals: Option<Vec<LocalVariable>>,
+    /// If this is an entry frame. Each entry frame corresponds to one native frame.
+    pub is_entry: bool,
 }
 
 #[derive(Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Clone, Serialize)]
@@ -158,13 +160,16 @@ where
             None
         };
 
+        let is_entry = frame.is_entry();
+
         frames.push(Frame {
             name,
             filename,
             line,
             short_filename: None,
             module: None,
             locals,
+            is_entry,
         });
         if frames.len() > 4096 {
             return Err(format_err!("Max frame recursion depth reached"));
@@ -278,6 +283,7 @@ impl ProcessInfo {
             short_filename: None,
             line: 0,
             locals: None,
+            is_entry: true,
         }
     }
 }

tests/integration_test.rs

@@ -229,6 +229,34 @@ fn test_unicode() {
     assert!(!traces[0].owns_gil);
 }
 
+#[test]
+fn test_cyrillic() {
+    #[cfg(target_os = "macos")]
+    {
+        if unsafe { libc::geteuid() } != 0 {
+            return;
+        }
+    }
+
+    // Identifiers with characters outside the ASCII range are supported from Python 3
+    let runner = TestRunner::new(Config::default(), "./tests/scripts/longsleep.py");
+    if runner.spy.version.major == 2 {
+        return;
+    }
+
+    let mut runner = TestRunner::new(Config::default(), "./tests/scripts/cyrillic.py");
+
+    let traces = runner.spy.get_stack_traces().unwrap();
+    assert_eq!(traces.len(), 1);
+    let trace = &traces[0];
+
+    assert_eq!(trace.frames[0].name, "кириллица");
+    assert_eq!(trace.frames[0].line, 4);
+
+    assert_eq!(trace.frames[1].name, "<module>");
+    assert_eq!(trace.frames[1].line, 7);
+}
+
 #[test]
 fn test_local_vars() {
     #[cfg(target_os = "macos")]

tests/scripts/cyrillic.py

@@ -0,0 +1,7 @@
+import time
+
+def кириллица(seconds):
+    time.sleep(seconds)
+
+if __name__ == "__main__":
+    кириллица(100)