From 00bec327a259341e9ae9c3812a902ef3791be94a Mon Sep 17 00:00:00 2001 From: Harry Callahan Date: Tue, 26 Mar 2024 21:55:18 +0000 Subject: [PATCH] [lowrisc] Add a Nix shell for lowrisc internal use This adds a devShell that employees can use to bootstrap access to non-public EDA tooling. Evaluation will fail without appropriate credentials to fetch the repository 'lowrisc-nix-private'. Disclaimer: EXPERIMENTAL These shells will only be functional in the appropriate restricted environments. This is an experiment at tracking dependencies on proprietary tooling that is less out-of-band compared to simply assuming the underlying environment is pre-populated. For obvious reasons, this will always have much weaker reproducibility guarantees than freely-available software and open-source deps. However we can still lean on Nix to make bootstrapping non-public environments fast, ergonomic and hopefully reproducible within the constricted space. Using a nix flake input that is a private repository, we can effectively pin a version of the private dependencies (hash+timestamp etc) without exposing what they are. As nix is lazily evaluated, these inputs will not attempt to be fetched unless we evaluate an output which depends on them, and hence they should happily co-exist with other flake attributes for most consumers. To avoid the flake.lock in this repository from exposing the transitive deps of the private input, that flake does not track it's inputs in the standard way. Hence, impure evaluation mode is required when using these outputs. e.g. ``` nix develop .#eda_shell_lowrisc nix develop .#eda_shell_lowrisc --command bash -c \ "make -C dv/uvm/core_ibex SIMULATOR=xlm ITERATIONS=4 TEST=riscv_rand_instr_test" ``` Signed-off-by: Harry Callahan --- dv/uvm/core_ibex/scripts/compile_tb.py | 14 +++++ dv/uvm/core_ibex/scripts/nix_lib.py | 48 +++++++++++++++++ flake.lock | 71 +++++++++++++++++++------- flake.nix | 26 +++++++++- nix/lowrisc.nix | 48 +++++++++++++++++ 5 files changed, 187 insertions(+), 20 deletions(-) create mode 100644 dv/uvm/core_ibex/scripts/nix_lib.py create mode 100644 nix/lowrisc.nix diff --git a/dv/uvm/core_ibex/scripts/compile_tb.py b/dv/uvm/core_ibex/scripts/compile_tb.py index 88711bc7f4..c5f86eb60e 100755 --- a/dv/uvm/core_ibex/scripts/compile_tb.py +++ b/dv/uvm/core_ibex/scripts/compile_tb.py @@ -14,6 +14,7 @@ from ibex_cmd import get_compile_opts from scripts_lib import run_one import riscvdv_interface +import nix_lib import logging logger = logging.getLogger(__name__) @@ -112,6 +113,19 @@ def _main() -> int: logger.warning(f"WARNING: Saw non-zero retcode while compiling testbench : logfile -> {md.tb_build_stdout}") return retcode + + # If in a nix shell, patch the compiled simulation executable to use our chosen shared libraries + if os.getenv("IBEX_NIX_SHELL_LIB") is not None: + if md.simulator == "xlm": + so = md.dir_tb / "xcelium.d" / "run.d" / "librun.so" + + # nix_lib.patch_rpath(so) # Doesn't work + nix_lib.patch_dtneeded(so) + + # Finally, strip the rpath of unecessary entries + cmd = ["patchelf", str(so), "--shrink-rpath"] + subprocess.check_output(cmd) + return 0 diff --git a/dv/uvm/core_ibex/scripts/nix_lib.py b/dv/uvm/core_ibex/scripts/nix_lib.py new file mode 100644 index 0000000000..7517cab638 --- /dev/null +++ b/dv/uvm/core_ibex/scripts/nix_lib.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# Copyright lowRISC contributors. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +import os +import subprocess +import pathlib3x as pathlib + + +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def patch_rpath(so: pathlib.Path): + """Patch the rpath of the simulation executable to resolve stdenv.cc correctly.""" + nix_gcc_lib_path = os.getenv("IBEX_NIX_SHELL_LIB") + + # Get the old rpath + cmd = ["patchelf", "--print-rpath", str(so)] + old_rpath = subprocess.check_output(cmd).decode() + logger.warning(f"Old rpath : {old_rpath}") + + # Add the nix gcc lib path to the head of the shared library's RPATH + new_rpath_str = f"{nix_gcc_lib_path}:{old_rpath}" + cmd = ["patchelf", "--set-rpath", new_rpath_str, str(so)] + new_rpath_output = subprocess.check_output(cmd).decode() + logger.warning(f"Output of --set-rpath : {new_rpath_output}") + + # Print the new rpath + cmd = ["patchelf", "--print-rpath", str(so)] + new_rpath = subprocess.check_output(cmd).decode() + logger.warning(f"New rpath : {new_rpath}") + +def patch_dtneeded(so: pathlib.Path): + """Patch some stdenv.cc shared library deps of the simulation .so to be static.""" + # We need to setup a couple of .so deps to be static, as the 'xrun' utility + # uses it's own search paths when discovering shared-library deps for the librun.so + # when simulating. + nix_gcc_lib_path = os.getenv("IBEX_NIX_SHELL_LIB") + + cmd1 = ["patchelf", "--replace-needed", "libstdc++.so.6", f"{nix_gcc_lib_path}/libstdc++.so.6", str(so)] + cmd2 = ["patchelf", "--replace-needed", "libgcc_s.so.1", f"{nix_gcc_lib_path}/libgcc_s.so.1", str(so)] + + subprocess.check_output(cmd1) + subprocess.check_output(cmd2) + diff --git a/flake.lock b/flake.lock index dc36be6620..36ec30c95a 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1705309234, - "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -25,11 +25,11 @@ "poetry2nix": "poetry2nix" }, "locked": { - "lastModified": 1710961538, - "narHash": "sha256-Rjzidr7HlJr5WA5JYkh2XgbHuTxGOO1rBKOc1v5pOhU=", + "lastModified": 1711555471, + "narHash": "sha256-TSpMRZ09Bt3rwjcvMxn6yAzJ4OOvzSO4ngpEqNt0QBg=", "owner": "lowRISC", "repo": "lowrisc-nix", - "rev": "a5ad90b25ce4d1019795cd154eebf659df2e6616", + "rev": "ad2aa6c4b91b1b29790b008531b14fed11d5151f", "type": "github" }, "original": { @@ -38,6 +38,24 @@ "type": "github" } }, + "lowrisc-nix-private": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1711624668, + "narHash": "sha256-h6t5XuwWT8aZhaj0IK4NIjJnU4U8IKjTIj3AUmNd7/E=", + "ref": "refs/heads/main", + "rev": "257480fc89f066f33ed33e907ef82955f87d96a5", + "revCount": 4, + "type": "git", + "url": "ssh://git@github.com/lowRISC/lowrisc-nix-private.git" + }, + "original": { + "type": "git", + "url": "ssh://git@github.com/lowRISC/lowrisc-nix-private.git" + } + }, "nix-github-actions": { "inputs": { "nixpkgs": [ @@ -47,11 +65,11 @@ ] }, "locked": { - "lastModified": 1698974481, - "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=", + "lastModified": 1703863825, + "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", "owner": "nix-community", "repo": "nix-github-actions", - "rev": "4bb5e752616262457bc7ca5882192a564c0472d2", + "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", "type": "github" }, "original": { @@ -62,11 +80,27 @@ }, "nixpkgs": { "locked": { - "lastModified": 1705774713, - "narHash": "sha256-j6ADaDH9XiumUzkTPlFyCBcoWYhO83lfgiSqEJF2zcs=", + "lastModified": 1711460390, + "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "44733514b72e732bd49f5511bd0203dea9b9a434", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1711460390, + "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1b64fc1287991a9cce717a01c1973ef86cb1af0b", + "rev": "44733514b72e732bd49f5511bd0203dea9b9a434", "type": "github" }, "original": { @@ -91,11 +125,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1705060653, - "narHash": "sha256-puYyylgrBS4AFAHeyVRTjTUVD8DZdecJfymWJe7H438=", + "lastModified": 1708589824, + "narHash": "sha256-2GOiFTkvs5MtVF65sC78KNVxQSmsxtk0WmV1wJ9V2ck=", "owner": "nix-community", "repo": "poetry2nix", - "rev": "e0b44e9e2d3aa855d1dd77b06f067cd0e0c3860d", + "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609", "type": "github" }, "original": { @@ -111,6 +145,7 @@ "flake-utils" ], "lowrisc-nix": "lowrisc-nix", + "lowrisc-nix-private": "lowrisc-nix-private", "nixpkgs": [ "lowrisc-nix", "nixpkgs" @@ -176,11 +211,11 @@ ] }, "locked": { - "lastModified": 1699786194, - "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", + "lastModified": 1708335038, + "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1", + "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index eea7c3beca..6449d4b64e 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,11 @@ # The input 'lowrisc-nix' contains some common dependencies that can be used # by lowRISC projects. There is also an associated public binary cache. lowrisc-nix.url = "github:lowRISC/lowrisc-nix"; + # The input 'lowrisc-nix-private' is access-controlled. + # Outputs which depend on this input are for internal use only, and will fail + # to evaluate without the appropriate credentials. + # All outputs which depend on this input are suffixed '_lowrisc' + lowrisc-nix-private.url = "git+ssh://git@github.com/lowRISC/lowrisc-nix-private.git"; nixpkgs.follows = "lowrisc-nix/nixpkgs"; flake-utils.follows = "lowrisc-nix/flake-utils"; @@ -29,6 +34,13 @@ }; }; + # This import creates internal-use only outputs, which build on + # input attributes that cannot be fetched without appropriate credentials. + lr = import ./nix/lowrisc.nix { + inherit inputs pkgs system; + extraDependencies = sim_shared_lib_deps; + }; + ################ # DEPENDENCIES # ################ @@ -101,6 +113,16 @@ ''; }; + # This shell uses mkShellNoCC as the stdenv CC can interfere with EDA tools. + eda_shell = pkgs.lib.makeOverridable pkgs.mkShellNoCC { + name = "ibex-devshell-eda"; + buildInputs = ibex_runtime_deps; + nativeBuildInputs = ibex_project_deps; + shellHook = '' + ${ibex_profile_common} + ''; + }; + syn_shell = shell.override (prev: { name = "ibex-devshell-synthesis"; nativeBuildInputs = prev.nativeBuildInputs ++ ibex_syn.deps; @@ -110,9 +132,9 @@ in { devShells.${system} = { default = inputs.self.devShells.${system}.shell; - inherit shell; + inherit eda_shell; inherit syn_shell; - }; + } // lr.devShells; }; } diff --git a/nix/lowrisc.nix b/nix/lowrisc.nix new file mode 100644 index 0000000000..30e0e7d04c --- /dev/null +++ b/nix/lowrisc.nix @@ -0,0 +1,48 @@ +# Copyright lowRISC Contributors. +# Licensed under the MIT License, see LICENSE for details. +# SPDX-License-Identifier: MIT +{ + inputs, + pkgs, + system, + extraDependencies, + ... +}: let + + # Proprietary EDA deps + # These dependencies are behind the 'lowrisc-nix-private' repository, which is access-controlled. + lowriscEdaDeps = ( + map (pkg: pkg.override { + # The 'extraDependencies' argument can be used to add deps to the wrapped environments the + # EDA tools run inside. Just adding the deps to the devShell environments is not sufficient, as + # the EDA tool wrappers end up shadowing the same paths with their own wrappers, and hence cannot + # see the additional deps. + inherit extraDependencies; + }) + (with inputs.lowrisc-nix-private.packages.${system}; [vcs xcelium]) + ); + + lowriscProfile = '' + # Xcelium + # When building the simulation executable with the EDA tooling wrapped in an FHSenv, we are + # depending on the stdenv.cc. Therefore, the appropriate shared libraries need to be + # located at runtime for these executables to run. The rpath is not set correctly for us to + # discover the correct libraries, and it does not appear to matter as when invoking the simulator + # the search paths of the xrun utility are used, not those of the librun.so library. + # However, setting the DT_NEEDED paths to be static/absolute does resolve correctly. + # Therefore, pass the correct search paths into the build here, and patchelf the librun.so object + # to setup DT_NEEDED correctly (in compile_tb.py) for the appropriate libs (libstdc++ / libgcc_s) + export IBEX_NIX_SHELL_LIB=${pkgs.stdenv.cc.cc.lib}/lib + ''; + + eda_shell_lowrisc = inputs.self.devShells.${system}.eda_shell.override (prev: { + name = "ibex-devshell-eda-lowrisc"; + nativeBuildInputs = prev.nativeBuildInputs ++ lowriscEdaDeps; + shellHook = prev.shellHook + lowriscProfile; + }); + +in rec { + devShells = { + inherit eda_shell_lowrisc; + }; +}