From 2264b4937bf8f8e7c5ebf98f88553e851b136f48 Mon Sep 17 00:00:00 2001 From: Colin Unger Date: Sat, 9 Nov 2024 17:45:40 -0800 Subject: [PATCH 1/5] Add support for cross-directory ccaching --- .flake/pkgs/ffdb/default.nix | 49 +++++++++++++++ .flake/pkgs/ffdb/ffdb.py | 112 +++++++++++++++++++++++++++++++++++ .gitignore | 3 + cmake/flexflow-utils.cmake | 2 +- flake.nix | 11 ++-- scripts/format.sh | 77 ------------------------ scripts/gdb/pretty_print.py | 95 ----------------------------- 7 files changed, 172 insertions(+), 177 deletions(-) create mode 100644 .flake/pkgs/ffdb/default.nix create mode 100644 .flake/pkgs/ffdb/ffdb.py delete mode 100755 scripts/format.sh delete mode 100644 scripts/gdb/pretty_print.py diff --git a/.flake/pkgs/ffdb/default.nix b/.flake/pkgs/ffdb/default.nix new file mode 100644 index 0000000000..f28af70cd3 --- /dev/null +++ b/.flake/pkgs/ffdb/default.nix @@ -0,0 +1,49 @@ +{ lib +, stdenv +, makeWrapper +, gdb +, python3 +, proj +}: + +stdenv.mkDerivation rec { + pname = "ffdb"; + version = "0.2"; + + pythonPath = with python3.pkgs; makePythonPath [ + proj + ]; + + dontBuild = true; + + nativeBuildInputs = [ makeWrapper ]; + + src = ./.; + + installPhase = '' + mkdir -p $out/share/ffdb + cp ffdb.py $out/share/ffdb + makeWrapper ${gdb}/bin/gdb $out/bin/ffdb \ + --add-flags "-q -x $out/share/ffdb/ffdb.py" \ + --set NIX_PYTHONPATH ${pythonPath} \ + --prefix PATH : ${lib.makeBinPath [ + python3 + ]} + ''; + + nativeCheckInputs = [ + gdb + python3 + proj + ]; + + + meta = with lib; { + # description = ""; + mainProgram = "ffdb"; + # homepage = "https://github.com/hugsy/gef"; + # license = licenses.mit; + # platforms = platforms.all; + # maintainers = with maintainers; [ freax13 ]; + }; +} diff --git a/.flake/pkgs/ffdb/ffdb.py b/.flake/pkgs/ffdb/ffdb.py new file mode 100644 index 0000000000..c3e539bc6e --- /dev/null +++ b/.flake/pkgs/ffdb/ffdb.py @@ -0,0 +1,112 @@ +from proj.config_file import get_config_root +from pathlib import Path +import gdb + +gdb.execute(f'directory {get_config_root(Path.cwd())}') +gdb.prompt_hook = lambda x: '(ffdb) ' +gdb.execute('set history save on') + +# python +# from proj.config_file import get_config_root +# from pathlib import Path +# print(Path.cwd()) +# #gdb.execute(f'directory {get_conf}/{}') +# end + +# show directories + +# import gdb.printing +# +# class NodePrinter: +# def __init__(self, val): +# self.val = val +# +# def to_string(self): +# ptr = self.val["ptr"] +# if ptr != 0: +# op_type = ptr.referenced_value()['op_type'] +# return f'Node' +# else: +# return f'Node' +# +# class EdgePrinter: +# def __init__(self, val): +# self.val = val +# +# def to_string(self): +# return f'Edge' +# +# class MachineViewPrinter: +# def __init__(self, val): +# self.val = val +# +# def to_string(self): +# toks = [] +# if self.val['device_type'] == 0: +# toks.append('type=GPU') +# else: +# toks.append('type=CPU') +# start_device_id = self.val['start_device_id'] +# for i in range(self.val['ndims']): +# dim = self.val['dim'][i] +# stride = self.val['stride'][i] +# toks.append(f'{i}=[{start_device_id}:{start_device_id+dim}:{stride}]') +# return f'MachineView<{" ".join(toks)}>' +# +# class DomainPrinter: +# def __init__(self, val): +# self.val = val +# +# def to_string(self): +# toks = [] +# ndim = self.val['dim'] +# for i in range(ndim): +# lo = self.val['rect_data'][i] +# hi = self.val['rect_data'][i + ndim] +# toks.append(f'{i}=[{lo}:{hi}]') +# return f'Domain<{" ".join(toks)}>' +# +# class TensorShapePrinter: +# def __init__(self, val): +# self.val = val +# +# def to_string(self): +# toks = [] +# ndim = self.val['num_dims'] +# for i in range(ndim): +# dim = self.val['dims'][i] +# size = dim['size'] +# degree = dim['degree'] +# parallel_idx = dim['parallel_idx'] +# toks.append(f'{i}=[s={size} d={degree} pi={parallel_idx}]') +# return f'TensorShape<{" ".join(toks)}>' +# +# class ParallelTensorBasePrinter: +# def __init__(self, val): +# self.val = val +# +# def to_string(self): +# toks = [] +# toks.append(f'guid={self.val["parallel_tensor_guid"]}') +# ndim = self.val['num_dims'] +# for i in range(ndim): +# dim = self.val['dims'][i] +# size = dim['size'] +# degree = dim['degree'] +# parallel_idx = dim['parallel_idx'] +# toks.append(f'{i}=[s={size} d={degree} pi={parallel_idx}]') +# return f'ParallelTensorBase<{" ".join(toks)}>' +# +# def build_pretty_printer(): +# pp = gdb.printing.RegexpCollectionPrettyPrinter( +# "flexflow") +# pp.add_printer('Node', '^FlexFlow::PCG::Node$', NodePrinter) +# pp.add_printer('Edge', '^FlexFlow::PCG::Edge$', EdgePrinter) +# pp.add_printer('MachineView', '^FlexFlow::MachineView$', MachineViewPrinter) +# pp.add_printer('Domain', '^Legion::Domain$', DomainPrinter) +# pp.add_printer('ParallelTensorShape', '^FlexFlow::ParallelTensorShape$', TensorShapePrinter) +# pp.add_printer('ParallelTensorBase', '^FlexFlow::ParallelTensorBase$', ParallelTensorBasePrinter) +# return pp +# +# gdb.printing.register_pretty_printer( +# gdb.current_objfile(), build_pretty_printer(), replace=True) diff --git a/.gitignore b/.gitignore index 397ac0974d..4b40a016df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# gdb history +.gdb_history + # dtgen files *.dtg.cc *.dtg.h diff --git a/cmake/flexflow-utils.cmake b/cmake/flexflow-utils.cmake index 90e100bb1b..69bc07bed2 100644 --- a/cmake/flexflow-utils.cmake +++ b/cmake/flexflow-utils.cmake @@ -39,7 +39,7 @@ function(ff_set_cxx_properties target) CXX_EXTENSIONS NO ) target_compile_options(${target} - PRIVATE $<$:> # add C++ compile flags here + PRIVATE $<$:> "-fdebug-prefix-map=${CMAKE_SOURCE_DIR}=." # add C++ compile flags here ) endfunction() diff --git a/flake.nix b/flake.nix index afbc2c1e37..38e59a81be 100644 --- a/flake.nix +++ b/flake.nix @@ -35,10 +35,13 @@ mkShell = pkgs.mkShell.override { stdenv = pkgs.cudaPackages.backendStdenv; }; + + proj = proj-repo.packages.${system}.proj; in { packages = { legion = pkgs.callPackage ./.flake/pkgs/legion.nix { }; + ffdb = pkgs.callPackage ./.flake/pkgs/ffdb { inherit proj; }; hpp2plantuml = pkgs.python3Packages.callPackage ./.flake/pkgs/hpp2plantuml.nix { }; rapidcheckFull = pkgs.symlinkJoin { name = "rapidcheckFull"; @@ -102,9 +105,7 @@ doxygen lcov # for code coverage ]) - (with proj-repo.packages.${system}; [ - proj - ]) + [ proj ] (with self.packages.${system}; [ legion hpp2plantuml @@ -128,7 +129,6 @@ gh-markdown-preview shellcheck plantuml - gdb ruff compdb jq @@ -148,6 +148,9 @@ black toml ]) + (with self.packages.${system}; [ + ffdb + ]) ]; }; }; diff --git a/scripts/format.sh b/scripts/format.sh deleted file mode 100755 index e4f1ec1611..0000000000 --- a/scripts/format.sh +++ /dev/null @@ -1,77 +0,0 @@ -#! /usr/bin/env bash - -set -euo pipefail - -GIT_ROOT="$(git rev-parse --show-toplevel)" -cd "$GIT_ROOT" - -TOOLS_PATH="$GIT_ROOT/.tools" -RELEASE="master-f4f85437" -CLANG_FORMAT_VERSION="16" -CLANG_FORMAT_PATH="$TOOLS_PATH/clang-format-$CLANG_FORMAT_VERSION-$RELEASE" - -mkdir -p "$TOOLS_PATH" - -error() { - >&2 echo "$@" - exit 1 -} - -get_os() { - UNAME_OUTPUT="$(uname -s)" - case "$UNAME_OUTPUT" in - Linux*) - OS=Linux - ;; - Darwin*) - OS=Mac - ;; - *) - error "Unknown OS $UNAME_OUTPUT. Exiting..." - esac - - echo "$OS" -} - -download_clang_tool() { - TOOL="$1" - VERSION="$2" - TARGET_PATH="$3" - - BASE_URL="https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/$RELEASE/" - - OS="$(get_os)" - case "$OS" in - Linux) - URL_OS="linux" - ;; - Mac) - URL_OS="macosx" - ;; - *) - error "Unknown return value from get_os: $OS. Exiting..." - esac - URL="$BASE_URL/clang-${TOOL}-${VERSION}_${URL_OS}-amd64" - echo "Downloading from $URL..." - - if command -v wget &> /dev/null; then - wget "$URL" -O "$TARGET_PATH" - elif command -v curl &> /dev/null; then - curl -L "$URL" -o "$TARGET_PATH" - else - error "Could not find either wget or curl. Exiting..." - fi -} - -if [[ ! -e $CLANG_FORMAT_PATH ]]; then - download_clang_tool format "$CLANG_FORMAT_VERSION" "$CLANG_FORMAT_PATH" - chmod u+x "$CLANG_FORMAT_PATH" -fi - -CLANG_FORMAT_CONFIG="$GIT_ROOT/.clang-format-for-format-sh" -mapfile -t FILES < <(git ls-files ':!:triton/**' '*.h' '*.cc' '*.cpp' '*.cu' '*.c' '*.decl') -if [[ -f $CLANG_FORMAT_CONFIG ]]; then - "$CLANG_FORMAT_PATH" --style=file:"$CLANG_FORMAT_CONFIG" -i "${FILES[@]}" -else - echo "error" -fi diff --git a/scripts/gdb/pretty_print.py b/scripts/gdb/pretty_print.py deleted file mode 100644 index 4cccc9b76b..0000000000 --- a/scripts/gdb/pretty_print.py +++ /dev/null @@ -1,95 +0,0 @@ -import gdb.printing - -class NodePrinter: - def __init__(self, val): - self.val = val - - def to_string(self): - ptr = self.val["ptr"] - if ptr != 0: - op_type = ptr.referenced_value()['op_type'] - return f'Node' - else: - return f'Node' - -class EdgePrinter: - def __init__(self, val): - self.val = val - - def to_string(self): - return f'Edge' - -class MachineViewPrinter: - def __init__(self, val): - self.val = val - - def to_string(self): - toks = [] - if self.val['device_type'] == 0: - toks.append('type=GPU') - else: - toks.append('type=CPU') - start_device_id = self.val['start_device_id'] - for i in range(self.val['ndims']): - dim = self.val['dim'][i] - stride = self.val['stride'][i] - toks.append(f'{i}=[{start_device_id}:{start_device_id+dim}:{stride}]') - return f'MachineView<{" ".join(toks)}>' - -class DomainPrinter: - def __init__(self, val): - self.val = val - - def to_string(self): - toks = [] - ndim = self.val['dim'] - for i in range(ndim): - lo = self.val['rect_data'][i] - hi = self.val['rect_data'][i + ndim] - toks.append(f'{i}=[{lo}:{hi}]') - return f'Domain<{" ".join(toks)}>' - -class TensorShapePrinter: - def __init__(self, val): - self.val = val - - def to_string(self): - toks = [] - ndim = self.val['num_dims'] - for i in range(ndim): - dim = self.val['dims'][i] - size = dim['size'] - degree = dim['degree'] - parallel_idx = dim['parallel_idx'] - toks.append(f'{i}=[s={size} d={degree} pi={parallel_idx}]') - return f'TensorShape<{" ".join(toks)}>' - -class ParallelTensorBasePrinter: - def __init__(self, val): - self.val = val - - def to_string(self): - toks = [] - toks.append(f'guid={self.val["parallel_tensor_guid"]}') - ndim = self.val['num_dims'] - for i in range(ndim): - dim = self.val['dims'][i] - size = dim['size'] - degree = dim['degree'] - parallel_idx = dim['parallel_idx'] - toks.append(f'{i}=[s={size} d={degree} pi={parallel_idx}]') - return f'ParallelTensorBase<{" ".join(toks)}>' - -def build_pretty_printer(): - pp = gdb.printing.RegexpCollectionPrettyPrinter( - "flexflow") - pp.add_printer('Node', '^FlexFlow::PCG::Node$', NodePrinter) - pp.add_printer('Edge', '^FlexFlow::PCG::Edge$', EdgePrinter) - pp.add_printer('MachineView', '^FlexFlow::MachineView$', MachineViewPrinter) - pp.add_printer('Domain', '^Legion::Domain$', DomainPrinter) - pp.add_printer('ParallelTensorShape', '^FlexFlow::ParallelTensorShape$', TensorShapePrinter) - pp.add_printer('ParallelTensorBase', '^FlexFlow::ParallelTensorBase$', ParallelTensorBasePrinter) - return pp - -gdb.printing.register_pretty_printer( - gdb.current_objfile(), build_pretty_printer(), replace=True) From 481930f1c152a343da5b95fc805efa59da65eaf2 Mon Sep 17 00:00:00 2001 From: Colin Unger Date: Sat, 9 Nov 2024 18:47:33 -0800 Subject: [PATCH 2/5] Add gdb alias for ffdb --- .flake/pkgs/ffdb/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.flake/pkgs/ffdb/default.nix b/.flake/pkgs/ffdb/default.nix index f28af70cd3..edd8d97a07 100644 --- a/.flake/pkgs/ffdb/default.nix +++ b/.flake/pkgs/ffdb/default.nix @@ -23,12 +23,13 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/share/ffdb cp ffdb.py $out/share/ffdb - makeWrapper ${gdb}/bin/gdb $out/bin/ffdb \ + makeWrapper ${gdb}/bin/gdb $out/bin/gdb \ --add-flags "-q -x $out/share/ffdb/ffdb.py" \ --set NIX_PYTHONPATH ${pythonPath} \ --prefix PATH : ${lib.makeBinPath [ python3 ]} + cp $out/bin/gdb $out/bin/ffdb ''; nativeCheckInputs = [ From 00b1a8424c51fc734c4caed6f42f6d65eb547502 Mon Sep 17 00:00:00 2001 From: Colin Unger Date: Sat, 9 Nov 2024 18:50:08 -0800 Subject: [PATCH 3/5] Update proj --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 87fae7f446..1fb4f26189 100644 --- a/flake.lock +++ b/flake.lock @@ -43,11 +43,11 @@ ] }, "locked": { - "lastModified": 1728341842, - "narHash": "sha256-XMS52KBSS6z3k2VaiVcHyZQD6b2QUm1wIvTClel4xwg=", + "lastModified": 1731206929, + "narHash": "sha256-5O85Ydkk4AG8F3Y5pFj3aywCZwGqmvOj1DFnIXgfyxs=", "owner": "lockshaw", "repo": "proj", - "rev": "830fb5b1a0c7087752693990e90bbbf021168dfe", + "rev": "99d4df1a81b3b7a6595e9e7913b20f9e6a7f5e21", "type": "github" }, "original": { From c49e137424eeacad0c80052937fee040a8f95195 Mon Sep 17 00:00:00 2001 From: Colin Unger Date: Sat, 9 Nov 2024 19:10:07 -0800 Subject: [PATCH 4/5] Change from fdebug-prefix-map to ffile-prefix-map to fix __FILE__ issues --- cmake/flexflow-utils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/flexflow-utils.cmake b/cmake/flexflow-utils.cmake index 69bc07bed2..7ba39e92c9 100644 --- a/cmake/flexflow-utils.cmake +++ b/cmake/flexflow-utils.cmake @@ -39,7 +39,7 @@ function(ff_set_cxx_properties target) CXX_EXTENSIONS NO ) target_compile_options(${target} - PRIVATE $<$:> "-fdebug-prefix-map=${CMAKE_SOURCE_DIR}=." # add C++ compile flags here + PRIVATE $<$:> "-ffile-prefix-map=${CMAKE_SOURCE_DIR}=." # add C++ compile flags here ) endfunction() From a90ce3cc022c45d80eedc270c598437f890c2d7e Mon Sep 17 00:00:00 2001 From: Colin Unger Date: Sat, 9 Nov 2024 19:19:50 -0800 Subject: [PATCH 5/5] Remove unnecessary stuff from ffdb nix file --- .flake/pkgs/ffdb/default.nix | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.flake/pkgs/ffdb/default.nix b/.flake/pkgs/ffdb/default.nix index edd8d97a07..8e3989372a 100644 --- a/.flake/pkgs/ffdb/default.nix +++ b/.flake/pkgs/ffdb/default.nix @@ -8,7 +8,7 @@ stdenv.mkDerivation rec { pname = "ffdb"; - version = "0.2"; + version = "0.1"; pythonPath = with python3.pkgs; makePythonPath [ proj @@ -37,14 +37,4 @@ stdenv.mkDerivation rec { python3 proj ]; - - - meta = with lib; { - # description = ""; - mainProgram = "ffdb"; - # homepage = "https://github.com/hugsy/gef"; - # license = licenses.mit; - # platforms = platforms.all; - # maintainers = with maintainers; [ freax13 ]; - }; }