diff --git a/requirements.txt b/requirements.txt
index b9d52c5b24..e38ba85136 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,7 +15,10 @@ transformers==4.38.1
 MonkeyType
 psutil
 pyyaml
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
 opencv-python
 submitit
 pynvml
diff --git a/torchbenchmark/models/Background_Matting/requirements.txt b/torchbenchmark/models/Background_Matting/requirements.txt
index 188e465c72..3250404275 100644
--- a/torchbenchmark/models/Background_Matting/requirements.txt
+++ b/torchbenchmark/models/Background_Matting/requirements.txt
@@ -1,4 +1,7 @@
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
 opencv-python
 pandas
 Pillow
diff --git a/torchbenchmark/models/hf_Whisper/requirements.txt b/torchbenchmark/models/hf_Whisper/requirements.txt
index fd0728f16f..461b9ef776 100644
--- a/torchbenchmark/models/hf_Whisper/requirements.txt
+++ b/torchbenchmark/models/hf_Whisper/requirements.txt
@@ -1 +1,5 @@
-numba
\ No newline at end of file
+numba
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
\ No newline at end of file
diff --git a/torchbenchmark/models/tacotron2/requirements.txt b/torchbenchmark/models/tacotron2/requirements.txt
index 39e8afb167..4f642207d5 100644
--- a/torchbenchmark/models/tacotron2/requirements.txt
+++ b/torchbenchmark/models/tacotron2/requirements.txt
@@ -1,4 +1,7 @@
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
 inflect
 scipy
 Unidecode
diff --git a/torchbenchmark/util/framework/detectron2/requirements.txt b/torchbenchmark/util/framework/detectron2/requirements.txt
index f38075ddd7..acec6a1840 100644
--- a/torchbenchmark/util/framework/detectron2/requirements.txt
+++ b/torchbenchmark/util/framework/detectron2/requirements.txt
@@ -1,3 +1,6 @@
 git+https://github.com/facebookresearch/detectron2.git@0df2d73d0013db7de629602c23cc120219b4f2b8
 omegaconf==2.3.0
-numpy
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
diff --git a/utils/build_requirements.txt b/utils/build_requirements.txt
new file mode 100644
index 0000000000..36490ccac8
--- /dev/null
+++ b/utils/build_requirements.txt
@@ -0,0 +1,6 @@
+# We need to pin numpy version to the same as the torch testing environment
+# which still supports python 3.8
+numpy==1.21.2; python_version < '3.11'
+numpy==1.26.0; python_version >= '3.11'
+psutil
+pyyaml
\ No newline at end of file
diff --git a/utils/cuda_utils.py b/utils/cuda_utils.py
index 80f677af1e..92815679df 100644
--- a/utils/cuda_utils.py
+++ b/utils/cuda_utils.py
@@ -9,6 +9,7 @@
 
 # defines the default CUDA version to compile against
 DEFAULT_CUDA_VERSION = "12.4"
+REPO_ROOT = Path(__file__).parent.parent
 
 CUDA_VERSION_MAP = {
     "12.4": {
@@ -17,11 +18,9 @@
     },
 }
 PIN_CMAKE_VERSION = "3.22.*"
-# the numpy version needs to be consistent with
-# https://github.com/pytorch/builder/blob/e66e48f9b1968213c6a7ce3ca8df6621435f0a9c/wheel/build_wheel.sh#L146
-PIN_NUMPY_VERSION = "1.23.5"
-TORCHBENCH_TORCH_NIGHTLY_PACKAGES = ["torch", "torchvision", "torchaudio"]
 
+TORCHBENCH_TORCH_NIGHTLY_PACKAGES = ["torch", "torchvision", "torchaudio"]
+BUILD_REQUIREMENTS_FILE = REPO_ROOT.joinpath("utils", "build_requirements.txt")
 
 def _nvcc_output_match(nvcc_output, target_cuda_version):
     regex = "release (.*),"
@@ -153,9 +152,8 @@ def install_torch_build_deps(cuda_version: str):
     build_deps = ["ffmpeg"]
     cmd = ["conda", "install", "-y"] + build_deps
     subprocess.check_call(cmd)
-    # pip deps
-    pip_deps = [f"numpy=={PIN_NUMPY_VERSION}"]
-    cmd = ["pip", "install"] + pip_deps
+    # pip build deps
+    cmd = ["pip", "install", "-r"] + str(BUILD_REQUIREMENTS_FILE.resolve())
     subprocess.check_call(cmd)
     # conda forge deps
     # ubuntu 22.04 comes with libstdcxx6 12.3.0