diff --git a/tools/ci_build/github/linux/build_linux_python_package_cuda12.sh b/tools/ci_build/github/linux/build_linux_python_package_cuda12.sh
new file mode 100755
index 0000000000000..a04e72ed1a348
--- /dev/null
+++ b/tools/ci_build/github/linux/build_linux_python_package_cuda12.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+set -e -x
+
+# This script invokes build.py
+
+mkdir -p /build/dist
+
+EXTRA_ARG=""
+
+# Put 3.8 at the last because Ubuntu 20.04 use python 3.8 and we will upload the intermediate build files of this 
+# config to Azure DevOps Artifacts and download them to a Ubuntu 20.04 machine to run the tests.
+PYTHON_EXES=("/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10" "/opt/python/cp311-cp311/bin/python3.11" "/opt/python/cp38-cp38/bin/python3.8")
+while getopts "d:p:x:c:" parameter_Option
+do case "${parameter_Option}"
+in
+#GPU or CPU.
+d) BUILD_DEVICE=${OPTARG};;
+p) PYTHON_EXES=(${OPTARG});;
+x) EXTRA_ARG=(${OPTARG});;
+c) BUILD_CONFIG=${OPTARG};;
+esac
+done
+
+BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--build_wheel")
+
+if [ "$BUILD_CONFIG" == "Debug" ]; then
+    CFLAGS="-ggdb3"
+    CXXFLAGS="-ggdb3"
+else
+    CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -Wl,--strip-all"
+    CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -O3 -pipe -Wl,--strip-all"
+    BUILD_ARGS+=("--enable_lto")
+fi
+
+# Depending on how the compiler has been configured when it was built, sometimes "gcc -dumpversion" shows the full version.
+GCC_VERSION=$(gcc -dumpversion | cut -d . -f 1)
+#-fstack-clash-protection prevents attacks based on an overlapping heap and stack.
+if [ "$GCC_VERSION" -ge 8 ]; then
+    CFLAGS="$CFLAGS -fstack-clash-protection"
+    CXXFLAGS="$CXXFLAGS -fstack-clash-protection"
+fi
+
+ARCH=$(uname -m)
+
+if [ "$ARCH" == "x86_64" ] && [ "$GCC_VERSION" -ge 9 ]; then
+    CFLAGS="$CFLAGS -fcf-protection"
+    CXXFLAGS="$CXXFLAGS -fcf-protection"
+fi
+
+echo "EXTRA_ARG:"
+echo $EXTRA_ARG
+
+if [ "$EXTRA_ARG" != "" ]; then
+    BUILD_ARGS+=("$EXTRA_ARG")
+fi
+
+if [ "$ARCH" == "x86_64" ]; then
+    #ARM build machines do not have the test data yet.
+    BUILD_ARGS+=("--enable_onnx_tests")
+fi
+
+if [ "$BUILD_DEVICE" == "GPU" ]; then
+    #Enable CUDA and TRT EPs.
+    ONNXRUNTIME_CUDA_VERSION="12.2"
+    BUILD_ARGS+=("--nvcc_threads=1" "--use_cuda" "--use_tensorrt" "--cuda_version=$ONNXRUNTIME_CUDA_VERSION" "--tensorrt_home=/usr" "--cuda_home=/usr/local/cuda-$ONNXRUNTIME_CUDA_VERSION" "--cudnn_home=/usr/local/cuda-$ONNXRUNTIME_CUDA_VERSION" "--cmake_extra_defines" "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80")
+fi
+
+export CFLAGS
+export CXXFLAGS
+for PYTHON_EXE in "${PYTHON_EXES[@]}"
+do
+  rm -rf /build/$BUILD_CONFIG
+  ${PYTHON_EXE} /onnxruntime_src/tools/ci_build/build.py "${BUILD_ARGS[@]}"
+
+  cp /build/$BUILD_CONFIG/dist/*.whl /build/dist
+done
+
+which ccache && ccache -sv && ccache -z
diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh
index 18ac6482827f9..0f36550819c4e 100755
--- a/tools/ci_build/github/linux/run_python_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh
@@ -23,7 +23,7 @@ docker run --rm \
     -e NIGHTLY_BUILD \
     -e BUILD_BUILDNUMBER \
     $ADDITIONAL_DOCKER_PARAMETER \
-    $DOCKER_IMAGE tools/ci_build/github/linux/build_linux_arm64_python_package.sh -d $DEVICE -c $BUILD_CONFIG -x $BUILD_EXTR_PAR
+    $DOCKER_IMAGE tools/ci_build/github/linux/build_linux_python_package_cuda12.sh -d $DEVICE -c $BUILD_CONFIG -x $BUILD_EXTR_PAR
 
 sudo rm -rf $BUILD_BINARIESDIRECTORY/$BUILD_CONFIG/onnxruntime $BUILD_BINARIESDIRECTORY/$BUILD_CONFIG/pybind11 \
     $BUILD_BINARIESDIRECTORY/$BUILD_CONFIG/models $BUILD_BINARIESDIRECTORY/$BUILD_CONFIG/_deps \