diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
index 3503857a9233c..725eeaa3707dd 100644
--- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
@@ -1,10 +1,10 @@
 trigger: none
 
 parameters:
-  - name: enable_linux_gpu
+  - name: enable_linux_cuda
     type: boolean
     default: true
-  - name: enable_windows_gpu
+  - name: enable_windows_cuda
     type: boolean
     default: true
   - name: cmake_build_type
@@ -17,26 +17,15 @@ parameters:
       - MinSizeRel
   - name: cuda_version
     type: string
-    default: '12.2'
+    default: '11.8'
     values:
       - 11.8
       - 12.2
-  - name: SpecificArtifact
-    displayName: Use Specific Artifact
-    type: boolean
-    default: false
-
-  - name: BuildId
-    displayName: Specific Artifact's BuildId
-    type: string
-    default: '0'
 
 stages:
   - template: stages/py-cuda-packaging-stage.yml
     parameters:
-      enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
-      enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
+      enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
+      enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
       cmake_build_type: ${{ parameters.cmake_build_type }}
       cuda_version: ${{ parameters.cuda_version }}
-      SpecificArtifact: ${{ parameters.SpecificArtifact }}
-      BuildId: ${{ parameters.BuildId }}
diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
index de17db216da9c..31edf473b6dd3 100644
--- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml
@@ -4,7 +4,7 @@ parameters:
   type: boolean
   default: true
 
-- name: enable_linux_gpu
+- name: enable_linux_cuda
   displayName: 'Whether Linux GPU package is built.'
   type: boolean
   default: true
@@ -14,7 +14,12 @@ parameters:
   type: boolean
   default: true
 
-- name: enable_windows_gpu
+- name: enable_windows_cuda
+  displayName: 'Whether Windows GPU package is built.'
+  type: boolean
+  default: true
+
+- name: enable_windows_dml
   displayName: 'Whether Windows GPU package is built.'
   type: boolean
   default: true
@@ -71,15 +76,20 @@ parameters:
   displayName: 'QNN SDK version. Only for QNN packages.'
   default: 2.27.0.240926
 
+- name: cuda_version
+  type: string
+  default: '12.2'
+  values:
+    - 11.8
+    - 12.2
+
 trigger: none
 
 stages:
-- template: templates/py-packaging-stage.yml
+- template: stages/py-cpu-packaging-stage.yml
   parameters:
-    enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
     enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
     enable_windows_cpu: ${{ parameters.enable_windows_cpu }}
-    enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
     enable_mac_cpu: ${{ parameters.enable_mac_cpu }}
     enable_linux_arm: ${{ parameters.enable_linux_arm }}
     enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }}
@@ -90,3 +100,17 @@ stages:
     cmake_build_type: ${{ parameters.cmake_build_type }}
     qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
     publish_symbols: true
+
+- template: stages/py-cuda-packaging-stage.yml
+  parameters:
+    enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
+    enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
+    cmake_build_type: ${{ parameters.cmake_build_type }}
+    cuda_version: ${{ parameters.cuda_version }}
+
+- ${{ if eq(parameters.enable_windows_dml, true) }}:
+  - template: stages/py-dml-packaging-stage.yml
+    parameters:
+      cmake_build_type: ${{ parameters.cmake_build_type }}
+      publish_symbols: true
+
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml
similarity index 75%
rename from tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
rename to tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml
index 10d7ce04747d9..c0d9492359c64 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml
@@ -10,21 +10,11 @@ parameters:
   type: boolean
   default: true
 
-- name: enable_linux_gpu
-  displayName: 'Whether Linux GPU package is built.'
-  type: boolean
-  default: true
-
 - name: enable_windows_cpu
   displayName: 'Whether Windows CPU package is built.'
   type: boolean
   default: true
 
-- name: enable_windows_gpu
-  displayName: 'Whether Windows GPU package is built.'
-  type: boolean
-  default: true
-
 - name: enable_mac_cpu
   displayName: 'Whether Mac CPU package is built.'
   type: boolean
@@ -65,10 +55,6 @@ parameters:
    - RelWithDebInfo
    - MinSizeRel
 
-- name: publish_symbols
-  type: boolean
-  default: false
-
 # Only applies to QNN packages.
 - name: qnn_sdk_version
   type: string
@@ -281,87 +267,6 @@ stages:
         displayName: 'Clean Agent Directories'
         condition: always()
 
-- ${{ if eq(parameters.enable_windows_gpu, true) }}:
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10'
-        PYTHON_VERSION: '3.10'
-        EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
-        ENV_SETUP_SCRIPT: setup_env_gpu.bat
-        EP_NAME: gpu
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10'
-        PYTHON_VERSION: '3.11'
-        EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
-        ENV_SETUP_SCRIPT: setup_env_gpu.bat
-        EP_NAME: gpu
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10'
-        PYTHON_VERSION: '3.12'
-        EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
-        ENV_SETUP_SCRIPT: setup_env_gpu.bat
-        EP_NAME: gpu
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10'
-        PYTHON_VERSION: '3.13'
-        EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8"  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
-        ENV_SETUP_SCRIPT: setup_env_gpu.bat
-        EP_NAME: gpu
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
-        PYTHON_VERSION: '3.10'
-        EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
-        ENV_SETUP_SCRIPT: setup_env.bat
-        EP_NAME: directml
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
-        PYTHON_VERSION: '3.11'
-        EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
-        ENV_SETUP_SCRIPT: setup_env.bat
-        EP_NAME: directml
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
-        PYTHON_VERSION: '3.12'
-        EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
-        ENV_SETUP_SCRIPT: setup_env.bat
-        EP_NAME: directml
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
-    - template: py-win-gpu.yml
-      parameters:
-        MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
-        PYTHON_VERSION: '3.13'
-        EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
-        ENV_SETUP_SCRIPT: setup_env.bat
-        EP_NAME: directml
-        publish_symbols: ${{ parameters.publish_symbols }}
-        cmake_build_type: ${{ parameters.cmake_build_type }}
-
 - ${{ if eq(parameters.enable_mac_cpu, true) }}:
   - stage: Python_Packaging_MacOS
     dependsOn: []
@@ -464,18 +369,6 @@ stages:
           extra_build_arg: ${{ parameters.build_py_parameters }}
           cmake_build_type: ${{ parameters.cmake_build_type }}
 
-
-  - ${{ if eq(parameters.enable_linux_gpu, true) }}:
-      - template: py-linux-gpu.yml
-        parameters:
-          arch: 'x86_64'
-          machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large'
-          docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
-          extra_build_arg: ${{ parameters.build_py_parameters }}
-          cmake_build_type: ${{ parameters.cmake_build_type }}
-          trt_version: '10.4.0.26-1.cuda11.8'
-          cuda_version: '11.8'
-
   - ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}:
     - stage: Python_Packaging_Windows_ARM64_QNN
       dependsOn: []
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml
index ae18687cb9e54..6f67d54ccc23e 100644
--- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml
@@ -5,12 +5,12 @@ parameters:
   type: string
   default: ''
 
-- name: enable_linux_gpu
+- name: enable_linux_cuda
   displayName: 'Whether Linux GPU package is built.'
   type: boolean
   default: true
 
-- name: enable_windows_gpu
+- name: enable_windows_cuda
   displayName: 'Whether Windows GPU package is built.'
   type: boolean
   default: true
@@ -34,16 +34,7 @@ parameters:
    - 11.8
    - 12.2
 
-- name: SpecificArtifact
-  displayName: Use Specific Artifact
-  type: boolean
-  default: false
-
-- name: BuildId
-  displayName: Specific Artifact's BuildId
-  type: string
-  default: '0'
-
+- name: Build
 - name: PythonVersions
   type: object
   displayName: 'Python versions to build'
@@ -54,22 +45,20 @@ parameters:
     - '3.13'
 
 stages:
-  - ${{ if eq(parameters.enable_windows_gpu, true) }}:
+  - ${{ if eq(parameters.enable_windows_cuda, true) }}:
     - ${{ each python_version in parameters.PythonVersions }}:
-      - template: ../templates/py-win-gpu.yml
+      - template: py-win-gpu-stage.yml
         parameters:
           PYTHON_VERSION: ${{ python_version }}
           EP_NAME: gpu
           CudaVersion: ${{ parameters.cuda_version }}
-          SpecificArtifact: ${{ parameters.SpecificArtifact }}
-          BuildId: ${{ parameters.BuildId }}
           ${{ if eq(parameters.cuda_version, '11.8') }}:
             EP_BUILD_FLAGS: --enable_lto --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8 --cuda_home=$(Agent.TempDirectory)\v11.8  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
           ${{ if eq(parameters.cuda_version, '12.2') }}:
             EP_BUILD_FLAGS: --enable_lto --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-12.6 --cuda_home=$(Agent.TempDirectory)\v12.2  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
 
-  - ${{ if eq(parameters.enable_linux_gpu, true) }}:
-      - template: ../templates/py-linux-gpu.yml
+  - ${{ if eq(parameters.enable_linux_cuda, true) }}:
+      - template: py-linux-gpu-stage.yml
         parameters:
           arch: 'x86_64'
           machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large'
diff --git a/tools/ci_build/github/azure-pipelines/stages/py-dml-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-dml-packaging-stage.yml
new file mode 100644
index 0000000000000..01b4462006887
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/stages/py-dml-packaging-stage.yml
@@ -0,0 +1,36 @@
+parameters:
+- name: publish_symbols
+  type: boolean
+  default: false
+
+# TODO: Now the Windows jobs use a different cmake build type. Consider to merge it.
+- name: cmake_build_type
+  type: string
+  displayName: 'Linux packages cmake build type. Linux Only.'
+  default: 'Release'
+  values:
+   - Debug
+   - Release
+   - RelWithDebInfo
+   - MinSizeRel
+
+- name: PythonVersions
+  type: object
+  displayName: 'Python versions to build'
+  default:
+    - '3.10'
+    - '3.11'
+    - '3.12'
+    - '3.13'
+
+stages:
+- ${{ each python_version in parameters.PythonVersions }}:
+  - template: py-win-gpu.yml
+    parameters:
+      MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10'
+      PYTHON_VERSION: ${{ python_version }}
+      EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
+      ENV_SETUP_SCRIPT: setup_env.bat
+      EP_NAME: directml
+      publish_symbols: ${{ parameters.publish_symbols }}
+      cmake_build_type: ${{ parameters.cmake_build_type }}
\ No newline at end of file
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu.yml
similarity index 96%
rename from tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
rename to tools/ci_build/github/azure-pipelines/stages/py-win-gpu.yml
index 71500e4ef9025..c8fb941b6c86b 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu.yml
@@ -28,16 +28,6 @@ parameters:
     - 11.8
     - 12.2
 
-- name: SpecificArtifact
-  displayName: Use Specific Artifact
-  type: boolean
-  default: false
-
-- name: BuildId
-  displayName: Specific Artifact's BuildId
-  type: string
-  default: '0'
-
 - name: cmake_build_type
   type: string
   displayName: 'Linux packages cmake build type. Linux Only.'
@@ -243,13 +233,11 @@ stages:
             addToPath: true
             architecture: 'x64'
 
-        - template: flex-downloadPipelineArtifact.yml
+        - template: ../templates/flex-downloadPipelineArtifact.yml
           parameters:
             ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
             StepName: 'Download Pipeline Artifact - Windows GPU Build'
             TargetPath: '$(Build.ArtifactStagingDirectory)'
-            SpecificArtifact: ${{ parameters.SpecificArtifact }}
-            BuildId: ${{ parameters.BuildId }}
 
         - task: PowerShell@2
           displayName: 'Install ONNX'