diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml
new file mode 100644
index 0000000000000..0a8fe2f50a29f
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-package-test-pipeline.yml
@@ -0,0 +1,24 @@
+resources:
+  pipelines:
+  - pipeline: build
+    source: 'Python CUDA ALT Packaging Pipeline'
+    trigger: true
+    branch: main # branch to pick the artifact, Used only for manual triggered pipeline runs for testing the pipeline itself
+
+stages:
+  # ****The following Stage depend on all previous tags. ***
+  # GPU resources are very limited,
+  # To utilize gpu resource more efficiently, run GPU job only after all cpus jobs succeed
+  - stage: Linux_Test_CUDA_Alt_x86_64_stage
+    dependsOn:
+    jobs:
+      - template: templates/py-packaging-linux-test-cuda.yml
+        parameters:
+          arch: 'x86_64'
+          machine_pool: 'Onnxruntime-Linux-GPU'
+          python_wheel_suffix: '_gpu'
+          timeout: 480
+          docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
+          trt_version: '10.4.0.26-1.cuda11.8'
+          cuda_version: '11.8'
+
diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
index e946fedd07a27..5094c56956978 100644
--- a/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml
@@ -9,7 +9,7 @@ stages:
   # ****The following Stage depend on all previous tags. ***
   # GPU resources are very limited,
   # To utilize gpu resource more efficiently, run GPU job only after all cpus jobs succeed
-  - stage: Linux_Test_GPU_x86_64_stage
+  - stage: Linux_Test_CUDA_x86_64_stage
     dependsOn:
     jobs:
       - template: templates/py-packaging-linux-test-cuda.yml
diff --git a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml
index c458f0cf4bfe2..a0e49692220f9 100644
--- a/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/py-package-test-pipeline.yml
@@ -42,27 +42,13 @@ stages:
 
 # GPU resources are very limited,
 # To utilize gpu resource more efficiently, run GPU job only after all cpus jobs succeed
-- stage: Linux_Test_GPU_x86_64_stage
-  dependsOn:
-    - Linux_Test_CPU_x86_64_stage
-    - Linux_Test_CPU_aarch64_stage
-    - Packages_Somking_Test
-  jobs:
-  - template: templates/py-packaging-linux-test-cuda.yml
-    parameters:
-      arch: 'x86_64'
-      machine_pool: 'Onnxruntime-Linux-GPU'
-      python_wheel_suffix: '_gpu'
-      timeout: 480
-      docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
-      trt_version: '10.4.0.26-1.cuda11.8'
-      cuda_version: '11.8'
-
 
 # if final job not extecuted, it will not run nightlly build
 - stage: Final
   dependsOn:
-    - Linux_Test_GPU_x86_64_stage
+  - Linux_Test_CPU_x86_64_stage
+  - Linux_Test_CPU_aarch64_stage
+  - Packages_Somking_Test
   jobs:
   - job: Final
     # Run this step only if all previous steps are succeeded and (this build was triggered by a resource trigger or it was triggered by another build).