diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
index 3a8ceef13b529..9f8a241d63979 100644
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
@@ -13,6 +13,10 @@ parameters:
   type: boolean
   default: false
 
+- name: InstallONNX
+  type: boolean
+  default: true
+
 - name: WITHCACHE
   type: boolean
   default: false
@@ -106,34 +110,35 @@ steps:
 
     displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
 
-- ${{ if eq(parameters.WITHCACHE, true) }}:
-  - task: Cache@2
-    # machinepool is used to ensure the compiler is same
+- $${{ if eq(parameters.InstallONNX, true}}:
+  - ${{ if eq(parameters.WITHCACHE, true) }}:
+    - task: Cache@2
+      # machinepool is used to ensure the compiler is same
+      inputs:
+        key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
+        path: $(Agent.TempDirectory)/deps_ccache
+        restoreKeys: |
+          "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
+      displayName: Cache Task
+
+  - task: PowerShell@2
+    displayName: 'Install ONNX'
     inputs:
-      key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
-      path: $(Agent.TempDirectory)/deps_ccache
-      restoreKeys: |
-        "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
-    displayName: Cache Task
-
-- task: PowerShell@2
-  displayName: 'Install ONNX'
-  inputs:
-    filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-    workingDirectory: '$(Build.BinariesDirectory)'
+      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
+      workingDirectory: '$(Build.BinariesDirectory)'
+      ${{ if eq(parameters.WITHCACHE, true) }}:
+        arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} -use_cache
+      ${{ else }}:
+        arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }}
     ${{ if eq(parameters.WITHCACHE, true) }}:
-      arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} -use_cache
-    ${{ else }}:
-      arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }}
-  ${{ if eq(parameters.WITHCACHE, true) }}:
-    env:
-      CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
-      CCACHE_COMPILERCHECK: content
-
-- ${{ if eq(parameters.WITHCACHE, true) }}:
-  - powershell: |
-      ccache -sv
-      ccache -z
-    displayName: cache stat
-    env:
-      CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
+      env:
+        CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
+        CCACHE_COMPILERCHECK: content
+
+  - ${{ if eq(parameters.WITHCACHE, true) }}:
+    - powershell: |
+        ccache -sv
+        ccache -z
+      displayName: cache stat
+      env:
+        CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
index 53eea1d69fb0e..c333c7ef084d0 100644
--- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@@ -68,6 +68,7 @@ stages:
           BuildConfig: Debug
           MachinePool: 'onnxruntime-Win-CPU-2022'
           WithCache: false
+          InstallONNX: false
           Today: $(TODAY)
 
       - task: PythonScript@0
@@ -155,7 +156,7 @@ stages:
         GenerateDocumentation: false
         WITH_CACHE: false
         MachinePool: 'onnxruntime-Win-CPU-2022'
-        
+
 - stage: x86_release
   dependsOn: []
   jobs:
@@ -256,5 +257,3 @@ stages:
         GenerateDocumentation: false
         WITH_CACHE: false
         MachinePool: 'onnxruntime-Win-CPU-2022'
-
-