Python ADO packaging pipeline

.gdn/.gdntsa

@@ -0,0 +1,3 @@
+{
+    "codebaseName": "onnxruntime_genai_main"
+}

.github/workflows/linux-cpu-arm64-build.yml

@@ -32,34 +32,34 @@ jobs:
       - name: Download Docker Image
         run: |
           set -e -x
-          az login --identity -u /subscriptions/00c06639-6ee4-454e-8058-8d8b1703bd87/resourcegroups/proj-onnxruntime-dev/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ortcibuild_readonly
+          az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
           az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
           python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/inference/aarch64/default/cpu/Dockerfile \
           --context tools/ci_build/github/linux/docker/inference/aarch64/default/cpu \
           --docker-build-args "--build-arg BUILD_UID=$( id -u )" \
           --container-registry onnxruntimebuildcache \
-          --repository onnxruntimecpubuild
+          --repository ort_genai_linux_arm64_gha
 
       - name: Doker -- Configure with CMake and GCC
         run: |
           docker run --rm \
           --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-          -w /onnxruntime_src onnxruntimecpubuild bash -c "/usr/bin/cmake --preset linux_gcc_cpu_release"
+          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "/usr/bin/cmake --preset linux_gcc_cpu_release"
 
       - name: Docker -- Build with CMake and GCC
         run: |
           docker run --rm \
           --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-          -w /onnxruntime_src onnxruntimecpubuild bash -c "/usr/bin/cmake --build --preset linux_gcc_cpu_release"
+          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "/usr/bin/cmake --build --preset linux_gcc_cpu_release"
 
       - name: Dokcer -- check test directory
         run: |
           docker run --rm \
           --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-          -w /onnxruntime_src onnxruntimecpubuild bash -c "ls -l /onnxruntime_src/build/gcc_cpu/release/test/"
+          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "ls -l /onnxruntime_src/build/gcc_cpu/release/test/"
 
       - name: Docker -- Run tests
         run: |
           docker run --rm \
           --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-          -w /onnxruntime_src onnxruntimecpubuild bash -c "/onnxruntime_src/build/gcc_cpu/release/test/unit_tests"   
+          -w /onnxruntime_src ort_genai_linux_arm64_gha bash -c "/onnxruntime_src/build/gcc_cpu/release/test/unit_tests"   

.github/workflows/linux-gpu-x64-build.yml

@@ -34,52 +34,45 @@ jobs:
       - name: Download Docker Image
         run: |
           set -e -x
-          az login --identity -u /subscriptions/00c06639-6ee4-454e-8058-8d8b1703bd87/resourcegroups/proj-onnxruntime-dev/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ortcibuild_readonly
+          az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
           az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
           python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/inference/x64/default/gpu/Dockerfile \
             --context tools/ci_build/github/linux/docker/inference/x64/default/gpu \
             --docker-build-args "--build-arg BUILD_UID=$( id -u )" \
             --container-registry onnxruntimebuildcache \
-            --repository onnxruntimegpubuild
+            --repository ort_genai_linux_gpu_gha
 
       - name: Print Docker Image Environment Variables
         run: |
           echo "Printing docker image environment variables"
-          docker run --rm onnxruntimegpubuild env
+          docker run --rm ort_genai_linux_gpu_gha env
 
       #TODO: Re-enable Test by removing -DENABLE_TESTS=OFF when the Linux GPU build is fixed
 
       - name: Build with Cmake in Docker
         run: |
-          echo "Running docker image onnxruntimegpubuild"
+          echo "Running docker image ort_genai_linux_gpu_gha"
           docker run \
             --gpus all \
             --rm \
             --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-            -w /onnxruntime_src onnxruntimegpubuild bash -c "/usr/bin/cmake -DENABLE_TESTS=OFF -DCMAKE_CUDA_ARCHITECTURES=86 --preset linux_gcc_cuda_release && /usr/bin/cmake --build --preset linux_gcc_cuda_release"
+            -w /onnxruntime_src ort_genai_linux_gpu_gha bash -c "echo $PATH && /usr/bin/cmake -DENABLE_TESTS=OFF -DCMAKE_CUDA_ARCHITECTURES=86 --preset linux_gcc_cuda_release && /usr/bin/cmake --build --preset linux_gcc_cuda_release"
+#      - name: Install the onnxruntime-genai Python wheel and run Python tests
+#        run: |
+#          echo "Installing the onnxruntime-genai Python wheel and running the Python tests"
+#          docker run \
+#            --gpus all \
+#            --rm \
+#            --volume $GITHUB_WORKSPACE:/onnxruntime_src \
+#            -w /onnxruntime_src ort_genai_linux_gpu_gha bash -c "python3 -m pip install /onnxruntime_src/build/gcc_cuda/release/wheel/onnxruntime_genai*.whl --user && python3 -m pip install -r test/python/requirements.txt --user && python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models"
+#            -w /onnxruntime_src ort_genai_linux_gpu_gha bash -c "/usr/bin/cmake -DENABLE_TESTS=OFF -DCMAKE_CUDA_ARCHITECTURES=86 --preset linux_gcc_cuda_release && /usr/bin/cmake --build --preset linux_gcc_cuda_release"
 
       # TODO: Re-enable these tests when python version is updated
       #      - name: Docker -- Run tests
       #        run: |
-      #          echo "Running docker image onnxruntimegpubuild"
+      #          echo "Running docker image ort_genai_linux_gpu_gha"
       #          docker run \
       #            --gpus all \
       #            --rm \
       #            --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-      #            -w /onnxruntime_src onnxruntimegpubuild bash -c "/onnxruntime_src/build/gcc_cuda/release/test/unit_tests"
-
-
-#      - name: Install the onnxruntime-genai Python wheel and run Python tests
-#        run: |
-#          echo "Installing the onnxruntime-genai Python wheel and running the Python tests"
-#          docker run \
-#            --gpus all \
-#            --rm \
-#            --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-#            -w /onnxruntime_src onnxruntimegpubuild \
-#            bash -c " \
-#              python3 -m pip install /onnxruntime_src/build/gcc_cuda/release/wheel/onnxruntime_genai*.whl --user && \
-#              python3 -m pip install -r test/python/requirements.txt --user && \
-#              python3 test/python/test_onnxruntime_genai.py \
-#                --cwd test/python \
-#                --test_models /test/test_models"
+      #            -w /onnxruntime_src ort_genai_linux_gpu_gha bash -c "/onnxruntime_src/build/gcc_cuda/release/test/unit_tests"

.pipelines/pypl-publishing.yml

@@ -1,67 +1,54 @@
 parameters:
-- name: enable_linux_cpu
-  displayName: 'Whether Linux CPU package is built.'
+- name: enable_win_cpu
+  displayName: 'Whether Windows CPU package is built.'
   type: boolean
   default: true
 
-- name: enable_linux_gpu
-  displayName: 'Whether Linux GPU package is built.'
+- name: enable_win_gpu
+  displayName: 'Whether Windows GPU package is built.'
   type: boolean
   default: true
 
-- name: enable_windows_cpu
-  displayName: 'Whether Windows CPU package is built.'
+- name: enable_win_arm
+  displayName: 'Whether Windows ARM package is built.'
   type: boolean
-  default: true
+  default: false
 
-- name: enable_windows_gpu
-  displayName: 'Whether Windows GPU package is built.'
+- name: enable_linux_cpu
+  displayName: 'Whether Linux CPU package is built.'
   type: boolean
   default: true
 
-- name: enable_mac_cpu
-  displayName: 'Whether Mac CPU package is built.'
+- name: enable_linux_gpu
+  displayName: 'Whether Linux GPU package is built.'
   type: boolean
-  default: false
+  default: true
 
 - name: enable_linux_arm
   displayName: 'Whether Linux ARM package is built.'
   type: boolean
   default: false
+- name: ort_version
+  displayName: 'OnnxRuntime version'
+  type: string
+  default: '1.17.0'
+
 resources:
   repositories:
   - repository: manylinux
     type: Github
     endpoint: Microsoft
     name: pypa/manylinux
     ref: 5eda9aded5462201e6310105728d33016e637ea7
+
 trigger: none
 stages:
-- stage: win-cpu-x64
-  jobs:
-  - job: win-cpu-x64
-    workspace:
-      clean: all
-    pool:
-      onnxruntime-Win-CPU-2022
-- stage: win-gpu-x64
-  jobs:
-  - job: win-gpu-x64
-    workspace:
-      clean: all
-    pool:
-      onnxruntime-Win2022-GPU-A10
-- stage: linux-cpu-x64
-  jobs:
-  - job: linux-cpu-x64
-    workspace:
-      clean: all
-    pool:
-      onnxruntime-Ubuntu2204-AMD-CPU
-- stage: linux-gpu-x64
-  jobs:
-  - job: linux-gpu-x64
-    workspace:
-      clean: all
-    pool:
-      onnxruntime-Ubuntu2204-AMD-CPU
+- template: stages/py-packaging-stage.yml
+  parameters:
+    enable_linux_cpu: ${{ parameters.enable_linux_cpu }}
+    enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
+    enable_linux_arm: ${{ parameters.enable_linux_arm }}
+    enable_win_cpu: ${{ parameters.enable_win_cpu }}
+    enable_win_gpu: ${{ parameters.enable_win_gpu }}
+    enable_win_arm: ${{ parameters.enable_win_arm }}
+    ort_version: ${{ parameters.ort_version }}

.pipelines/stages/jobs/py-linux-cpu-packaging-job.yml

@@ -0,0 +1,167 @@
+parameters:
+- name: arch
+  type: string
+  default: 'x64'
+- name: ort_version
+  type: string
+  default: '1.17.0'
+jobs:
+- job: Linux_CPU_Wheels
+  strategy:
+    matrix:
+      x64:
+        buildArch: 'x64'
+        machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
+        docker_arch: 'x64'
+      ${{ if eq(parameters.arch, 'arm64') }}:
+        arm64:
+          buildArch: 'arm64'
+          machine_pool: 'onnxruntime-linux-ARM64-CPU-2019'
+          docker_arch: 'aarch64'
+  timeoutInMinutes: 240
+  workspace:
+    clean: all
+  pool: $(machine_pool)
+  variables:
+  # The build machine pool doesn't have dotnet, so it can't run CG.
+  - name: skipComponentGovernanceDetection
+    value: true
+
+  steps:
+  - checkout: self # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-genai,
+    clean: true
+    submodules: recursive
+
+  - checkout: manylinux # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/manylinux,
+    submodules: false
+
+  - template: steps/set-nightly-build-option-variable-step.yml
+
+  - task: DownloadGitHubRelease@0
+    inputs:
+      connection: 'GitHub - Release'
+      userRepository: 'microsoft/onnxruntime'
+      defaultVersionType: 'specificTag'
+      version: 'v${{ parameters.ort_version }}'
+      itemPattern: 'onnxruntime-linux-$(buildArch)-${{ parameters.ort_version }}*'
+      downloadPath: '$(Build.SourcesDirectory)'
+    displayName: Download ONNXRuntime
+      -
+  - task: ExtractFiles@1
+    inputs:
+      archiveFilePatterns: '**/*.tgz'
+      destinationFolder: '$(Build.SourcesDirectory)/onnxruntime-genai'
+      cleanDestinationFolder: false
+      overwriteExistingFiles: true
+    displayName: Unzip OnnxRuntime
+
+  - bash: |
+      mv  onnxruntime-linux-$(buildArch)-${{ parameters.ort_version }} ort
+    displayName: Rename Onnxruntime to ort
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+
+  - script: |
+      set -e -x
+      az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+      az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+      python3 tools/ci_build/get_docker_image.py --dockerfile tools/ci_build/github/linux/docker/manylinux/Dockerfile.manylinux2_28_cpu \
+        --context tools/ci_build/github/linux/docker/manylinux \
+        --docker-build-args "--build-arg BUILD_UID=$( id -u )" \
+        --container-registry onnxruntimebuildcache \
+        --manylinux-src $(Build.SourcesDirectory)/manylinux \
+        --multiple_repos \
+        --repository onnxruntimecpubuild$(docker_arch)
+    displayName: 'Get Docker Image'
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+
+  - script: |
+      set -e -x
+      docker run \
+      --rm \
+      --volume $(Build.SourcesDirectory)/onnxruntime-genai:/onnxruntime_src \
+      -w /onnxruntime_src/ onnxruntimecpubuild$(docker_arch) \
+      bash -c " \
+          /usr/bin/cmake --preset linux_gcc_cpu_release \
+          -DENABLE_TESTS=OFF \
+          -DMANYLINUX=ON \
+          -DPYTHON_EXECUTABLE=/opt/python/cp38-cp38/bin/python3.8 && \
+          /usr/bin/cmake --build --preset linux_gcc_cpu_release"
+    displayName: 'Build Python Wheel 3.8'
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+
+  - script: |
+      set -e -x
+      docker run \
+      --rm \
+      --volume $(Build.SourcesDirectory)/onnxruntime-genai:/onnxruntime_src \
+      -w /onnxruntime_src/ onnxruntimecpubuild$(docker_arch) \
+        bash -c " \
+          /usr/bin/cmake --preset linux_gcc_cpu_release \
+          -DENABLE_TESTS=OFF \
+          -DMANYLINUX=ON \
+          -DPYTHON_EXECUTABLE=/opt/python/cp39-cp39/bin/python3.9 && \
+          /usr/bin/cmake --build --preset linux_gcc_cpu_release"
+    displayName: 'Build Python Wheel 3.9'
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+  - script: |
+      set -e -x
+      docker run \
+      --rm \
+      --volume $(Build.SourcesDirectory)/onnxruntime-genai:/onnxruntime_src \
+      -w /onnxruntime_src/ onnxruntimecpubuild$(docker_arch) \
+        bash -c " \
+          /usr/bin/cmake --preset linux_gcc_cpu_release \
+          -DENABLE_TESTS=OFF \
+          -DMANYLINUX=ON \
+          -DPYTHON_EXECUTABLE=/opt/python/cp310-cp310/bin/python3.10 && \
+          /usr/bin/cmake --build --preset linux_gcc_cpu_release"
+    displayName: 'Build Python Wheel 3.10'
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+  - script: |
+      set -e -x
+      docker run \
+      --rm \
+      --volume $(Build.SourcesDirectory)/onnxruntime-genai:/onnxruntime_src \
+      -w /onnxruntime_src/ onnxruntimecpubuild$(docker_arch) \
+        bash -c " \
+          /usr/bin/cmake --preset linux_gcc_cpu_release \
+          -DENABLE_TESTS=OFF \
+          -DMANYLINUX=ON \
+          -DPYTHON_EXECUTABLE=/opt/python/cp311-cp311/bin/python3.11 && \
+          /usr/bin/cmake --build --preset linux_gcc_cpu_release"
+    displayName: 'Build Python Wheel 3.11'
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+  - script: |
+      set -e -x
+      docker run \
+      --rm \
+      --volume $(Build.SourcesDirectory)/onnxruntime-genai:/onnxruntime_src \
+      -w /onnxruntime_src/ onnxruntimecpubuild$(docker_arch) \
+        bash -c " \
+          /usr/bin/cmake --preset linux_gcc_cpu_release \
+          -DENABLE_TESTS=OFF \
+          -DMANYLINUX=ON \
+          -DPYTHON_EXECUTABLE=/opt/python/cp312-cp312/bin/python3.12 && \
+          /usr/bin/cmake --build --preset linux_gcc_cpu_release"
+    displayName: 'Build Python Wheel 3.12'
+    workingDirectory: '$(Build.SourcesDirectory)/onnxruntime-genai'
+
+  - task: CopyFiles@2
+    displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/onnxruntime-genai/build/gcc_cpu/release/wheel'
+      Contents: '*manylinux*.whl'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)'
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish Artifact: ONNXRuntime python wheel'
+    inputs:
+      ArtifactName: onnxruntime-genai-linux-cpu-$(buildArch)
+
+  - template: steps/component-governance-component-detection-step.yml
+    parameters:
+      condition: 'succeeded'
+
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()