Skip to content

Commit

Permalink
Adding py-ado-release to pipeline (#233)
Browse files Browse the repository at this point in the history
  • Loading branch information
jchen351 authored Mar 27, 2024
1 parent 1a13bae commit f9e8e40
Show file tree
Hide file tree
Showing 12 changed files with 163 additions and 182 deletions.
6 changes: 0 additions & 6 deletions .pipelines/nuget-publishing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@ parameters:
type: boolean
default: false

- name: publish_to_nuget
displayName: 'Publish to NuGet.org'
type: boolean
default: false

resources:
repositories:
- repository: manylinux
Expand All @@ -61,5 +56,4 @@ stages:
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
ort_version: ${{ parameters.ort_version }}
cuda_version: ${{ parameters.cuda_version }}
publish_to_nuget: ${{ parameters.publish_to_nuget }}
publish_to_ado_feed: ${{ parameters.publish_to_ado_feed }}
6 changes: 6 additions & 0 deletions .pipelines/pypl-publishing.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ parameters:
- '11.8'
- '12.2'

- name: publish_to_ado_feed
displayName: 'Whether to publish the packages to ADO feed.'
type: boolean
default: false

resources:
repositories:
- repository: manylinux
Expand All @@ -50,3 +55,4 @@ stages:
enable_win_cuda: ${{ parameters.enable_win_cuda }}
ort_version: ${{ parameters.ort_version }}
cuda_version: ${{ parameters.cuda_version }}
publish_to_ado_feed: ${{ parameters.publish_to_ado_feed }}
29 changes: 16 additions & 13 deletions .pipelines/stages/jobs/nuget-packaging-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,20 @@ parameters:
default: ''
- name: os
type: string
values:
- 'linux'
- 'win'
- name: publish_to_ado_feed
type: boolean
- name: publish_to_nuget
type: boolean

jobs:
- job: nuget_${{ parameters.os }}_${{ parameters.ep }}_${{ parameters.arch }}_packaging
${{ if eq(parameters.os, 'linux') }}:
pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
${{ if eq(parameters.os, 'win') }}:
pool: 'onnxruntime-Win-CPU-2022'
timeoutInMinutes: 180
# set variables here to be used in the template and steps
# set variables here to be used in the template and steps
variables:
- name: arch
value: ${{ parameters.arch }}
Expand Down Expand Up @@ -60,18 +62,19 @@ jobs:
workspace:
clean: all
steps:
- template: steps/capi-${{ parameters.os }}-step.yml
parameters:
target: 'onnxruntime-genai'
- ${{ if eq(parameters.os, 'linux') }}:
- template: steps/capi-linux-step.yml
parameters:
target: 'onnxruntime-genai'
# TODO: Add a step to build the linux nuget package

# TODO: Add a step to build the linux nuget package
- ${{ if eq(parameters.os, 'win') }}:
- template: steps/nuget-${{ parameters.os }}-step.yml
- ${{ if or(eq(parameters.publish_to_nuget, true), eq(parameters.publish_to_ado_feed, true))}}:
- template: steps/nuget-releasing-step.yml
parameters:
publish_to_ado_feed: ${{ parameters.publish_to_ado_feed }}
publish_to_nuget: ${{ parameters.publish_to_nuget }}
- template: steps/capi-win-step.yml
parameters:
target: 'onnxruntime-genai'
- template: steps/nuget-win-step.yml
- ${{ if eq(parameters.publish_to_ado_feed, true)}}:
- template: steps/nuget-ado-feed-releasing-step.yml

- template: steps/compliant-and-cleanup-step.yml

64 changes: 0 additions & 64 deletions .pipelines/stages/jobs/py-linux-packaging-job.yml

This file was deleted.

105 changes: 105 additions & 0 deletions .pipelines/stages/jobs/py-packaging-job.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
parameters:
- name: arch
type: string
- name: ep
type: string
- name: ort_version
type: string
- name: cuda_version
type: string
default: ''
- name: os
type: string
values:
- 'linux'
- 'win'
- name: publish_to_ado_feed
type: boolean

jobs:
- job: python_${{ parameters.os }}_${{ parameters.ep }}_${{ parameters.arch }}_packaging
${{ if eq(parameters.os, 'linux') }}:
pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
${{ if eq(parameters.os, 'win') }}:
pool: 'onnxruntime-Win-CPU-2022'
strategy:
matrix:
Python38:
PyDotVer: '3.8'
PyNoDotVer: '38'
Python39:
PyDotVer: '3.9'
PyNoDotVer: '39'
Python310:
PyDotVer: '3.10'
PyNoDotVer: '310'
Python311:
PyDotVer: '3.11'
PyNoDotVer: '311'
Python312:
PyDotVer: '3.12'
PyNoDotVer: '312'
timeoutInMinutes: 240
workspace:
clean: all
# set variables here to be used in the template and steps
variables:
- name: skipComponentGovernanceDetection
${{ if eq(parameters.os, 'linux') }}:
value: true
${{ if eq(parameters.os, 'win') }}:
value: false
- name: arch
value: ${{ parameters.arch }}
- name: artifactName
value: 'onnxruntime-genai-${{ parameters.os }}-${{ parameters.ep }}-${{ parameters.arch }}'
- name: buildConfig
value: 'Release'
- name: buildDir
value: 'build/${{ parameters.ep }}'
- name: cuda_version
value: ${{ parameters.cuda_version }}
- name: ep
value: ${{ parameters.ep }}
- name: ort_version
value: ${{ parameters.ort_version }}
- name: ort_filename
${{ if eq(parameters.ep, 'cpu') }}:
value: 'onnxruntime-${{ parameters.os }}-${{ parameters.arch }}-${{ parameters.ort_version }}'
${{ else}}:
${{if eq(parameters.cuda_version, '11.8') }}:
value: 'onnxruntime-${{ parameters.os }}-${{ parameters.arch }}-gpu-${{ parameters.ort_version }}'
${{ else }}:
value: 'onnxruntime-${{ parameters.os }}-${{ parameters.arch }}-cuda12-${{ parameters.ort_version }}'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: $(PyDotVer)
addToPath: true
architecture: $(arch)
- task: PythonScript@0
inputs:
scriptSource: inline
script: |
import sys
import subprocess
subprocess.call(['pip', 'install', '-q', 'setuptools', 'wheel', 'build', 'packaging', 'twine'])
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'

- ${{ if eq(parameters.os, 'linux') }}:
- template: steps/capi-linux-step.yml
parameters:
target: 'python'

# Windows job needs to set the python version and install the required packages
- ${{ if eq(parameters.os, 'win') }}:
- template: steps/capi-win-step.yml
parameters:
target: 'python'

- ${{ if eq(parameters.publish_to_ado_feed, true)}}:
- template: steps/py-ado-feed-releasing-step.yml

- template: steps/compliant-and-cleanup-step.yml

71 changes: 0 additions & 71 deletions .pipelines/stages/jobs/py-win-packaging-job.yml

This file was deleted.

2 changes: 1 addition & 1 deletion .pipelines/stages/jobs/steps/capi-linux-step.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ steps:
- task: BinSkim@4
displayName: 'Run BinSkim'
inputs:
AnalyzeTargetGlob: '$(Build.Repository.LocalPath)/**/*.pyd'
AnalyzeTargetGlob: '$(Build.Repository.LocalPath)/build/**/*cpython*.so'
continueOnError: true

- bash: |
Expand Down
2 changes: 1 addition & 1 deletion .pipelines/stages/jobs/steps/capi-win-step.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ steps:
- ${{ if eq(parameters.target, 'onnxruntime-genai') }}:
- template: compliant/win-esrp-dll-step.yml
parameters:
FolderPath: '$(buildDir)'
FolderPath: '$(Build.Repository.LocalPath)\$(buildDir)'
DisplayName: 'ESRP - Sign C++ dlls'
Pattern: '*genai.dll'

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,3 @@
parameters:
- name: publish_to_ado_feed
type: boolean
- name: publish_to_nuget
type: boolean
steps:
- task: NuGetToolInstaller@1
inputs:
Expand Down Expand Up @@ -39,11 +34,10 @@ steps:
parameters:
packageFolder: '$(GDN_CODESIGN_TARGETDIRECTORY)'
#This task must be run on a Windows machine
- ${{ if eq(parameters.publish_to_ado_feed, true) }}:
- task: NuGetCommand@2
displayName: 'NuGet push to Azure DevOps Feed'
inputs:
command: push
packagesToPush: '$(GDN_CODESIGN_TARGETDIRECTORY)/*.nupkg'
publishVstsFeed: 'PublicPackages/onnxruntime-genai'
allowPackageConflicts: true
- task: NuGetCommand@2
displayName: 'NuGet push to Azure DevOps Feed'
inputs:
command: push
packagesToPush: '$(GDN_CODESIGN_TARGETDIRECTORY)/*.nupkg'
publishVstsFeed: 'PublicPackages/onnxruntime-genai'
allowPackageConflicts: true
10 changes: 10 additions & 0 deletions .pipelines/stages/jobs/steps/py-ado-feed-releasing-step.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
steps:
- task: TwineAuthenticate@1
inputs:
artifactFeed: PublicPackages/onnxruntime-genai
- script: 'python -m twine upload -r onnxruntime-genai --config-file $(PYPIRC_PATH) --non-interactive *.whl'
workingDirectory: '$(Build.ArtifactStagingDirectory)/wheel'
displayName: 'Uploading wheels to PublicPackages/onnxruntime-genai'
retryCountOnTaskFailure: 3
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
Loading

0 comments on commit f9e8e40

Please sign in to comment.