From 907dd9fa81503e9ab14250dda80985324bf0d960 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 15 Dec 2023 20:18:27 -0800 Subject: [PATCH 1/2] Update Nuget publishing jobs --- .../nuget-cuda-publishing-pipeline.yml | 28 +++---- .../github/azure-pipelines/publish-nuget.yml | 41 +++++++++- .../stages/nuget-cuda-publishing-stage.yml | 79 +++++++++++-------- 3 files changed, 99 insertions(+), 49 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml index 0332be4883e2d..2801466e52539 100644 --- a/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml @@ -1,24 +1,22 @@ +resources: + pipelines: + - pipeline: build + source: 'Nuget-CUDA-Packaging-Pipeline' + trigger: + branches: + include: + - main + branch: main + parameters: - name: nightly - type: string - default: '1' - - name: build_id - type: string - default: 'latest' - - name: project - type: string - default: 'Lotus' - - name: pipeline - type: string - default: 'Nuget-CUDA-Packaging-Pipeline' + type: boolean + default: true stages: - template: stages/nuget-cuda-publishing-stage.yml parameters: - build_id: ${{ parameters.build_id }} - project: ${{ parameters.project }} - pipeline: ${{ parameters.pipeline }} - ${{ if ne(parameters.nightly, '1') }}: + ${{ if ne(parameters.nightly, true) }}: artifact_feed: onnxruntime-cuda-12 ${{ else }}: artifact_feed: ort-cuda-12-nightly \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/publish-nuget.yml b/tools/ci_build/github/azure-pipelines/publish-nuget.yml index 8e029f4e679b2..19ede05eb12bd 100644 --- a/tools/ci_build/github/azure-pipelines/publish-nuget.yml +++ b/tools/ci_build/github/azure-pipelines/publish-nuget.yml @@ -2,7 +2,10 @@ resources: pipelines: - pipeline: build source: 'Zip-Nuget-Java-Nodejs Packaging Pipeline' - trigger: true + trigger: + branches: + include: + - main branch: main stages: @@ -13,7 +16,7 @@ stages: clean: all variables: - name: GDN_CODESIGN_TARGETDIRECTORY - value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + value: '$(Agent.TempDirectory)\binfiles' pool: 'onnxruntime-Win-CPU-2022' steps: @@ -92,6 +95,40 @@ stages: artifact: 'drop-signed-nuget-ROCm' - script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-ROCm\*" $(Build.BinariesDirectory)\nuget-artifact\final-package + - script: | + dir $(Build.BinariesDirectory)\nuget-artifact\final-package + cd $(Build.BinariesDirectory)\nuget-artifact\final-package + nuget verify -Signatures *.nupkg + displayName: List Downloaded Package + + - powershell: | + New-Item -Path $(Agent.TempDirectory) -Name "binfiles" -ItemType "directory" + $base_path_name = Join-Path -Path $(Agent.TempDirectory) -ChildPath "binfiles" + Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\final-package -Filter *.nupkg | + Foreach-Object { + $dir_name = Join-Path -Path $base_path_name -ChildPath $_.Basename + $cmd = "7z.exe x $($_.FullName) -y -o$dir_name" + Write-Output $cmd + Invoke-Expression -Command $cmd + } + dir $(Agent.TempDirectory) + tree $(Agent.TempDirectory) + workingDirectory: '$(Agent.TempDirectory)' + + - task: CodeSign@1 + displayName: 'Run Codesign Validation' + + + - task: PublishSecurityAnalysisLogs@3 + displayName: 'Publish Security Analysis Logs' + continueOnError: true + + - task: PostAnalysis@2 + inputs: + GdnBreakAllTools: true + GdnBreakPolicy: M365 + GdnBreakPolicyMinSev: Error + #TODO: allow choosing different feeds - task: NuGetCommand@2 displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY' diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml index 3699d5b24ae12..0e8f5c867cc12 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml @@ -1,48 +1,63 @@ parameters: - - name: build_id - type: string - - name: project - type: string - - name: pipeline - type: string - name: artifact_feed type: string default: 'onnxruntime-cuda-12' - - name: dependencies - type: string - default: 'none' stages: - stage: NuGet_Publishing_GPU - ${{ if ne(parameters.dependencies, 'none') }}: - dependsOn: - ${{ if eq(parameters.dependencies, 'none') }}: - dependsOn: [] jobs: - job: + workspace: + clean: all + variables: + - name: GDN_CODESIGN_TARGETDIRECTORY + value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' pool: 'onnxruntime-Win-CPU-2022' steps: - checkout: none - - script: | - echo "Project: ${{ parameters.project }}" - echo "Build ID: ${{ parameters.build_id }}" - echo "Pipeline: ${{ parameters.pipeline }}" - echo "Artifact Feed: ${{ parameters.artifact_feed }}" - displayName: 'Print Parameters' - - task: DownloadPipelineArtifact@2 - displayName: 'Download NuGet artifact drop-signed-nuget-GPU' + + - task: NuGetToolInstaller@1 + inputs: + versionSpec: 6.8.x + + - script: mkdir "$(Build.BinariesDirectory)\nuget-artifact\final-package" + + - download: build + displayName: 'Download Pipeline Artifact - Signed NuGet Package' + artifact: 'drop-signed-nuget-GPU' + + - script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-GPU\*" "$(Build.BinariesDirectory)\nuget-artifact\final-package" + + - powershell: | + New-Item -Path $(Agent.TempDirectory) -Name "binfiles" -ItemType "directory" + $base_path_name = Join-Path -Path $(Agent.TempDirectory) -ChildPath "binfiles" + Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\final-package -Filter *.nupkg | + Foreach-Object { + $dir_name = Join-Path -Path $base_path_name -ChildPath $_.Basename + $cmd = "7z.exe x $($_.FullName) -y -o$dir_name" + Write-Output $cmd + Invoke-Expression -Command $cmd + } + dir $(Agent.TempDirectory) + tree $(Agent.TempDirectory) + workingDirectory: '$(Agent.TempDirectory)' + + - task: CodeSign@1 + displayName: 'Run Codesign Validation' + + - task: CodeSign@1 + displayName: 'Run Codesign Validation' + + - task: PublishSecurityAnalysisLogs@3 + displayName: 'Publish Security Analysis Logs' + continueOnError: true + + - task: PostAnalysis@2 inputs: - artifact: drop-signed-nuget-GPU - targetPath: $(Build.BinariesDirectory)/nuget-artifact/final-package - ${{ if ne(parameters.build_id, 'latest') }}: - buildType: 'specific' - project: '${{ parameters.project }}' - pipeline: '${{ parameters.pipeline }}' - buildVersionToDownload: 'specific' - buildId: '${{ parameters.build_id }}' - - script: | - ls $(Build.BinariesDirectory)/nuget-artifact/final-package - displayName: List Downloaded Package + GdnBreakAllTools: true + GdnBreakPolicy: M365 + GdnBreakPolicyMinSev: Error + - template: ../nuget/templates/get-nuget-package-version-as-variable.yml parameters: packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package' From ab29122545aea9664efe754c4d022f573460bb32 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 19 Dec 2023 14:52:21 -0800 Subject: [PATCH 2/2] update --- .../azure-pipelines/stages/nuget-cuda-publishing-stage.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml index 0e8f5c867cc12..252b96e54bab0 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-cuda-publishing-stage.yml @@ -42,9 +42,6 @@ stages: tree $(Agent.TempDirectory) workingDirectory: '$(Agent.TempDirectory)' - - task: CodeSign@1 - displayName: 'Run Codesign Validation' - - task: CodeSign@1 displayName: 'Run Codesign Validation'