From 228713f635caeba12d0aa7e254772b8b7a230e17 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 29 May 2024 16:24:23 -0700 Subject: [PATCH] adding publishing stage to publish java CUDA 12 pkg to ado (#20834) --- java/build.gradle | 59 +++++++++++++++---- java/settings.xml | 9 +++ .../nuget-cuda-publishing-pipeline.yml | 20 +++++-- .../stages/java-cuda-publishing-stage.yml | 56 ++++++++++++++++++ 4 files changed, 125 insertions(+), 19 deletions(-) create mode 100644 java/settings.xml create mode 100644 tools/ci_build/github/azure-pipelines/stages/java-cuda-publishing-stage.yml diff --git a/java/build.gradle b/java/build.gradle index fd66ec220b78f..da43554cef9c8 100644 --- a/java/build.gradle +++ b/java/build.gradle @@ -4,6 +4,7 @@ plugins { id 'signing' id 'jacoco' id "com.diffplug.spotless" version "6.25.0" + id "net.linguica.maven-settings" version "0.5" } allprojects { @@ -19,6 +20,12 @@ version = rootProject.file('../VERSION_NUMBER').text.trim() def cmakeBuildDir = System.properties['cmakeBuildDir'] def useCUDA = System.properties['USE_CUDA'] def useROCM = System.properties['USE_ROCM'] + +def adoArtifact = project.findProperty('adoArtifact') +def adoAccessToken = project.findProperty('adoAccessToken') +// Only publish to ADO feed if all two properties are set +def publishToAdo = adoArtifact != null && adoAccessToken != null + boolean enableTrainingApis = (System.properties['ENABLE_TRAINING_APIS'] ?: "0") == "1" def cmakeJavaDir = "${cmakeBuildDir}/java" def cmakeNativeLibDir = "${cmakeJavaDir}/native-lib" @@ -37,6 +44,11 @@ def trainingDescription = 'ONNX Runtime Training is a training and inference pac '(Open Neural Network Exchange) models. This package is targeted for Learning on The Edge aka On-Device Training ' + 'See https://github.com/microsoft/onnxruntime-training-examples/tree/master/on_device_training for more details.' +// We need to have a custom settings.xml so codeql can bypass the need for settings.security.xml +mavenSettings { + userSettingsFileName = "${projectDir}/settings.xml" +} + java { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 @@ -206,9 +218,13 @@ publishing { publications { maven(MavenPublication) { groupId = project.group - artifactId = mavenArtifactId - - from components.java + if(publishToAdo) { + artifactId = 'onnxruntime_gpu' + artifact (adoArtifact) + } else { + artifactId = mavenArtifactId + from components.java + } pom { name = enableTrainingApis ? 'onnxruntime-training' : 'onnx-runtime' description = enableTrainingApis ? trainingDescription : defaultDescription @@ -239,24 +255,41 @@ publishing { } } repositories { - maven { - url 'https://oss.sonatype.org/service/local/staging/deploy/maven2/' - credentials { - username mavenUser - password mavenPwd + if (publishToAdo) { + maven { + url "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/${System.getenv('ADOFeedName')}/maven/v1" + name System.getenv('ADOFeedName') + authentication { + basic(BasicAuthentication) + } + credentials { + username 'aiinfra' + password "${project.findProperty('adoAccessToken')}" + } + } + } else { + maven { + url 'https://oss.sonatype.org/service/local/staging/deploy/maven2/' + credentials { + username mavenUser + password mavenPwd + } } } } } - // Generates a task signMavenPublication that will // build all artifacts. signing { // Queries env vars: // ORG_GRADLE_PROJECT_signingKey // ORG_GRADLE_PROJECT_signingPassword but can be changed to properties - def signingKey = findProperty("signingKey") - def signingPassword = findProperty("signingPassword") - useInMemoryPgpKeys(signingKey, signingPassword) - sign publishing.publications.maven + def signingKey = findProperty("signingKey") + def signingPassword = findProperty("signingPassword") + // Skip signing if no key is provided + if (signingKey != null && signingPassword != null) { + useInMemoryPgpKeys(signingKey, signingPassword) + sign publishing.publications.maven + sign publishing.publications.mavenAdo + } } diff --git a/java/settings.xml b/java/settings.xml new file mode 100644 index 0000000000000..21d7d0fb56c96 --- /dev/null +++ b/java/settings.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml index 2801466e52539..b0cc253ae0973 100644 --- a/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget-cuda-publishing-pipeline.yml @@ -9,14 +9,22 @@ resources: branch: main parameters: - - name: nightly + - name: isReleaseBuild type: boolean - default: true + default: false + +variables: + - name: ArtifactFeed + ${{ if eq(parameters.isReleaseBuild, false) }}: + value: ort-cuda-12-nightly + ${{ else }}: + value: onnxruntime-cuda-12 stages: - template: stages/nuget-cuda-publishing-stage.yml parameters: - ${{ if ne(parameters.nightly, true) }}: - artifact_feed: onnxruntime-cuda-12 - ${{ else }}: - artifact_feed: ort-cuda-12-nightly \ No newline at end of file + artifact_feed: $(ArtifactFeed) + +- template: stages/java-cuda-publishing-stage.yml + parameters: + artifact_feed: $(ArtifactFeed) diff --git a/tools/ci_build/github/azure-pipelines/stages/java-cuda-publishing-stage.yml b/tools/ci_build/github/azure-pipelines/stages/java-cuda-publishing-stage.yml new file mode 100644 index 0000000000000..70d92286b3964 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/stages/java-cuda-publishing-stage.yml @@ -0,0 +1,56 @@ +parameters: +- name: artifact_feed + type: string + +stages: +- stage: JAR_Publishing_GPU + dependsOn: [] + jobs: + - job: JAR_Publishing_GPU + #TD-DO: figure out a way to package nightly jar. Currently Java version are set from VERSION_NUMBER file + condition: ${{ eq(parameters.artifact_feed, 'onnxruntime-cuda-12') }} + workspace: + clean: all + pool: 'onnxruntime-Win-CPU-2022' + variables: + - name: SYSTEM_ACCESSTOKEN + value: $(System.AccessToken) + - name: ADOFeedName + value: ${{ parameters.artifact_feed }} + - name: GDN_CODESIGN_TARGETDIRECTORY + value: '$(Build.BinariesDirectory)/final-package' + - name: artifactName + value: onnxruntime-java-gpu + + steps: + - script: mkdir "$(GDN_CODESIGN_TARGETDIRECTORY)" + + - download: build + displayName: 'Download Pipeline Artifact - $(artifactName)' + artifact: '$(artifactName)' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Pipeline.Workspace)/build/$(artifactName)' + Contents: | + onnxruntime_gpu*.jar + onnxruntime_gpu*.pom + TargetFolder: '$(GDN_CODESIGN_TARGETDIRECTORY)' + CleanTargetFolder: true + + - task: PowerShell@2 + displayName: 'Bundle Jar and POM files into a single jar file' + inputs: + targetType: 'inline' + script: | + jar cvf bundle.jar ` + onnxruntime_gpu-*.pom ` + onnxruntime_gpu-*.jar + workingDirectory: '$(GDN_CODESIGN_TARGETDIRECTORY)' + + - task: Gradle@3 + inputs: + gradleWrapperFile: '$(Build.SourcesDirectory)/java/gradlew.bat' + workingDirectory: '$(Build.SourcesDirectory)/java' + tasks: 'publish' + options: '-PadoAccessToken=$(SYSTEM_ACCESSTOKEN) -PadoArtifact=$(GDN_CODESIGN_TARGETDIRECTORY)/bundle.jar'