diff --git a/.pipelines/prchecks/PackageBuildPRCheck.yml b/.pipelines/prchecks/PackageBuildPRCheck.yml index 314773cada9..5d7b1fb4e13 100644 --- a/.pipelines/prchecks/PackageBuildPRCheck.yml +++ b/.pipelines/prchecks/PackageBuildPRCheck.yml @@ -80,9 +80,11 @@ extends: # GCC fails to build as a regular package. ignoredSpecs: ["gcc"] - - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)" + - script: | + echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)" + echo "##vso[task.setvariable variable=toolchainTarballName;isOutput=true]toolchain_built_rpms_all.tar.gz" name: "ToolchainArtifactName" - displayName: "Set variable for published artifact name" + displayName: "Set variables for published toolchain tarball" # 1. Automatic publishing won't work if 'isCustom: true' is set on the pool. We cannot do 'isCustom: false' because # then OneBranch attempts to perform additional actions (adding build tags for instance), which require additional permissions @@ -104,24 +106,38 @@ extends: isCustom: true name: ${{ configuration.agentPool }} variables: + inputArtifactsLocation: $(Agent.TempDirectory) ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt) ob_outputDirectory: $(Build.ArtifactStagingDirectory) + outputRPMsTarballName: "rpms.tar.gz" toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ] + toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ] steps: + - task: DownloadPipelineArtifact@2 + displayName: "Download toolchain" + inputs: + artifact: $(toolchainArtifactName) + patterns: "**/$(toolchainTarballName)" + targetPath: $(inputArtifactsLocation) + - template: .pipelines/templates/PackageBuild.yml@self parameters: checkBuildRetries: "1" - customToolchainArtifactName: $(toolchainArtifactName) + customToolchainTarballName: $(toolchainTarballName) + inputArtifactsFolder: $(inputArtifactsLocation) isCheckBuild: true isQuickRebuildPackages: true isUseCCache: true maxCPU: "${{ configuration.maxCPUs }}" outputArtifactsFolder: $(ob_outputDirectory) + outputRPMsTarballName: $(outputRPMsTarballName) pipArtifactFeeds: "mariner/Mariner-Pypi-Feed" selfRepoName: self testSuiteName: "[${{ configuration.name }}] Package test" - - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)" + - script: | + echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)" + echo "##vso[task.setvariable variable=rpmsTarballName;isOutput=true]$(outputRPMsTarballName)" name: "RPMsArtifactName" displayName: "Set variable for published artifact name" @@ -142,15 +158,25 @@ extends: isCustom: true name: ${{ configuration.agentPool }} variables: + inputArtifactsLocation: $(Agent.TempDirectory) ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt) ob_outputDirectory: $(Build.ArtifactStagingDirectory) testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ] toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ] + toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ] steps: + - task: DownloadPipelineArtifact@2 + displayName: "Download toolchain" + inputs: + artifact: $(toolchainArtifactName) + patterns: "**/$(toolchainTarballName)" + targetPath: $(inputArtifactsLocation) + - template: .pipelines/templates/PackageBuild.yml@self parameters: checkBuildRetries: "1" - customToolchainArtifactName: $(toolchainArtifactName) + customToolchainTarballName: $(toolchainTarballName) + inputArtifactsFolder: $(inputArtifactsLocation) isAllowToolchainRebuilds: true isCheckBuild: true isQuickRebuildPackages: true @@ -179,8 +205,18 @@ extends: isCustom: true name: ${{ configuration.agentPool }} variables: + inputArtifactsLocation: $(Agent.TempDirectory) rpmsArtifactName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsArtifactName'] ] + rpmsTarballName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsTarballName'] ] steps: + - task: DownloadPipelineArtifact@2 + displayName: "Download RPMs tarball" + inputs: + artifact: $(rpmsArtifactName) + patterns: "**/$(rpmsTarballName)" + targetPath: $(inputArtifactsLocation) + - template: .pipelines/templatesWithCheckout/SodiffCheck.yml@self parameters: - inputArtifactName: $(rpmsArtifactName) + inputArtifactsFolder: $(inputArtifactsLocation) + inputRPMsTarballName: $(rpmsTarballName) diff --git a/.pipelines/templates/PackageBuild.yml b/.pipelines/templates/PackageBuild.yml index 4ce30b7e65f..abb9ac1d9ca 100644 --- a/.pipelines/templates/PackageBuild.yml +++ b/.pipelines/templates/PackageBuild.yml @@ -14,13 +14,9 @@ parameters: type: number default: 12 - - name: customToolchainArtifactName - type: string - default: "" - - name: customToolchainTarballName type: string - default: "toolchain_built_rpms_all.tar.gz" + default: "" - name: extraPackageRepos type: string @@ -30,12 +26,16 @@ parameters: type: boolean default: true - - name: inputCacheArtifacts + - name: inputArtifactsFolder + type: string + default: "$(Agent.TempDirectory)" + + - name: inputCacheRPMsTarballs type: object default: [] # Sample: - # - name: build-artifacts - # rpmsTarball: cache.tar.gz + # - cache.tar.gz + # - cache2.tar.gz - name: isAllowToolchainRebuilds type: string @@ -160,15 +160,9 @@ steps: artifactFeeds: "${{ parameters.pipArtifactFeeds }}" displayName: "Authenticate to custom pip artifact feeds" - - ${{ if parameters.customToolchainArtifactName }}: - - task: DownloadPipelineArtifact@2 - displayName: "Download toolchain" - inputs: - artifact: "${{ parameters.customToolchainArtifactName }}" - patterns: "**/${{ parameters.customToolchainTarballName }}" - + - ${{ if parameters.customToolchainTarballName }}: - script: | - toolchain_archive="$(find "$(Pipeline.Workspace)" -name "${{ parameters.customToolchainTarballName }}" -print -quit)" + toolchain_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.customToolchainTarballName }}" -print -quit)" if [[ ! -f "$toolchain_archive" ]]; then echo "ERROR: toolchain archive not found!" >&2 exit 1 @@ -178,17 +172,11 @@ steps: sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" toolchain TOOLCHAIN_ARCHIVE="$toolchain_archive" displayName: "Populate toolchain" - - ${{ each inputCacheArtifact in parameters.inputCacheArtifacts }}: - - task: DownloadPipelineArtifact@2 - displayName: "Download input cache RPM from ${{ inputCacheArtifact.name }}" - inputs: - artifact: "${{ inputCacheArtifact.name }}" - patterns: "**/${{ inputCacheArtifact.rpmsTarball }}" - + - ${{ each inputCacheRPMsTarball in parameters.inputCacheRPMsTarballs }}: - script: | - rpms_archive="$(find "$(Pipeline.Workspace)" -name "${{ inputCacheArtifact.rpmsTarball }}" -print -quit)" + rpms_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ inputCacheRPMsTarball }}" -print -quit)" if [[ ! -f "$rpms_archive" ]]; then - echo "ERROR: cache RPMs archive '${{ inputCacheArtifact.rpmsTarball }}' not found!" >&2 + echo "ERROR: cache RPMs archive '${{ inputCacheRPMsTarball }}' not found!" >&2 exit 1 fi @@ -200,7 +188,7 @@ steps: check_build_retries_arg="CHECK_BUILD_RETRIES=${{ parameters.checkBuildRetries }}" fi - if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then + if [[ -n "${{ parameters.customToolchainTarballName }}" ]]; then toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)" fi diff --git a/.pipelines/templatesWithCheckout/SodiffCheck.yml b/.pipelines/templatesWithCheckout/SodiffCheck.yml index c4103caa78c..004fbaaa1e9 100644 --- a/.pipelines/templatesWithCheckout/SodiffCheck.yml +++ b/.pipelines/templatesWithCheckout/SodiffCheck.yml @@ -6,8 +6,9 @@ parameters: type: string default: "$(Build.SourcesDirectory)" - - name: inputArtifactName + - name: inputArtifactsFolder type: string + default: "$(Agent.TempDirectory)" - name: inputRPMsTarballName type: string @@ -26,19 +27,11 @@ parameters: default: "$(Agent.TempDirectory)/SourcesWorkspace" steps: - - task: DownloadPipelineArtifact@2 - displayName: "Download sources for signing" - inputs: - artifact: ${{ parameters.inputArtifactName }} - patterns: | - **/${{ parameters.inputRPMsTarballName }} - targetPath: "$(Agent.TempDirectory)" - - script: | set -e mkdir -p "${{ parameters.sourcesWorkspace }}" - find "$(Agent.TempDirectory)" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf + find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf displayName: "Extract sources tarball" - script: |