diff --git a/.pipelines/DSC-Official.yml b/.pipelines/DSC-Official.yml index 011b63a7..e1d55d51 100644 --- a/.pipelines/DSC-Official.yml +++ b/.pipelines/DSC-Official.yml @@ -1,6 +1,12 @@ name: DSC-Release-$(Build.BuildId) trigger: none +parameters: +- name: 'debugConsole' + displayName: 'Enable debug console' + type: boolean + default: false + pr: branches: include: @@ -23,6 +29,8 @@ resources: extends: template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates parameters: + featureFlags: + debugConsole: ${{ parameters.debugConsole }} customTags: 'ES365AIMigrationTooling' globalSdl: disableLegacyManifest: true @@ -58,7 +66,6 @@ extends: ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' steps: - checkout: self - target: host - pwsh: | $packageVersion = $(repoRoot)/build.ps1 -GetPackageVersion $vstsCommandString = "vso[task.setvariable variable=Version;isoutput=true]$packageVersion" @@ -71,52 +78,67 @@ extends: strategy: matrix: Windows x64: + Suffix: x64 buildName: x86_64-pc-windows-msvc Windows x64_arm64: + Suffix: arm64 buildName: aarch64-pc-windows-msvc variables: PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ] - ob_sdl_tsa_configFile: $(Build.SourcesDirectory)\DSC\.config\tsaoptions.json + ob_sdl_tsa_configFile: '$(Build.SourcesDirectory)\DSC\.config\tsaoptions.json' ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - repoRoot: $(Build.SourcesDirectory)\DSC - signSrcPath: $(repoRoot)/out - ob_artifactBaseName: 'DSC-$(buildName)' + ob_artifactSuffix: $(Suffix) + repoRoot: '$(Build.SourcesDirectory)\DSC' + signSrcPath: '$(Build.SourcesDirectory)\out' ob_sdl_sbom_enabled: true ob_signing_setup_enabled: true ob_sdl_codeql_compiled_enabled: false pool: type: windows - displayName: Build + displayName: BuildWin steps: - checkout: self - target: host + env: + ob_restore_phase: true - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step. inputs: Enabled: true AnalyzeInPipeline: true Language: rust + env: + ob_restore_phase: true - pwsh: | - $tmpdir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid()) - New-Item -ItemType Directory -Path $tmpdir + $tmpdir = "$(Agent.TempDirectory)" Write-Host "##vso[task.setvariable variable=CARGO_TARGET_DIR;]$tmpdir" displayName: 🛠️ Workaround for the LoadLibrary ACCESS_VIOLATION OneBranch issue + env: + ob_restore_phase: true - pwsh: | Set-Location "$(Build.SourcesDirectory)/DSC" ./build.ps1 -Release -Architecture $(buildName) -SkipLinkCheck displayName: 'Build $(buildName)' + env: + ob_restore_phase: true condition: succeeded() - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step. condition: always() + env: + ob_restore_phase: true - pwsh: | $null = New-Item -ItemType Directory -Path "$(PackageRoot)" -ErrorAction Ignore $null = New-Item -ItemType Directory -Path "$(PackageRoot)/out" -ErrorAction Ignore - $outPath = New-Item -ItemType Directory -Path "$(PackageRoot)/out/$(buildName)" -ErrorAction Ignore + $null = New-Item -ItemType Directory -Path "$(signSrcPath)" -ErrorAction Ignore # workaround known issue of building in OneBranch copying from TMP folder $null = New-Item -ItemType Directory -Path "$(signSrcPath)" -ErrorAction Ignore # copy only the exes from the TMP folder since it contains intermediately built files we don't want to sign - Copy-Item "$env:CARGO_TARGET_DIR/*.exe" "$(signSrcPath)" - # Copy-Item -Path "$(Build.SourcesDirectory)/DSC/bin/$(buildName)/$(BuildConfiguration)/*" -Destination $outPath -Verbose -Force - displayName: Copy binaries + write-host 'Binaries in $(env:CARGO_TARGET_DIR)' + Copy-Item -Path "$env:CARGO_TARGET_DIR/$(buildName)/$(BuildConfiguration)/*.exe" -Destination "$(signSrcPath)" -Verbose + Copy-Item -Path "$(Build.SourcesDirectory)/DSC/bin/$(buildName)/$(BuildConfiguration)/*" -Recurse -Destination "$(signSrcPath)" -Verbose -Force + write-host 'Binaries in $(signSrcPath)' + dir -r "$(signSrcPath)" + displayName: Copy built binaries + env: + ob_restore_phase: true condition: succeeded() - task: onebranch.pipeline.signing@1 displayName: Sign 1st party files @@ -124,24 +146,28 @@ extends: command: 'sign' signing_profile: external_distribution files_to_sign: | - *.exe; - *.json; - *.ps1; + **\*.exe; + **\*.ps1; + **\*.psd1; + **\*.psm1; search_root: $(signSrcPath) - task: CopyFiles@2 - displayName: "Copy signed files to ob_outputDirectory - '$(ob_outputDirectory)'" + displayName: "Copy signed files to build target dir" inputs: SourceFolder: "$(signSrcPath)" - Contents: '*' - TargetFolder: $(ob_outputDirectory) + Contents: '**' + TargetFolder: $(Build.SourcesDirectory)/DSC/bin/$(buildName)/$(BuildConfiguration) + OverWrite: true - pwsh: | - compress-archive -Path "$(ob_outputDirectory)/*" -DestinationPath "$(ob_outputDirectory)/DSC-$(PackageVersion)-$(buildName).zip" - displayName: 'Compress $(buildName)' + Set-Location "$(Build.SourcesDirectory)/DSC" + ./build.ps1 -PackageType zip -Architecture $(buildName) -Release + Copy-Item ./bin/*.zip "$(ob_outputDirectory)" + displayName: 'Zip $(buildName)' condition: succeeded() - pwsh: | Set-Location "$(Build.SourcesDirectory)/DSC" - ./build.ps1 -msix -skipbuild - Copy-Item *.msix "$(ob_outputDirectory)" + ./build.ps1 -PackageType msix -Architecture $(buildName) -Release -UseX64MakeAppx + Copy-Item ./bin/msix/*.msix "$(ob_outputDirectory)" -Verbose displayName: 'Create msix for $(buildName)' condition: succeeded() @@ -152,9 +178,20 @@ extends: pool: type: windows steps: + - download: current + artifact: drop_BuildAndSign_BuildWinx64 + patterns: '*.msix' + - download: current + artifact: drop_BuildAndSign_BuildWinarm64 + patterns: '*.msix' + - checkout: self - pwsh: | Set-Location "$(Build.SourcesDirectory)/DSC" - ./build.ps1 -msixbundle + $null = New-Item -ItemType Directory -Path "./bin/msix" -Force -ErrorAction Ignore + Copy-Item "$(Pipeline.Workspace)/drop_BuildAndSign_BuildWinx64/*.msix" ./bin/msix -Verbose + Copy-Item "$(Pipeline.Workspace)/drop_BuildAndSign_BuildWinarm64/*.msix" ./bin/msix -Verbose + ./build.ps1 -PackageType msixbundle + Copy-Item ./bin/*.msixbundle "$(ob_outputDirectory)" displayName: 'Create msixbundle' condition: succeeded() @@ -185,12 +222,10 @@ extends: steps: - pwsh: | ./build.ps1 -Release -Architecture x86_64-unknown-linux-gnu + ./build.ps1 -PackageType tgz -Architecture x86_64-unknown-linux-gnu -Release + Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)" displayName: 'Build x86_64-unknown-linux-gnu' condition: succeeded() - - pwsh: | - tar czf '$(ob_outputDirectory)/DSC-$(PackageVersion)-x86_64-unknown-linux-gnu.tar.gz' -C $(Build.SourcesDirectory)/bin/x86_64-unknown-linux-gnu/$(BuildConfiguration) . - displayName: 'Compress x86_64-unknown-linux-gnu' - condition: succeeded() - job: BuildLinuxArm64 dependsOn: SetPackageVersion @@ -204,19 +239,17 @@ extends: steps: - pwsh: | ./build.ps1 -Release -Architecture aarch64-unknown-linux-gnu + ./build.ps1 -PackageType tgz -Architecture aarch64-unknown-linux-gnu -Release + Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)" displayName: 'Build aarch64-unknown-linux-gnu' condition: succeeded() - - pwsh: | - tar czf '$(ob_outputDirectory)/DSC-$(PackageVersion)-aarch64-unknown-linux-gnu.tar.gz' -C $(Build.SourcesDirectory)/bin/aarch64-unknown-linux-gnu/$(BuildConfiguration) . - displayName: 'Compress aarch64-unknown-linux-gnu' - condition: succeeded() - job: BuildMac dependsOn: SetPackageVersion variables: PackageVersion: $[ dependencies.SetPackageVersion.outputs['Package.Version'] ] ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: Build + displayName: BuildMac pool: type: linux isCustom: true @@ -231,12 +264,11 @@ extends: steps: - pwsh: | ./build.ps1 -Release -Architecture $(buildName) + ./build.ps1 -PackageType tgz -Architecture $(buildName) -Release + Copy-Item ./bin/*.tar.gz "$(ob_outputDirectory)" + Write-Host "##vso[artifact.upload containerfolder=release;artifactname=release]$(ob_outputDirectory)/DSC-$(PackageVersion)-$(buildName).tar.gz" displayName: 'Build $(buildName)' condition: succeeded() - - pwsh: | - tar czf '$(ob_outputDirectory)/DSC-$(PackageVersion)-$(buildName).tar.gz' -C $(Build.SourcesDirectory)/bin/$(buildName)/$(BuildConfiguration) . - displayName: 'Compress $(buildName)' - condition: succeeded() - stage: Release dependsOn: BuildAndSign diff --git a/build.ps1 b/build.ps1 index 8077922d..f186ed7f 100644 --- a/build.ps1 +++ b/build.ps1 @@ -7,11 +7,12 @@ param( $architecture = 'current', [switch]$Clippy, [switch]$SkipBuild, - [switch]$Msix, - [switch]$MsixBundle, + [ValidateSet('msix','msixbundle','tgz','zip')] + $packageType, [switch]$Test, [switch]$GetPackageVersion, - [switch]$SkipLinkCheck + [switch]$SkipLinkCheck, + [switch]$UseX64MakeAppx ) if ($GetPackageVersion) { @@ -23,28 +24,45 @@ if ($GetPackageVersion) { return $match.Matches.Groups[1].Value } -## Test if Rust is installed -if (!(Get-Command 'cargo' -ErrorAction Ignore)) { - Write-Verbose -Verbose "Rust not found, installing..." - if (!$IsWindows) { - curl https://sh.rustup.rs -sSf | sh -s -- -y - $env:PATH += ":$env:HOME/.cargo/bin" - } - else { - Invoke-WebRequest 'https://static.rust-lang.org/rustup/dist/i686-pc-windows-gnu/rustup-init.exe' -OutFile 'temp:/rustup-init.exe' - Write-Verbose -Verbose "Use the default settings to ensure build works" - & 'temp:/rustup-init.exe' -y - $env:PATH += ";$env:USERPROFILE\.cargo\bin" - Remove-Item temp:/rustup-init.exe -ErrorAction Ignore - } -} +$filesForWindowsPackage = @( + 'dsc.exe', + 'assertion.dsc.resource.json', + 'group.dsc.resource.json', + 'powershell.dsc.resource.json', + 'PSDesiredStateConfiguration/', + 'psDscAdapter/', + 'reboot_pending.dsc.resource.json', + 'reboot_pending.resource.ps1', + 'registry.dsc.resource.json', + 'registry.exe', + 'RunCommandOnSet.dsc.resource.json', + 'RunCommandOnSet.exe', + 'windowspowershell.dsc.resource.json', + 'wmi.dsc.resource.json', + 'wmi.resource.ps1' +) -if ($IsLinux -and (Test-Path /etc/mariner-release)) { - tdnf install -y openssl-devel -} +$filesForLinuxPackage = @( + 'dsc', + 'assertion.dsc.resource.json', + 'group.dsc.resource.json', + 'powershell.dsc.resource.json', + 'psDscAdapter/', + 'RunCommandOnSet.dsc.resource.json', + 'runcommandonset' +) -rustup default stable -$BuildToolsPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC" +$filesForMacPackage = @( + 'dsc', + 'assertion.dsc.resource.json', + 'brew.dsc.resource.json', + 'brew.dsc.resource.sh', + 'group.dsc.resource.json', + 'powershell.dsc.resource.json', + 'psDscAdapter/', + 'RunCommandOnSet.dsc.resource.json', + 'runcommandonset' +) function Find-LinkExe { try { @@ -61,8 +79,32 @@ function Find-LinkExe { } } -if ($Msix -or $MsixBundle) { +if ($null -ne $packageType) { $SkipBuild = $true +} else { + ## Test if Rust is installed + if (!(Get-Command 'cargo' -ErrorAction Ignore)) { + Write-Verbose -Verbose "Rust not found, installing..." + if (!$IsWindows) { + curl https://sh.rustup.rs -sSf | sh -s -- -y + $env:PATH += ":$env:HOME/.cargo/bin" + } + else { + Invoke-WebRequest 'https://static.rust-lang.org/rustup/dist/i686-pc-windows-gnu/rustup-init.exe' -OutFile 'temp:/rustup-init.exe' + Write-Verbose -Verbose "Use the default settings to ensure build works" + & 'temp:/rustup-init.exe' -y + $env:PATH += ";$env:USERPROFILE\.cargo\bin" + Remove-Item temp:/rustup-init.exe -ErrorAction Ignore + } + } + + if ($IsLinux -and (Test-Path /etc/mariner-release)) { + tdnf install -y openssl-devel + } + + $BuildToolsPath = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC" + + rustup default stable } if (!$SkipBuild -and !$SkipLinkCheck -and $IsWindows -and !(Get-Command 'link.exe' -ErrorAction Ignore)) { @@ -111,35 +153,38 @@ else { } if (!$SkipBuild) { - if (Test-Path $target) { - Remove-Item $target -Recurse -ErrorAction Stop - } - New-Item -ItemType Directory $target > $null - -# make sure dependencies are built first so clippy runs correctly -$windows_projects = @("pal", "registry", "reboot_pending", "wmi-adapter") - -# projects are in dependency order -$projects = @( - "tree-sitter-dscexpression", - "security_context_lib", - "dsc_lib", - "dsc", - "osinfo", - "powershell-adapter", - "process", - "tools/dsctest", - "tools/test_group_resource", - "y2j", - "resources/brew", - "runcommandonset" -) -$pedantic_unclean_projects = @() -$clippy_unclean_projects = @("tree-sitter-dscexpression") -$skip_test_projects_on_windows = @("tree-sitter-dscexpression") + if (Test-Path $target) { + Remove-Item $target -Recurse -ErrorAction Ignore + } + New-Item -ItemType Directory $target -ErrorAction Ignore > $null + + # make sure dependencies are built first so clippy runs correctly + $windows_projects = @("pal", "registry", "reboot_pending", "wmi-adapter") + + # projects are in dependency order + $projects = @( + "tree-sitter-dscexpression", + "security_context_lib", + "dsc_lib", + "dsc", + "osinfo", + "powershell-adapter", + "process", + "resources/brew", + "runcommandonset", + "tools/dsctest", + "tools/test_group_resource", + "y2j" + ) + $pedantic_unclean_projects = @() + $clippy_unclean_projects = @("tree-sitter-dscexpression") + $skip_test_projects_on_windows = @("tree-sitter-dscexpression") if ($IsWindows) { $projects += $windows_projects + Save-Module -Path $target -Name 'PSDesiredStateConfiguration' -RequiredVersion '2.0.7' -Repository PSGallery -Force + # Need to unhide all the files so that packaging works + Get-ChildItem -Path $target -Recurse -Hidden | ForEach-Object { $_.Attributes = 'Normal' } } $failed = $false @@ -216,8 +261,8 @@ $skip_test_projects_on_windows = @("tree-sitter-dscexpression") } } -$relative = Resolve-Path $target -Relative -if (!$Clippy) { +if (!$Clippy -and !$SkipBuild) { + $relative = Resolve-Path $target -Relative Write-Host -ForegroundColor Green "`nEXE's are copied to $target ($relative)" # remove the other target in case switching between them @@ -318,7 +363,7 @@ function Find-MakeAppx() { $makeappx = Get-Command makeappx -CommandType Application -ErrorAction Ignore if ($null -eq $makeappx) { # try to find - if ($architecture -eq 'aarch64-pc-windows-msvc') { + if (!$UseX64MakeAppx -and $architecture -eq 'aarch64-pc-windows-msvc') { $arch = 'arm64' } else { @@ -334,24 +379,19 @@ function Find-MakeAppx() { $makeappx } -if ($MsixBundle) { - if ($Msix) { - throw "Creating MsixBundle requires all msix packages to already be created" - } +$productVersion = ((Get-Content $PSScriptRoot/dsc/Cargo.toml) -match '^version\s*=\s*') -replace 'version\s*=\s*"(.*?)"', '$1' +if ($packageType -eq 'msixbundle') { if (!$IsWindows) { throw "MsixBundle is only supported on Windows" } - $productVersion = ((Get-Content $PSScriptRoot/dsc/Cargo.toml) -match '^version\s*=\s*') -replace 'version\s*=\s*"(.*?)"', '$1' - $isPreview = $productVersion -like '*-*' $packageName = "DSC-$productVersion-Win" $makeappx = Find-MakeAppx - & $makeappx bundle /d $PSScriptRoot /p "$PSScriptRoot\$packageName.msixbundle" + $msixPath = Join-Path $PSScriptRoot 'bin' 'msix' + & $makeappx bundle /d $msixPath /p "$PSScriptRoot\bin\$packageName.msixbundle" return -} - -if ($Msix) { +} elseif ($packageType -eq 'msix') { if (!$IsWindows) { throw "MSIX is only supported on Windows" } @@ -363,7 +403,6 @@ if ($Msix) { $makeappx = Find-MakeAppx $makepri = Get-Item (Join-Path $makeappx.Directory "makepri.exe") -ErrorAction Stop $displayName = "DesiredStateConfiguration" - $productVersion = ((Get-Content $PSScriptRoot/dsc/Cargo.toml) -match '^version\s*=\s*') -replace 'version\s*=\s*"(.*?)"', '$1' $isPreview = $productVersion -like '*-*' $productName = "DesiredStateConfiguration" if ($isPreview) { @@ -389,25 +428,18 @@ if ($Msix) { $appxManifest = $appxManifest.Replace('$VERSION$', $ProductVersion).Replace('$ARCH$', $Arch).Replace('$PRODUCTNAME$', $productName).Replace('$DISPLAYNAME$', $displayName).Replace('$PUBLISHER$', $releasePublisher) $msixTarget = Join-Path $PSScriptRoot 'bin' $architecture 'msix' if (Test-Path $msixTarget) { - Remove-Item $msixTarget -Recurse -ErrorAction Stop + Remove-Item $msixTarget -Recurse -ErrorAction Stop -Force } New-Item -ItemType Directory $msixTarget > $null Set-Content -Path "$msixTarget\AppxManifest.xml" -Value $appxManifest -Force - $filesForMsix = @( - 'dsc.exe', - 'assertion.dsc.resource.json', - 'group.dsc.resource.json', - 'parallel.dsc.resource.json', - 'powershellgroup.dsc.resource.json', - 'powershellgroup.resource.ps1', - 'wmigroup.dsc.resource.json.optout', - 'wmigroup.resource.ps1' - ) - - foreach ($file in $filesForMsix) { - Copy-Item "$target\$file" $msixTarget -ErrorAction Stop + foreach ($file in $filesForWindowsPackage) { + if ((Get-Item "$target\$file") -is [System.IO.DirectoryInfo]) { + Copy-Item "$target\$file" "$msixTarget\$file" -Recurse -ErrorAction Stop + } else { + Copy-Item "$target\$file" $msixTarget -ErrorAction Stop + } } # Necessary image assets need to be in source assets folder @@ -440,12 +472,69 @@ if ($Msix) { } Write-Verbose "Creating msix package" -Verbose - $packageName = "$productName-$productVersion-$arch" - & $makeappx pack /o /v /h SHA256 /d $msixTarget /p (Join-Path -Path (Get-Location) -ChildPath "$packageName.msix") + + $targetFolder = Join-Path $PSScriptRoot 'bin' 'msix' + if (Test-Path $targetFolder) { + Remove-Item $targetFolder -Recurse -ErrorAction Stop -Force + } else { + New-Item -ItemType Directory $targetFolder > $null + } + + $packageName = Join-Path $targetFolder "$productName-$productVersion-$arch.msix" + & $makeappx pack /o /v /h SHA256 /d $msixTarget /p $packageName if ($LASTEXITCODE -ne 0) { throw "Failed to create msix package" } - Write-Verbose "Created $packageName.msix" -Verbose + + Write-Host -ForegroundColor Green "`nMSIX package is created at $packageName.msix" +} elseif ($packageType -eq 'zip') { + $zipTarget = Join-Path $PSScriptRoot 'bin' $architecture 'zip' + if (Test-Path $zipTarget) { + Remove-Item $zipTarget -Recurse -ErrorAction Stop -Force + } + + New-Item -ItemType Directory $zipTarget > $null + + foreach ($file in $filesForWindowsPackage) { + if ((Get-Item "$target\$file") -is [System.IO.DirectoryInfo]) { + Copy-Item "$target\$file" "$zipTarget\$file" -Recurse -ErrorAction Stop + } else { + Copy-Item "$target\$file" $zipTarget -ErrorAction Stop + } + } + + $packageName = "DSC-$productVersion-$architecture.zip" + $zipFile = Join-Path $PSScriptRoot 'bin' $packageName + Compress-Archive -Path "$zipTarget/*" -DestinationPath $zipFile -Force + Write-Host -ForegroundColor Green "`nZip file is created at $zipFile" +} elseif ($packageType -eq 'tgz') { + $tgzTarget = Join-Path $PSScriptRoot 'bin' $architecture 'tgz' + if (Test-Path $tgzTarget) { + Remove-Item $tgzTarget -Recurse -ErrorAction Stop -Force + } + + New-Item -ItemType Directory $tgzTarget > $null + + if ($IsLinux) { + $filesForPackage = $filesForLinuxPackage + } elseif ($IsMacOS) { + $filesForPackage = $filesForMacPackage + } else { + Write-Error "Unsupported platform for tgz package" + } + + foreach ($file in $filesForPackage) { + if ((Get-Item "$target\$file") -is [System.IO.DirectoryInfo]) { + Copy-Item "$target\$file" "$tgzTarget\$file" -Recurse -ErrorAction Stop + } else { + Copy-Item "$target\$file" $tgzTarget -ErrorAction Stop + } + } + + $packageName = "DSC-$productVersion-$architecture.tar.gz" + $tgzFile = Join-Path $PSScriptRoot 'bin' $packageName + tar cvf $tgzFile -C $tgzTarget . + Write-Host -ForegroundColor Green "`nTgz file is created at $tgzFile" } $env:RUST_BACKTRACE=1