From f4045eb9b8e679171348f067f0ea2cfcd2e5bce5 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:42:22 -0500 Subject: [PATCH 1/5] Merge Swagger and TypeSpec code generation into one script --- eng/pipelines/templates/jobs/ci.yml | 2 +- .../Compare-CurrentToCodegeneration.ps1 | 97 +++++++++++++---- ...peSpec-Compare-CurrentToCodegeneration.ps1 | 102 ------------------ 3 files changed, 79 insertions(+), 122 deletions(-) delete mode 100644 eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1 diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 6e92a8cdd4f86..e7334422aa621 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -349,7 +349,7 @@ jobs: JdkVersion: ${{ parameters.JavaBuildVersion }} - task: PowerShell@2 - displayName: Verify Code Generation + displayName: Verify Autorest and TypeSpec Code Generation inputs: pwsh: true filePath: $(Build.SourcesDirectory)/eng/scripts/Compare-CurrentToCodegeneration.ps1 diff --git a/eng/scripts/Compare-CurrentToCodegeneration.ps1 b/eng/scripts/Compare-CurrentToCodegeneration.ps1 index dfb644209f70a..d124aaf6f6573 100644 --- a/eng/scripts/Compare-CurrentToCodegeneration.ps1 +++ b/eng/scripts/Compare-CurrentToCodegeneration.ps1 @@ -1,44 +1,54 @@ <# .SYNOPSIS Invokes all 'Update-Codegeneration.ps1' scripts found within the specified directory and compares it against current -code. +code and syncs TypeSpec defintion and generates SDK for RPs under the input directory and have 'tsp-location.yaml'. .DESCRIPTION Invokes all 'Update-Codegeneration.ps1' scripts found within the specified directory and compares it against current -code. +code and syncs TypeSpec defintion and generates SDK for RPs under the input directory and have 'tsp-location.yaml'. If the regenerated code is different than the current code this will tell the differences, the files the differences are in, and exit with a failure status. .PARAMETER Directory -The directory that will be searched for 'Update-Codegeneration.ps1' scripts. The default is the root directory of the -Azure SDK for Java repository. CI jobs should use the 'ServiceDirectory', such as /sdk/storage. +The directory that will be searched for 'Update-Codegeneration.ps1' scripts and 'tsp-location.yaml'. The default is +the root directory of the Azure SDK for Java repository. One can also input service directory like: +'-Directory /sdk/storage' or '-Directory sdk/anomalydetector/azure-ai-anomalydetector'. #> param( - [Parameter(Mandatory = $false)] - [string]$Directory + [Parameter(Mandatory = $false)] + [string]$Directory ) +function Reset-Repository { + # Clean up generated code, so that next step will not be affected. + git reset --hard + git clean -fd . +} + $path = "" if ($Directory) { - $path = $Directory + $path = $Directory } $swaggers = Get-ChildItem -Path $path -Filter "Update-Codegeneration.ps1" -Recurse -if ($swaggers.Count -eq 0) { - Write-Host " +$tspYamls = Get-ChildItem -Path $path -Filter "tsp-location.yaml" -Recurse +if ($swaggers.Count -eq 0 -and $tspYamls.Count -eq 0) { + Write-Host " - =========================================== - No Swagger files to regenerate - =========================================== +=========================================== +No Swagger or TypeSpec files to regenerate +=========================================== - " - exit 0 +" + exit 0 } - -Write-Host " +# Stores SDKs that failed to regenerate code successfully +$failedSdk = $null +if ($swaggers.Count -gt 0) { + Write-Host " =================================== Invoking Autorest code regeneration @@ -46,8 +56,49 @@ Invoking Autorest code regeneration " -foreach ($script in $swaggers) { - Invoke-Expression $script.FullName + foreach ($script in $swaggers) { + Invoke-Expression $script.FullName + if ($LastExitCode -ne 0) { + $failedSdk += $script.Directory.FullName + } + } +} + +if ($tspYamls.Count -gt 0) { + Write-Host " + +=========================================== +Installing typespec-client-generator-cli +=========================================== + +" + + npm install -g @azure-tools/typespec-client-generator-cli + + Write-Host " + +=========================================== +Invoking tsp-client update +=========================================== + +" + foreach ($tspLocationPath in $tpsYamls) { + $sdkPath = (get-item $tspLocationPath).Directory.FullName + Write-Host "Generate SDK for $sdkPath" + Push-Location + Set-Location -Path $sdkPath + tsp-client update + if ($LastExitCode -ne 0) { + $failedSdk += $sdkPath + } + Pop-Location + } +} + +if ($failedSdk.Length -gt 0) { + Write-Host "Code generation failed for following modules: $failedSdk" + Reset-Repository + exit 1 } Write-Host " @@ -59,7 +110,7 @@ Verify no diff " # prevent warning related to EOL differences which triggers an exception for some reason -& git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" +git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" ":(exclude)**/src/test/**" ":(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" if ($LastExitCode -ne 0) { $status = git status -s | Out-String @@ -67,5 +118,13 @@ if ($LastExitCode -ne 0) { The following files are out of date: $status " + Reset-Repository exit 1 } + +# Delete out TypeSpec temporary folders if they still exist. +Get-ChildItem -Path $path -Filter TempTypeSpecFiles -Recurse -Directory | ForEach-Object { + Remove-Item -Path $_.FullName -Recurse -Force +} + +Reset-Repository diff --git a/eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1 b/eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1 deleted file mode 100644 index db1ba8e35fbc6..0000000000000 --- a/eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1 +++ /dev/null @@ -1,102 +0,0 @@ -<#yaml -.SYNOPSIS -Sync TypeSpec defintion and generate SDK for RPs under the input directory and have 'tsp-location.yaml'. - -.DESCRIPTION -Sync TypeSpec defintion and generate SDK for RPs under the input directory and have 'tsp-location.yaml'. - -If the regenerated code is different from current code, this will tell the files the differences -are in, and exit with a failure status. - -.PARAMETER Directory -The directory that will be used to get 'tsp-location.yaml' and generate SDK. The default is the root directory of the -Azure SDK for Java repository. One can also input service directory like: /sdk/storage, sdk/anomalydetector/azure-ai-anomalydetector. -#> - -param( - [Parameter(Mandatory = $false)] - [string]$Directory -) - -function Reset-Repository { - # Clean up generated code, so that next step will not be affected. - git reset --hard - git clean -fd . -} - -$tpsYamls = Get-ChildItem -Path $Directory -Filter "tsp-location.yaml" -Recurse -if ($tspYamls.Count -eq 0) { - Write-Host " - - =========================================== - No TypeSpec files to regenerate - =========================================== - - " - exit 0 -} - -Write-Host " - -=========================================== -Installing typespec-client-generator-cli -=========================================== - -" - -npm install -g @azure-tools/typespec-client-generator-cli - -Write-Host " - -=========================================== -Invoking tsp-client update -=========================================== - -" - -$failedSdk = $null -foreach ($tspLocationPath in $tpsYamls) { - $sdkPath = (get-item $tspLocationPath).Directory.FullName - Write-Host "Generate SDK for $sdkPath" - Push-Location - Set-Location -Path $sdkPath - tsp-client update - if ($LastExitCode -ne 0) { - $failedSdk += $sdkPath - } - Pop-Location -} - -if ($failedSdk.Length -gt 0) { - Write-Host "Code generation failed for following modules: $failedSdk" - Reset-Repository - exit 1 -} - -Write-Host " - -============== -Verify no diff -============== - -" - -# prevent warning related to EOL differences which triggers an exception for some reason -git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" ":(exclude)**/src/test/**" ":(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" - -if ($LastExitCode -ne 0) { - $status = git status -s | Out-String - Write-Host " -The following files are out of date: -$status -" - Reset-Repository - exit 1 -} - -# Delete out TypeSpec temporary folders if they still exist. -Get-ChildItem -Path $Directory -Filter TempTypeSpecFiles -Recurse -Directory | ForEach-Object { - Remove-Item -Path $_.FullName -Recurse -Force -} - -Reset-Repository From c77f489578b6605af48a1e24fe0a241725cc493d Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:46:37 -0500 Subject: [PATCH 2/5] Fix one more typo --- eng/scripts/Compare-CurrentToCodegeneration.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Compare-CurrentToCodegeneration.ps1 b/eng/scripts/Compare-CurrentToCodegeneration.ps1 index d124aaf6f6573..f40bbd36a0eb4 100644 --- a/eng/scripts/Compare-CurrentToCodegeneration.ps1 +++ b/eng/scripts/Compare-CurrentToCodegeneration.ps1 @@ -82,7 +82,7 @@ Invoking tsp-client update =========================================== " - foreach ($tspLocationPath in $tpsYamls) { + foreach ($tspLocationPath in $tspYamls) { $sdkPath = (get-item $tspLocationPath).Directory.FullName Write-Host "Generate SDK for $sdkPath" Push-Location From 4e51c6f0f0ecf7761298a45e39dabd9d633272e3 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:35:38 -0500 Subject: [PATCH 3/5] Remove call to deleted script, support TypeSpec ignore error --- eng/pipelines/templates/jobs/ci.yml | 10 +-- .../Compare-CurrentToCodegeneration.ps1 | 61 ++++++++++++++----- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index e7334422aa621..e9940ba1f6ddb 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -355,15 +355,7 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Compare-CurrentToCodegeneration.ps1 arguments: > -Directory sdk/${{ parameters.ServiceDirectory }} - - - task: PowerShell@2 - displayName: Verify TypeSpec Code Generation - continueOnError: ${{ eq(parameters.IgnoreVerifyTypeSpecCodeGenerationError, 'true') }} - inputs: - pwsh: true - filePath: $(Build.SourcesDirectory)/eng/scripts/TypeSpec-Compare-CurrentToCodegeneration.ps1 - arguments: > - -Directory sdk/${{ parameters.ServiceDirectory }} + -IgnoreVerifyTypeSpecCodeGenerationError ${{ parameters.IgnoreVerifyTypeSpecCodeGenerationError }} - template: /eng/pipelines/templates/steps/run-and-validate-linting.yml parameters: diff --git a/eng/scripts/Compare-CurrentToCodegeneration.ps1 b/eng/scripts/Compare-CurrentToCodegeneration.ps1 index f40bbd36a0eb4..8b5bfca876a32 100644 --- a/eng/scripts/Compare-CurrentToCodegeneration.ps1 +++ b/eng/scripts/Compare-CurrentToCodegeneration.ps1 @@ -14,11 +14,24 @@ are in, and exit with a failure status. The directory that will be searched for 'Update-Codegeneration.ps1' scripts and 'tsp-location.yaml'. The default is the root directory of the Azure SDK for Java repository. One can also input service directory like: '-Directory /sdk/storage' or '-Directory sdk/anomalydetector/azure-ai-anomalydetector'. + +.PARAMETER IgnoreVerifyTypeSpecCodeGenerationError +Determines if the script should exit with an error if the TypeSpec code generation fails. The default is false. +The script will exit with an error if the Swagger code generation fails. + +.PARAMETER UpdateOnly +Determines if the script should only update the code generation and not verify the changes. The default is false. #> param( [Parameter(Mandatory = $false)] - [string]$Directory + [string]$Directory, + + [Parameter(Mandatory = $false)] + [boolean]$IgnoreVerifyTypeSpecCodeGenerationError = $false, + + [Parameter(Mandatory = $false)] + [boolean]$UpdateOnly = $false ) function Reset-Repository { @@ -46,7 +59,10 @@ No Swagger or TypeSpec files to regenerate } # Stores SDKs that failed to regenerate code successfully -$failedSdk = $null +$swaggerFailedSdk = $null +$swaggerDiff = $null +$typespecFailedSdk = $null +$typespecDiff = $null if ($swaggers.Count -gt 0) { Write-Host " @@ -59,9 +75,12 @@ Invoking Autorest code regeneration foreach ($script in $swaggers) { Invoke-Expression $script.FullName if ($LastExitCode -ne 0) { - $failedSdk += $script.Directory.FullName + $swaggerFailedSdk += $script.Directory.FullName } } + + # prevent warning related to EOL differences which triggers an exception for some reason + $swaggerDiff = git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" } if ($tspYamls.Count -gt 0) { @@ -89,16 +108,20 @@ Invoking tsp-client update Set-Location -Path $sdkPath tsp-client update if ($LastExitCode -ne 0) { - $failedSdk += $sdkPath + $typespecFailedSdk += $sdkPath } Pop-Location } + + # prevent warning related to EOL differences which triggers an exception for some reason + $typespecDiff = git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" ":(exclude)**/src/test/**" ":(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" } -if ($failedSdk.Length -gt 0) { - Write-Host "Code generation failed for following modules: $failedSdk" +if ($swaggerFailedSdk.Length -gt 0 -or $typespecFailedSdk.Length -gt 0) { + Write-Host "Code generation failed for following modules: $swaggerFailedSdk $typespecFailedSdk" Reset-Repository - exit 1 + # Only exit with an error if Swagger had a failure or TypeSpec had a failure and IgnoreVerifyTypeSpecCodeGenerationError is false + exit ($swaggerFailedSdk.Length -gt 0 -or ($typespecFailedSdk.Length -gt 0 -and -not $IgnoreVerifyTypeSpecCodeGenerationError)) ? 1 : 0 } Write-Host " @@ -109,22 +132,30 @@ Verify no diff " -# prevent warning related to EOL differences which triggers an exception for some reason -git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code -- "*.java" ":(exclude)**/src/test/**" ":(exclude)**/src/samples/**" ":(exclude)**/src/main/**/implementation/**" +if ($swaggerDiff.Length -gt 0 -or $typespecDiff.Length -gt 0) { + if ($swaggerDiff.Length -gt 0) { + Write-Host "Swagger code generation failed. The following files are out of date:" + Write-Host $swaggerDiff + } + + if ($typespecDiff.Length -gt 0) { + Write-Host "TypeSpec code generation failed. The following files are out of date:" + Write-Host $typespecDiff + } -if ($LastExitCode -ne 0) { - $status = git status -s | Out-String - Write-Host " + $status = git status -s | Out-String + Write-Host " The following files are out of date: $status " - Reset-Repository - exit 1 + Reset-Repository + # Only exit with an error if Swagger had a diff or TypeSpec had a diff and IgnoreVerifyTypeSpecCodeGenerationError is false + exit ($swaggerDiff.Length -gt 0 -or ($typespectypespecDiffFailedSdk.Length -gt 0 -and -not $IgnoreVerifyTypeSpecCodeGenerationError)) ? 1 : 0 } # Delete out TypeSpec temporary folders if they still exist. Get-ChildItem -Path $path -Filter TempTypeSpecFiles -Recurse -Directory | ForEach-Object { - Remove-Item -Path $_.FullName -Recurse -Force + Remove-Item -Path $_.FullName -Recurse -Force } Reset-Repository From 3449e37f909b5b437047a6f8f844a3732ed4c06b Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:57:15 -0500 Subject: [PATCH 4/5] Change how ignore TypeSpec failure is passed --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index e9940ba1f6ddb..bb1facf730f73 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -355,7 +355,7 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Compare-CurrentToCodegeneration.ps1 arguments: > -Directory sdk/${{ parameters.ServiceDirectory }} - -IgnoreVerifyTypeSpecCodeGenerationError ${{ parameters.IgnoreVerifyTypeSpecCodeGenerationError }} + -IgnoreVerifyTypeSpecCodeGenerationError ${{ eq(parameters.IgnoreVerifyTypeSpecCodeGenerationError, 'true') }} - template: /eng/pipelines/templates/steps/run-and-validate-linting.yml parameters: From babb3477e3705e325ccd7d6fe8a41f9ae457105c Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:31:43 -0500 Subject: [PATCH 5/5] Fix boolean passing --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index bb1facf730f73..5f99b385ec9bf 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -355,7 +355,7 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/scripts/Compare-CurrentToCodegeneration.ps1 arguments: > -Directory sdk/${{ parameters.ServiceDirectory }} - -IgnoreVerifyTypeSpecCodeGenerationError ${{ eq(parameters.IgnoreVerifyTypeSpecCodeGenerationError, 'true') }} + -IgnoreVerifyTypeSpecCodeGenerationError $${{ parameters.IgnoreVerifyTypeSpecCodeGenerationError }} - template: /eng/pipelines/templates/steps/run-and-validate-linting.yml parameters: