From 91bf777d3bc00861eed957f745eb5754df194423 Mon Sep 17 00:00:00 2001 From: yf711 Date: Thu, 5 Dec 2024 10:09:47 -0800 Subject: [PATCH] fix --- ...linux-gpu-tensorrt-daily-perf-pipeline.yml | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml index e19cdaaa5035f..21d168b50e6ae 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml @@ -17,7 +17,7 @@ parameters: - BIN - name: UseTensorrtOssParser - displayName: Use TensorRT-OSS Parser (not compatible with BIN + displayName: Use TensorRT-OSS Parser (not compatible with BIN) type: boolean default: false @@ -58,92 +58,92 @@ jobs: variables: - name: branchName - ${{ if not(eq(variables['ortBranchOverride'], '' }}: - value: $(ortBranchOverride + ${{ if not(eq(variables['ortBranchOverride'], '')) }}: + value: $(ortBranchOverride) ${{ else }}: - value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '' ] + value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] - name: trtVersion - ${{ if eq(parameters.TrtVersion, 'BIN' }}: - value: $(tarTrtVersion + ${{ if eq(parameters.TrtVersion, 'BIN') }}: + value: $(tarTrtVersion) ${{ else }}: value: ${{ parameters.TrtVersion }} - name: trtEPOptionsArg - ${{ if not(eq(length(parameters.TrtEPOptions, 0 }}: - value: --trt_ep_options ${{ join(',',parameters.TrtEPOptions }} + ${{ if not(eq(length(parameters.TrtEPOptions), 0)) }}: + value: --trt_ep_options ${{ join(',',parameters.TrtEPOptions) }} - name: cudaEPOptionsArg - ${{ if not(eq(length(parameters.CUDAEPOptions, 0 }}: - value: --cuda_ep_options ${{ join(',',parameters.CUDAEPOptions }} + ${{ if not(eq(length(parameters.CUDAEPOptions), 0)) }}: + value: --cuda_ep_options ${{ join(',',parameters.CUDAEPOptions) }} - name: optional_arguments - value: -a "-a -z -g $(optimizeGraph -b $(bindInputs $(trtEPOptionsArg $(cudaEPOptionsArg" + value: -a "-a -z -g $(optimizeGraph) -b $(bindInputs) $(trtEPOptionsArg) $(cudaEPOptionsArg)" - name: image - value: ort-image-$(Build.BuildId + value: ort-image-$(Build.BuildId) - name: parser - ${{ if eq(parameters.UseTensorrtOssParser, true }}: - value: --use_tensorrt_oss_parser $(parameters.UseTensorrtOssParser }} + ${{ if eq(parameters.UseTensorrtOssParser, true) }}: + value: --use_tensorrt_oss_parser $(parameters.UseTensorrtOssParser) }} steps: - - ${{ if and(eq(parameters.TrtVersion, 'BIN', eq(parameters.UseTensorrtOssParser, false }}: - - script: 'ls -al $(trtBinsDir' + - ${{ if and(eq(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}: + - script: 'ls -al $(trtBinsDir)' displayName: 'Show available TensorRT .tar.gz packages' - - script: 'cp $(trtBinsDir/TensorRT-$(trtVersion.Linux.x86_64-gnu.cuda-$(tarCudaVersion.tar.gz $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build/' + - script: 'cp $(trtBinsDir)/TensorRT-$(trtVersion).Linux.x86_64-gnu.cuda-$(tarCudaVersion).tar.gz $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/' displayName: 'Copy TensorRT .tar.gz package into Docker build directory' - - script: 'python3 $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory -i $(image -b $(branchName -t $(trtVersion -a 75 --install_bin --tar_cuda_version=$(tarCudaVersion --trt_bins_dir=.' - displayName: 'Install TensorRT $(tarTrtVersion from binaries and build latest ORT Image' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build' + - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --install_bin --tar_cuda_version=$(tarCudaVersion) --trt_bins_dir=.' + displayName: 'Install TensorRT $(tarTrtVersion) from binaries and build latest ORT Image' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' # Build ORT with TensorRT built-in parser - - ${{ if and(ne(parameters.TrtVersion, 'BIN', eq(parameters.UseTensorrtOssParser, false }}: - - script: 'python3 $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory -i $(image -b $(branchName -t $(trtVersion -a 75' + - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}: + - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75' displayName: 'Build latest ORT Image with TensorRT built-in parser' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' # Build ORT with certain version of TensorRT OSS parser - - ${{ if and(ne(parameters.TrtVersion, 'BIN', eq(parameters.UseTensorrtOssParser, true, ne(variables.onnxTensorrtCommitId, '' }}: - - script: 'python3 $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory -i $(image -b $(branchName -t $(trtVersion -a 75 -o $(onnxTensorrtCommitId --use_tensorrt_oss_parser' - displayName: 'Build latest ORT Image with TensorRT OSS parser and commit id $(onnxTensorrtCommitId' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build' + - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true), ne(variables.onnxTensorrtCommitId, '')) }}: + - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 -o $(onnxTensorrtCommitId) --use_tensorrt_oss_parser' + displayName: 'Build latest ORT Image with TensorRT OSS parser and commit id $(onnxTensorrtCommitId)' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' # Build ORT with default version of TensorRT OSS parser - - ${{ if and(ne(parameters.TrtVersion, 'BIN', eq(parameters.UseTensorrtOssParser, true, eq(variables.onnxTensorrtCommitId, '' }}: - - script: 'python3 $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory -i $(image -b $(branchName -t $(trtVersion -a 75 --use_tensorrt_oss_parser' + - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true), eq(variables.onnxTensorrtCommitId, '')) }}: + - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --use_tensorrt_oss_parser' displayName: 'Build latest ORT Image with TensorRT OSS parser' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/build' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' - - ${{ if eq(parameters.MemTest, true }}: - - script: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/mem_test/run_mem_test_docker.sh -d $(image -p $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/mem_test/ -w /code/ -l false -c ${{ parameters.ConcurrencyTest }}' + - ${{ if eq(parameters.MemTest, true) }}: + - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/run_mem_test_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/ -w /code/ -l false -c ${{ parameters.ConcurrencyTest }}' displayName: 'Run Memory Test and Concurrency Test' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/mem_test/' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/' - ${{ each option in parameters.ModelGroups }}: - - script: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/run_perf_docker.sh -d $(image -p $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf -v $(modelVolume -b true -o ${{option}} -m $(${{option}} -e "$(epList" $(optional_arguments' + - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/run_perf_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf -v $(modelVolume) -b true -o ${{option}} -m $(${{option}}) -e "$(epList)" $(optional_arguments)' displayName: '${{option}} perf' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/' # Prepare and Publish Artifacts - - script: 'mkdir $(Build.SourcesDirectory/Artifact' + - script: 'mkdir $(Build.SourcesDirectory)/Artifact' displayName: 'Prepare Artifacts Directory' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/' - condition: always( + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/' + condition: always() - - ${{ if not(eq(length(parameters.ModelGroups, 0 }}: - - script: 'cp -r $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/result/ $(Build.SourcesDirectory/Artifact' + - ${{ if not(eq(length(parameters.ModelGroups), 0)) }}: + - script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/result/ $(Build.SourcesDirectory)/Artifact' displayName: 'Copy Artifacts' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/' - - ${{ if eq(parameters.MemTest, true }}: - - script: 'cp -r $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/mem_test/build/result $(Build.SourcesDirectory/Artifact/result_mem_test' + - ${{ if eq(parameters.MemTest, true) }}: + - script: 'cp -r $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/build/result $(Build.SourcesDirectory)/Artifact/result_mem_test' displayName: 'Copy Artifacts' - workingDirectory: '$(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/' - condition: always( + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/' + condition: always() - script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs' displayName: 'Install dashboard dependencies' @@ -162,11 +162,11 @@ jobs: scriptLocation: inlineScript scriptType: bash inlineScript: | - short_hash=$(git rev-parse --short HEAD && - commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd && - python3 $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/parse_mem_concurrency_test.py -r $(Build.SourcesDirectory/Artifact/result_mem_test -c $short_hash -d $commit_date -u "$(reportUrl?buildId=$(Build.BuildId" -t $(trtVersion -b $(branchName --kusto_conn $(kustoConn --database $(database $(parser + short_hash=$(git rev-parse --short HEAD) && + commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) && + python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/parse_mem_concurrency_test.py -r $(Build.SourcesDirectory)/Artifact/result_mem_test -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t $(trtVersion) -b $(branchName) --kusto_conn $(kustoConn) --database $(database) $(parser) - - ${{ if eq(parameters.PostToDashboard, true }}: + - ${{ if eq(parameters.PostToDashboard, true) }}: - script: 'python3 -m pip install pandas azure-kusto-data[pandas] azure-kusto-ingest[pandas] coloredlogs' displayName: 'Install dashboard dependencies' @@ -185,17 +185,17 @@ jobs: scriptLocation: inlineScript scriptType: bash inlineScript: | - short_hash=$(git rev-parse --short HEAD && - commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd && - python3 $(Build.SourcesDirectory/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory/Artifact/result -c $short_hash -d $commit_date -u "$(reportUrl?buildId=$(Build.BuildId" -t $(trtVersion -b $(branchName --kusto_conn $(kustoConn --database $(database $(parser + short_hash=$(git rev-parse --short HEAD) && + commit_date=$(git log -1 --date=iso-strict --pretty=format:%cd) && + python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/post.py -r $(Build.SourcesDirectory)/Artifact/result -c $short_hash -d $commit_date -u "$(reportUrl)?buildId=$(Build.BuildId)" -t $(trtVersion) -b $(branchName) --kusto_conn $(kustoConn) --database $(database) $(parser) - task: PublishBuildArtifacts@1 inputs: - pathtoPublish: '$(Build.SourcesDirectory/Artifact' - artifactName: 'result-$(Build.BuildNumber' + pathtoPublish: '$(Build.SourcesDirectory)/Artifact' + artifactName: 'result-$(Build.BuildNumber)' - template: templates/component-governance-component-detection-steps.yml parameters : condition : 'succeeded' - - template: templates/clean-agent-build-directory-step.yml + - template: templates/clean-agent-build-directory-step.yml \ No newline at end of file