From 5c7e6b2e2af922e3dc7e58c584e454074a394d3d Mon Sep 17 00:00:00 2001 From: Yifan Li <109183385+yf711@users.noreply.github.com> Date: Mon, 12 Feb 2024 23:04:08 -0800 Subject: [PATCH] [EP Perf] Add CI option to enable TRT-OSS parser (#19448) ### Description * Introducing CI option to enable TRT-OSS parser, during ep perf testing: ![image](https://github.com/microsoft/onnxruntime/assets/109183385/a9ba6393-6b94-4b8f-8ca4-ba7bc7954504) By default, open-sourced onnx-tensorrt parser listed under [cmake/deps.txt](https://github.com/microsoft/onnxruntime/blob/main/cmake/deps.txt#L39-L40) will be used if enabling this option. ### To verify this option and check the difference during ORT image build: If this option is enabled: image If this option is not enabled (by default): image * update default usage of cmake/trt version to the latest ### Motivation and Context Make it easier to test oss parser and find potential gap between tensorrt builtin/oss parser. Schedule runs with oss parser will be set after this PR gets merged --- .../tools/tensorrt/perf/build/build_image.py | 18 ++++++- .../tensorrt/perf/build/ort_build_latest.py | 48 +++++++++++-------- ...linux-gpu-tensorrt-daily-perf-pipeline.yml | 21 ++++++-- .../Dockerfile.ubuntu_cuda11_8_tensorrt8_6 | 5 +- 4 files changed, 63 insertions(+), 29 deletions(-) diff --git a/onnxruntime/python/tools/tensorrt/perf/build/build_image.py b/onnxruntime/python/tools/tensorrt/perf/build/build_image.py index b98aafc27579a..2ae64a72d08fe 100644 --- a/onnxruntime/python/tools/tensorrt/perf/build/build_image.py +++ b/onnxruntime/python/tools/tensorrt/perf/build/build_image.py @@ -45,7 +45,7 @@ def get_common_docker_build_args(args: argparse.Namespace) -> List[str]: :return: A list of common 'docker build' arguments. """ - return [ + command = [ "--no-cache", "-t", f"{args.image_name}", @@ -54,6 +54,14 @@ def get_common_docker_build_args(args: argparse.Namespace) -> List[str]: "--build-arg", f"ONNXRUNTIME_BRANCH={args.branch}", ] + if args.use_tensorrt_oss_parser: + command.extend( + [ + "--build-arg", + "PARSER_CONFIG=--use_tensorrt_oss_parser", + ] + ) + return command def is_valid_ver_str(version: str, min_comps: int = 0, max_comps: int = 0) -> bool: @@ -187,7 +195,7 @@ def parse_arguments() -> argparse.Namespace: parser.add_argument("-r", "--repo_path", required=True, help="Path to the onnxruntime repository") parser.add_argument("-i", "--image_name", required=True, help="The resulting Docker image name") parser.add_argument("-b", "--branch", default="main", help="Name of the onnxruntime git branch to checkout") - parser.add_argument("-t", "--trt_version", default="8.4.1.5", help="TensorRT version (e.g., 8.4.1.5)") + parser.add_argument("-t", "--trt_version", default="8.6.1.6", help="TensorRT version (e.g., 8.6.1.6)") parser.add_argument("-a", "--cuda_arch", default="75", help="CUDA architecture (e.g., 75)") # Command-line options for installing TensorRT from binaries. @@ -208,6 +216,12 @@ def parse_arguments() -> argparse.Namespace: help="CUDA version (e.g., 8.6) used to find TensorRT EA binary tar.gz package", ) parser.add_argument("--trt_bins_dir", default="", help="Directory containing TensorRT tar.gz package") + parser.add_argument( + "--use_tensorrt_oss_parser", + action="store_true", + default=False, + help="Use TensorRT OSS Parser", + ) return parser.parse_args() diff --git a/onnxruntime/python/tools/tensorrt/perf/build/ort_build_latest.py b/onnxruntime/python/tools/tensorrt/perf/build/ort_build_latest.py index 6e20071683d90..c7d4a7836132a 100755 --- a/onnxruntime/python/tools/tensorrt/perf/build/ort_build_latest.py +++ b/onnxruntime/python/tools/tensorrt/perf/build/ort_build_latest.py @@ -13,6 +13,12 @@ def parse_arguments(): parser.add_argument("-b", "--branch", required=False, default="master", help="Github branch to test perf off of") parser.add_argument("-s", "--save", required=False, help="Directory to archive wheel file") parser.add_argument("-a", "--use_archived", required=False, help="Archived wheel file") + parser.add_argument( + "--use_tensorrt_oss_parser", + action="store_true", + default=False, + help="Use TensorRT OSS Parser", + ) args = parser.parse_args() return args @@ -35,14 +41,14 @@ def install_new_ort_wheel(ort_master_path): def main(): args = parse_arguments() - cmake_tar = "cmake-3.18.4-Linux-x86_64.tar.gz" + cmake_tar = "cmake-3.28.3-linux-x86_64.tar.gz" if not os.path.exists(cmake_tar): - subprocess.run(["wget", "-c", "https://cmake.org/files/v3.18/" + cmake_tar], check=True) + subprocess.run(["wget", "-c", "https://cmake.org/files/v3.28/" + cmake_tar], check=True) tar = tarfile.open(cmake_tar) tar.extractall() tar.close() - os.environ["PATH"] = os.path.join(os.path.abspath("cmake-3.18.4-Linux-x86_64"), "bin") + ":" + os.environ["PATH"] + os.environ["PATH"] = os.path.join(os.path.abspath("cmake-3.28.3-linux-x86_64"), "bin") + ":" + os.environ["PATH"] os.environ["CUDACXX"] = os.path.join(args.cuda_home, "bin", "nvcc") ort_master_path = args.ort_master_path @@ -57,24 +63,24 @@ def main(): subprocess.run(["git", "fetch"], check=True) subprocess.run(["git", "checkout", args.branch], check=True) subprocess.run(["git", "pull", "origin", args.branch], check=True) - subprocess.run( - [ - "./build.sh", - "--config", - "Release", - "--use_tensorrt", - "--tensorrt_home", - args.tensorrt_home, - "--cuda_home", - args.cuda_home, - "--cudnn", - "/usr/lib/x86_64-linux-gnu", - "--build_wheel", - "--skip_tests", - "--parallel", - ], - check=True, - ) + command = [ + "./build.sh", + "--config", + "Release", + "--use_tensorrt", + "--tensorrt_home", + args.tensorrt_home, + "--cuda_home", + args.cuda_home, + "--cudnn", + "/usr/lib/x86_64-linux-gnu", + "--build_wheel", + "--skip_tests", + "--parallel", + ] + if args.use_tensorrt_oss_parser: + command.append("--use_tensorrt_oss_parser") + subprocess.run(command, check=True) ort_wheel_file = install_new_ort_wheel(ort_master_path) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml index e75bb68a8bfeb..eaadc6ad728c0 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml @@ -15,6 +15,11 @@ parameters: - 8.6.1.6 - BIN +- name: UseTensorrtOssParser + displayName: Use TensorRT-OSS Parser + type: boolean + default: false + - name: ModelGroups type: object default: @@ -73,7 +78,7 @@ jobs: value: ort-image-$(Build.BuildId) steps: - - ${{ if eq(parameters.TrtVersion, 'BIN') }}: + - ${{ if and(eq(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}: - script: 'ls -al $(trtBinsDir)' displayName: 'Show available TensorRT .tar.gz packages' @@ -83,11 +88,19 @@ jobs: - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --install_bin --tar_cuda_version=$(tarCudaVersion) --tar_cudnn_version=$(tarCudnnVersion) --trt_bins_dir=.' displayName: 'Install TensorRT from binaries and build latest ORT Image' workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' - - ${{ else }}: + + # Build ORT with TensorRT built-in parser + - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, false)) }}: - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75' - displayName: 'Build latest ORT Image' + displayName: 'Build latest ORT Image with TensorRT built-in parser' workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' - + + # Build ORT with TensorRT OSS parser + - ${{ if and(ne(parameters.TrtVersion, 'BIN'), eq(parameters.UseTensorrtOssParser, true)) }}: + - script: 'python3 $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build/build_image.py -r $(Build.SourcesDirectory) -i $(image) -b $(branchName) -t $(trtVersion) -a 75 --use_tensorrt_oss_parser' + displayName: 'Build latest ORT Image with TensorRT OSS parser' + workingDirectory: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/build' + - ${{ if eq(parameters.MemTest, true) }}: - script: '$(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/run_mem_test_docker.sh -d $(image) -p $(Build.SourcesDirectory)/onnxruntime/python/tools/tensorrt/perf/mem_test/ -w /code/ -l false' displayName: 'Run Memory Test' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6 b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6 index 04a6af962b5e6..f1ffba3b3e1c9 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6 +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6 @@ -82,8 +82,9 @@ RUN if [ -z "$ONNXRUNTIME_COMMIT_ID" ] ; then echo "Building branch ${ONNXRUNTIM git reset --hard ${ONNXRUNTIME_COMMIT_ID} && git submodule update --recursive ; fi # Build ORT -ENV CUDA_MODULE_LOADING "LAZY" -RUN /bin/sh build.sh --parallel --build_shared_lib --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' +ENV CUDA_MODULE_LOADING "LAZY" +ARG PARSER_CONFIG="" +RUN /bin/sh build.sh ${PARSER_CONFIG} --parallel --build_shared_lib --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' # Switch to root to continue following steps of CI USER root