From 82eb84c84e9c04eb75a45583b023ea7604540b67 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 24 Jan 2024 16:51:02 -0800 Subject: [PATCH] update --- .pipelines/windowsai-steps.yml | 2 +- tools/ci_build/build.py | 4 ++-- .../github/azure-pipelines/linux-ci-pipeline.yml | 6 +++--- .../azure-pipelines/linux-cpu-aten-pipeline.yml | 4 ++-- .../azure-pipelines/linux-cpu-eager-pipeline.yml | 2 +- .../linux-cpu-minimal-build-ci-pipeline.yml | 12 ++++++------ .../github/azure-pipelines/linux-gpu-ci-pipeline.yml | 4 ++-- .../linux-gpu-tensorrt-ci-pipeline.yml | 2 +- .../github/azure-pipelines/linux-qnn-ci-pipeline.yml | 4 ++-- .../azure-pipelines/mac-coreml-ci-pipeline.yml | 2 +- .../github/azure-pipelines/mac-ios-ci-pipeline.yml | 2 +- .../mac-objc-static-analysis-ci-pipeline.yml | 2 +- .../azure-pipelines/nuget/templates/dml-vs-2022.yml | 4 ++-- .../orttraining-linux-ci-pipeline.yml | 2 +- .../orttraining-py-packaging-pipeline-cpu.yml | 2 +- .../qnn-ep-nuget-packaging-pipeline.yml | 2 +- .../azure-pipelines/templates/c-api-linux-cpu.yml | 2 +- .../templates/jobs/win-ci-vs-2022-job.yml | 2 +- .../templates/mac-cpu-packaging-steps.yml | 4 ++-- .../azure-pipelines/templates/py-packaging-stage.yml | 4 ++-- .../templates/py-packaging-training-cuda-stage.yml | 2 +- .../github/azure-pipelines/templates/py-win-gpu.yml | 2 +- .../github/azure-pipelines/templates/win-ci.yml | 6 +++--- .../github/azure-pipelines/win-ci-pipeline.yml | 2 +- .../azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml | 4 ++-- .../github/azure-pipelines/win-qnn-ci-pipeline.yml | 2 +- .../github/linux/build_cuda_c_api_package.sh | 2 +- .../github/linux/build_linux_python_package.sh | 2 +- .../github/linux/build_tensorrt_c_api_package.sh | 2 +- .../build_full_ort_and_create_ort_files.sh | 2 +- .../ort_minimal/build_minimal_ort_and_run_tests.sh | 2 +- tools/ci_build/github/linux/run_build.sh | 2 +- tools/scripts/python_test.sh | 2 +- 33 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.pipelines/windowsai-steps.yml b/.pipelines/windowsai-steps.yml index 893f12ed2fe02..acae78dd712e6 100644 --- a/.pipelines/windowsai-steps.yml +++ b/.pipelines/windowsai-steps.yml @@ -84,7 +84,7 @@ jobs: 7z x cmake-3.26.3-windows-x86_64.zip set PYTHONHOME=$(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools set PYTHONPATH=$(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools - $(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools\python.exe "$(Build.SourcesDirectory)\tools\ci_build\build.py" --build_dir $(Build.BinariesDirectory) --parallel --use_preset_compile_flags --build_shared_lib --enable_onnx_tests --ms_experimental --use_dml --use_winml --cmake_generator "Visual Studio 17 2022" --update --config RelWithDebInfo --enable_qspectre --enable_lto --use_telemetry --disable_rtti --enable_wcos $(BuildFlags) --cmake_extra_defines "CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" "CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" CMAKE_SYSTEM_VERSION=10.0.19041.0 --cmake_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\ctest.exe + $(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools\python.exe "$(Build.SourcesDirectory)\tools\ci_build\build.py" --build_dir $(Build.BinariesDirectory) --parallel --use_binskim_compliant_compile_flags --build_shared_lib --enable_onnx_tests --ms_experimental --use_dml --use_winml --cmake_generator "Visual Studio 17 2022" --update --config RelWithDebInfo --enable_qspectre --enable_lto --use_telemetry --disable_rtti --enable_wcos $(BuildFlags) --cmake_extra_defines "CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" "CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" CMAKE_SYSTEM_VERSION=10.0.19041.0 --cmake_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\ctest.exe workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Generate cmake config' diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 7f69d345625b6..f47dfc5dddd08 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -443,7 +443,7 @@ def convert_arg_line_to_args(self, arg_line): "--enable_address_sanitizer", action="store_true", help="Enable address sanitizer. Windows/Linux/MacOS only." ) # The following flag is mostly designed to be used in ONNX Runtime's Azure DevOps/Github build pipelines. Its main purpose is to make the built binaries pass BinSkim scan. - parser.add_argument("--use_preset_compile_flags", action="store_true", help="Use preset compile flags.") + parser.add_argument("--use_binskim_compliant_compile_flags", action="store_true", help="Use preset compile flags.") parser.add_argument( "--disable_memleak_checker", action="store_true", @@ -1501,7 +1501,7 @@ def generate_build_tree( # Setup default values for cflags/cxxflags/ldflags. # The values set here are purely for security and compliance purposes. ONNX Runtime should work fine without these flags. if ( - (args.use_preset_compile_flags or args.enable_address_sanitizer) + (args.use_binskim_compliant_compile_flags or args.enable_address_sanitizer) and not args.ios and not args.android and not args.build_wasm diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 93e4ada0fc406..a4bd24b4dd18b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -93,7 +93,7 @@ stages: --config Debug \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_csharp \ --enable_onnx_tests --enable_address_sanitizer \ --update --build; @@ -102,7 +102,7 @@ stages: --config Debug \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_csharp \ --enable_onnx_tests --enable_address_sanitizer \ --test;" @@ -228,7 +228,7 @@ stages: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --build_csharp \ --enable_onnx_tests \ diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml index e0c85fd86ead4..31decb0c2ffcc 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml @@ -94,7 +94,7 @@ jobs: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --skip_tests \ --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \ @@ -126,7 +126,7 @@ jobs: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --test \ --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON" diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml index 3e821b762757d..b3f5ff9631412 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml @@ -80,7 +80,7 @@ jobs: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --enable_lazy_tensor --enable_training --build_wheel --skip_test \ workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index a9ef5a81226a3..1053a2518125f 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -141,7 +141,7 @@ jobs: --config Debug \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --skip_tests \ --minimal_build \ --disable_exceptions \ @@ -222,7 +222,7 @@ jobs: --build_dir /build/5 --cmake_generator Ninja \ --config Debug \ --skip_submodule_sync \ - --build_shared_lib --use_preset_compile_flags \ + --build_shared_lib --use_binskim_compliant_compile_flags \ --parallel \ --minimal_build extended workingDirectory: $(Build.SourcesDirectory) @@ -246,7 +246,7 @@ jobs: --skip_submodule_sync \ --build_shared_lib \ --build_wheel \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --skip_tests \ --disable_ml_ops \ --disable_types sparsetensor float8 optional \ @@ -272,7 +272,7 @@ jobs: --config MinSizeRel \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --minimal_build \ --disable_exceptions \ --disable_ml_ops \ @@ -300,7 +300,7 @@ jobs: --cmake_generator Ninja \ --config MinSizeRel \ --skip_submodule_sync \ - --build_shared_lib --use_preset_compile_flags \ + --build_shared_lib --use_binskim_compliant_compile_flags \ --parallel \ --minimal_build extended \ --disable_exceptions \ @@ -330,7 +330,7 @@ jobs: --cmake_generator Ninja \ --config MinSizeRel \ --skip_submodule_sync \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --android \ --android_sdk_path /android_home \ --android_ndk_path /ndk_home \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 559960c1a5c62..b19a8b11db265 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -131,7 +131,7 @@ jobs: --config Release --update --build \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda-${{parameters.CudaVersion}} --cudnn_home=/usr/local/cuda-${{parameters.CudaVersion}} \ --enable_cuda_profiling --enable_cuda_nhwc_ops \ @@ -215,7 +215,7 @@ jobs: cd /onnxruntime_src/java && /onnxruntime_src/java/gradlew cmakeCheck -DcmakeBuildDir=/build/Release -DUSE_CUDA=1; \ cd /tmp; \ /tmp/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_preset_compile_flags --build_wheel --enable_onnx_tests \ + --build_dir /build --config Release --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests \ --use_cuda --cuda_version=${{parameters.CudaVersion}} --cuda_home=/usr/local/cuda --cudnn_home=/usr/local/cuda \ --enable_pybind --build_java --ctest_path '' " diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index a49aba9c2baed..75e4ba54006d8 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -114,7 +114,7 @@ jobs: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --enable_onnx_tests \ --use_cuda --cuda_home=/usr/local/cuda-${{ parameters.CudaVersion }} --cudnn_home=/usr/local/cuda-${{ parameters.CudaVersion }} \ diff --git a/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml index 526f9d2dfc6ac..0312b70d2b1d5 100644 --- a/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-qnn-ci-pipeline.yml @@ -63,7 +63,7 @@ jobs: python3 tools/ci_build/build.py \ --build_dir build \ --config Release \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --use_qnn \ --qnn_home $(QNN_SDK_ROOT) \ --cmake_generator=Ninja \ @@ -73,7 +73,7 @@ jobs: - script: | python3 tools/ci_build/build.py \ --build_dir build \ - --config Release --use_preset_compile_flags \ + --config Release --use_binskim_compliant_compile_flags \ --test \ --qnn_home $(QNN_SDK_ROOT) \ --cmake_generator=Ninja \ diff --git a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml index 90b17a403714f..a3f56f5c448a9 100644 --- a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml @@ -57,7 +57,7 @@ jobs: --build_dir build \ --skip_submodule_sync \ --cmake_generator=Ninja \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_shared_lib \ --config Debug \ --use_cache \ diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml index 9c69a1f70f33e..a1ca68c8279e7 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml @@ -62,7 +62,7 @@ jobs: --use_xcode \ --config RelWithDebInfo \ --build_apple_framework \ - --parallel --use_preset_compile_flags + --parallel --use_binskim_compliant_compile_flags displayName: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator env: CC: clang diff --git a/tools/ci_build/github/azure-pipelines/mac-objc-static-analysis-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-objc-static-analysis-ci-pipeline.yml index d4f12c4ba360f..7e8e72cad179f 100644 --- a/tools/ci_build/github/azure-pipelines/mac-objc-static-analysis-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-objc-static-analysis-ci-pipeline.yml @@ -26,7 +26,7 @@ jobs: --enable_training_apis \ --cmake_extra_defines CMAKE_EXPORT_COMPILE_COMMANDS=ON \ --update --skip_submodule_sync \ - --build --parallel --use_preset_compile_flags --target onnx_proto + --build --parallel --use_binskim_compliant_compile_flags --target onnx_proto displayName: Generate compile_commands.json and ONNX protobuf files - script: | diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml index eb0e952547820..9393fb07d718a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml @@ -103,7 +103,7 @@ stages: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '$(BuildCommand) --use_preset_compile_flags --parallel --path_to_protoc_exe $(Build.BinariesDirectory)\installed\bin\protoc.exe --build_csharp --update --config $(BuildConfig) ${{ variables.build_py_lto_flag }}' + arguments: '$(BuildCommand) --use_binskim_compliant_compile_flags --parallel --path_to_protoc_exe $(Build.BinariesDirectory)\installed\bin\protoc.exe --build_csharp --update --config $(BuildConfig) ${{ variables.build_py_lto_flag }}' workingDirectory: '$(Build.BinariesDirectory)' - ${{ if notIn(parameters['sln_platform'], 'Win32', 'x64') }}: @@ -176,7 +176,7 @@ stages: python.exe -m pip install -q --upgrade %WHEEL_FILENAME% set PATH=%PATH%;$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig) @echo %PATH% - python $(Build.SourcesDirectory)\tools\ci_build\build.py $(BuildCommand) --parallel --use_preset_compile_flags --test --config $(BuildConfig) ${{ variables.build_py_lto_flag }} + python $(Build.SourcesDirectory)\tools\ci_build\build.py $(BuildCommand) --parallel --use_binskim_compliant_compile_flags --test --config $(BuildConfig) ${{ variables.build_py_lto_flag }} workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' displayName: 'Run tests' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index 351114426b8a3..d8f02054a3216 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -102,7 +102,7 @@ jobs: --config Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --enable_onnx_tests \ --enable_training \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml index 61b50ffbe79d7..bf1ba71b7b818 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml @@ -69,7 +69,7 @@ stages: --config Debug Release \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --enable_onnx_tests \ --enable_pybind --enable_training diff --git a/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml index bb417fe2356aa..b0509467e1689 100644 --- a/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml @@ -37,7 +37,7 @@ jobs: buildArch: x64 setVcvars: true ALLOW_RELEASED_ONNX_OPSET_ONLY: '1' - commonBuildArgs: '--compile_no_warning_as_error --build_dir $(Build.BinariesDirectory)\Windows --skip_submodule_sync --build_shared_lib --cmake_generator "Visual Studio 17 2022" --config ${{ parameters.build_config }} --use_qnn --qnn_home C:\data\qnnsdk\${{parameters.QnnSdk}} --parallel --use_preset_compile_flags ' + commonBuildArgs: '--compile_no_warning_as_error --build_dir $(Build.BinariesDirectory)\Windows --skip_submodule_sync --build_shared_lib --cmake_generator "Visual Studio 17 2022" --config ${{ parameters.build_config }} --use_qnn --qnn_home C:\data\qnnsdk\${{parameters.QnnSdk}} --parallel --use_binskim_compliant_compile_flags ' steps: - template: templates/set-version-number-variables-step.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index 28d55996484bc..2da3b8a9bc7b8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -64,7 +64,7 @@ jobs: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}} /bin/bash -c "python3.9 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ - --skip_submodule_sync --parallel --use_preset_compile_flags --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-${{parameters.OnnxruntimeArch}}" + --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-${{parameters.OnnxruntimeArch}}" workingDirectory: $(Build.SourcesDirectory) displayName: 'Build' diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml index 961b198b661e9..dd703f3199d9b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml @@ -138,7 +138,7 @@ jobs: Today: $(TODAY) CacheDir: $(ORT_CACHE_DIR) AdditionalKey: " $(System.StageName) | ${{ parameters.BuildConfig }} " - BuildPyArguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --use_preset_compile_flags --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}' + BuildPyArguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --use_binskim_compliant_compile_flags --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}' MsbuildArguments: '-maxcpucount' BuildArch: ${{ parameters.buildArch }} Platform: ${{ parameters.msbuildPlatform }} diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml index e8d2b8a8075e9..7672b604a5268 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml @@ -47,14 +47,14 @@ steps: BuildStep: - script: | rm -rf $(Build.BinariesDirectory)/Release - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_preset_compile_flags --build_shared_lib --config Release + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --config Release displayName: 'Build ${{ parameters.MacosArch }}' env: CCACHE_DIR: ${{ parameters.CacheDir }} - ${{ if eq(parameters.MacosArch, 'x86_64') }}: - script: | - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --test ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_preset_compile_flags --build_shared_lib --config Release + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --test ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --config Release displayName: 'Running Tests' - task: ShellScript@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 87dabf1201b09..146e3e58444c1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -182,7 +182,7 @@ stages: --enable_pybind --enable_onnx_tests ${{ parameters.build_py_parameters }} - --parallel --use_preset_compile_flags --update + --parallel --use_binskim_compliant_compile_flags --update $(TelemetryOption) workingDirectory: '$(Build.BinariesDirectory)' @@ -388,7 +388,7 @@ stages: set -e -x export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-universal2 python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --use_preset_compile_flags --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir $(Build.BinariesDirectory) --use_coreml --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --config Release --build_wheel ${{ parameters.build_py_parameters }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" --update --build displayName: 'Command Line Script' - script: | diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml index 0b600eb5ac91c..c6921e151a029 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml @@ -171,7 +171,7 @@ stages: --build_dir /build \ --config ${{ variables['buildConfig'] }} \ --skip_submodule_sync \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --enable_onnx_tests \ ${{ parameters.build_py_parameters }} \ diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index 6846979e73cb7..18368e59cad52 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -116,7 +116,7 @@ jobs: --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests - --parallel --use_preset_compile_flags --update + --parallel --use_binskim_compliant_compile_flags --update $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 992fa5ca9fbb0..8ed22153fd947 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -150,9 +150,9 @@ stages: inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: - arguments: '--config RelWithDebInfo --use_preset_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --test_all_timeout 72000' + arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --test_all_timeout 72000' ${{ else }}: - arguments: '--config RelWithDebInfo --use_preset_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ' + arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -172,7 +172,7 @@ stages: condition: and(succeeded(), eq('${{ parameters.runTests}}', true)) inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --use_preset_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' workingDirectory: '$(Build.BinariesDirectory)' - script: | diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 27e86cd917596..53eea1d69fb0e 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -74,7 +74,7 @@ stages: displayName: 'Build and Test' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_preset_compile_flags --cmake_generator "Visual Studio 17 2022" --disable_memleak_checker --enable_address_sanitizer + arguments: --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --cmake_generator "Visual Studio 17 2022" --disable_memleak_checker --enable_address_sanitizer workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 802438d382d9e..6cbe20bb93463 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -54,7 +54,7 @@ jobs: WithCache: True Today: $(TODAY) AdditionalKey: "gpu-tensorrt | RelWithDebInfo" - BuildPyArguments: '--config RelWithDebInfo --parallel --use_preset_compile_flags --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75' + BuildPyArguments: '--config RelWithDebInfo --parallel --use_binskim_compliant_compile_flags --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75' MsbuildArguments: $(MsbuildArguments) BuildArch: 'x64' Platform: 'x64' @@ -74,7 +74,7 @@ jobs: del wheel_filename_file python.exe -m pip install -q --upgrade %WHEEL_FILENAME% set PATH=$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo;%PATH% - python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --use_preset_compile_flags --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 + python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --use_binskim_compliant_compile_flags --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' displayName: 'Run tests' diff --git a/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml index 2e9b19f589c19..6246bb83566e5 100644 --- a/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-qnn-ci-pipeline.yml @@ -76,7 +76,7 @@ jobs: WithCache: True Today: $(TODAY) AdditionalKey: "win-qnn | $(BuildConfig)" - BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --compile_no_warning_as_error --update --cmake_generator "Visual Studio 17 2022" --use_qnn --qnn_home $(QNN_SDK_ROOT) --parallel --use_preset_compile_flags' + BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --compile_no_warning_as_error --update --cmake_generator "Visual Studio 17 2022" --use_qnn --qnn_home $(QNN_SDK_ROOT) --parallel --use_binskim_compliant_compile_flags' MsbuildArguments: $(MsbuildArguments) BuildArch: $(buildArch) Platform: 'x64' diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh index 2482bfe825386..aec02f76693b4 100755 --- a/tools/ci_build/github/linux/build_cuda_c_api_package.sh +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -4,6 +4,6 @@ docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume \ $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ --volume /data/models:/build/models:ro --volume /data/onnx:/data/onnx:ro -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \ /usr/bin/python3.9 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ ---skip_submodule_sync --parallel --use_preset_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION \ +--skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION \ --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION \ --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index a0f5c6a7254a0..bc57cf4120d25 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -23,7 +23,7 @@ c) BUILD_CONFIG=${OPTARG};; esac done -BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--use_preset_compile_flags" "--build_wheel") +BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--use_binskim_compliant_compile_flags" "--build_wheel") if [ "$BUILD_CONFIG" != "Debug" ]; then BUILD_ARGS+=("--enable_lto") diff --git a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh index 84c06f2dd5978..7d65a6f738a5e 100755 --- a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh +++ b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh @@ -4,4 +4,4 @@ mkdir -p $HOME/.onnx docker run --gpus all -e CFLAGS -e CXXFLAGS -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ ---skip_submodule_sync --parallel --use_preset_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' +--skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80' diff --git a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh index 1d3f25df61bbb..640028ee7678c 100755 --- a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh @@ -22,7 +22,7 @@ python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir ${BUILD_DIR} --cmake_generator Ninja \ --config Debug \ --skip_submodule_sync \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --build_wheel \ --skip_tests \ --enable_training_ops \ diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh index c331fb4a27793..58d493086ece9 100755 --- a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -72,7 +72,7 @@ python3 /onnxruntime_src/tools/ci_build/build.py \ --config Debug \ --skip_submodule_sync \ --build_shared_lib \ - --parallel --use_preset_compile_flags \ + --parallel --use_binskim_compliant_compile_flags \ --minimal_build ${MINIMAL_BUILD_ARGS} \ --disable_ml_ops \ --include_ops_by_config ${REDUCED_OPS_CONFIG_FILE} \ diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index a61712adf4369..25b3610872a04 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -37,7 +37,7 @@ if [ $BUILD_OS = "yocto" ]; then make -j$(nproc) else - COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --use_preset_compile_flags --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest" + COMMON_BUILD_ARGS="--skip_submodule_sync --enable_onnx_tests --parallel --use_binskim_compliant_compile_flags --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest" if [ $BUILD_DEVICE = "gpu" ]; then _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) diff --git a/tools/scripts/python_test.sh b/tools/scripts/python_test.sh index 804905ad0cd68..39d9ed432a1dc 100755 --- a/tools/scripts/python_test.sh +++ b/tools/scripts/python_test.sh @@ -24,5 +24,5 @@ python3 -m pip install $build_dir/$config/dist/*.whl echo Run $config unit tests pushd $build_dir/$config/ -python3 $src_dir/tools/ci_build/build.py --build_dir $build_dir --cmake_generator Ninja --config $config --test --skip_submodule_sync --build_shared_lib --parallel --use_preset_compile_flags --build_wheel --enable_onnx_tests --enable_transformers_tool_test --ctest_path "" +python3 $src_dir/tools/ci_build/build.py --build_dir $build_dir --cmake_generator Ninja --config $config --test --skip_submodule_sync --build_shared_lib --parallel --use_binskim_compliant_compile_flags --build_wheel --enable_onnx_tests --enable_transformers_tool_test --ctest_path "" popd