From 0e8d4c3d2119f74df774420e4991bb3d9c09ed95 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 12 Jan 2024 07:24:40 -0800 Subject: [PATCH] Enable Address Sanitizer in CI (#19073) ### Description 1. Add two build jobs for enabling Address Sanitizer in CI. One for Windows CPU, One for Linux CPU. 2. Set default compiler flags/linker flags in build.py for normal Windows/Linux/MacOS build. This can help control compiler flags in a more centralized way. 3. All Windows binaries in our official packages will be built with "/PROFILE" flag. Symbols of onnxruntime.dll can be found at [Microsoft public symbol server](https://learn.microsoft.com/en-us/windows-hardware/drivers/debugger/microsoft-public-symbols). Limitations: 1. On Linux Address Sanitizer ignores RPATH settings in ELF binaries. Therefore once Address Sanitizer is enabled, before running tests we need to manually set LD_LIBRARY_PATH properly otherwise libonnxruntime.so may not be able to find custom ops and shared EPs. 4. On Linux we also need to set LD_PRELOAD before running some tests(if the main executable, like python, is not built with address sanitizer. On Windows we do not need to. 5. On Windows before running python tests we should manually copy address sanitizer DLL to the onnxruntime/capi directory, because python 3.8 and above has enabled "Safe DLL Search Mode" that wouldn't use the information provided by PATH env. 6. On Linux Address Sanitizer found a lot of memory leaks from our python binding code. Therefore right now we cannot enable Address Sanitizer when building ONNX Runtime with python binding. 7. Address Sanitizer itself uses a lot of memory address space and delays memory deallocations, which is easy to cause OOM issues in 32-bit applications. We cannot run all the tests in onnxruntime_test_all in 32-bit mode with Address Sanitizer due to this reason. However, we still can run individual tests in such a way. We just cannot run all of them in one process. ### Motivation and Context To catch memory issues. --- .pipelines/windowsai-steps.yml | 2 +- cmake/adjust_global_compile_flags.cmake | 42 +---- onnxruntime/test/framework/bfc_arena_test.cc | 4 + onnxruntime/test/framework/tunable_op_test.cc | 3 + .../test/logging_apis/test_logging_apis.cc | 12 +- onnxruntime/test/shared_lib/test_inference.cc | 7 +- .../test/shared_lib/test_ort_format_models.cc | 15 +- tools/ci_build/build.py | 148 ++++++++++++++++-- .../c-api-noopenmp-packaging-pipelines.yml | 2 + .../azure-pipelines/linux-ci-pipeline.yml | 137 +++++++++------- .../azure-pipelines/mac-ios-ci-pipeline.yml | 2 + .../azure-pipelines/post-merge-jobs.yml | 3 +- .../azure-pipelines/py-packaging-pipeline.yml | 6 - .../azure-pipelines/templates/c-api-cpu.yml | 3 +- .../templates/c-api-linux-cpu.yml | 10 +- .../templates/jobs/win-ci-prebuild-steps.yml | 35 +++++ .../linux-cpu-packaging-pipeline.yml | 4 - .../templates/py-packaging-stage.yml | 7 +- .../azure-pipelines/templates/py-win-gpu.yml | 2 +- .../templates/use-xcode-version.yml | 2 +- .../azure-pipelines/templates/win-ci.yml | 6 +- .../azure-pipelines/win-ci-pipeline.yml | 30 +++- .../win-gpu-reduce-op-ci-pipeline.yml | 5 +- .../win-gpu-tensorrt-ci-pipeline.yml | 24 ++- .../github/linux/build_cuda_c_api_package.sh | 6 +- .../linux/build_linux_python_package.sh | 22 +-- .../github/linux/build_rocm_c_api_package.sh | 2 - .../linux/build_tensorrt_c_api_package.sh | 2 - .../x64/default/cpu/scripts/install_centos.sh | 4 +- .../github/linux/docker/manylinux.patch | 6 +- tools/ci_build/github/windows/helpers.ps1 | 40 ++++- .../windows/install_third_party_deps.ps1 | 6 +- 32 files changed, 386 insertions(+), 213 deletions(-) diff --git a/.pipelines/windowsai-steps.yml b/.pipelines/windowsai-steps.yml index 292ce60c6b6cf..6e551d8187171 100644 --- a/.pipelines/windowsai-steps.yml +++ b/.pipelines/windowsai-steps.yml @@ -84,7 +84,7 @@ jobs: 7z x cmake-3.26.3-windows-x86_64.zip set PYTHONHOME=$(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools set PYTHONPATH=$(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools - $(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools\python.exe "$(Build.SourcesDirectory)\tools\ci_build\build.py" --build_dir $(Build.BinariesDirectory) --build_shared_lib --enable_onnx_tests --ms_experimental --use_dml --use_winml --cmake_generator "Visual Studio 17 2022" --update --config RelWithDebInfo --enable_lto --use_telemetry --disable_rtti --enable_wcos $(BuildFlags) --cmake_extra_defines "CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" "CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" CMAKE_SYSTEM_VERSION=10.0.19041.0 --cmake_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\ctest.exe + $(Build.BinariesDirectory)\${{ parameters.PythonPackageName }}.3.9.7\tools\python.exe "$(Build.SourcesDirectory)\tools\ci_build\build.py" --build_dir $(Build.BinariesDirectory) --build_shared_lib --enable_onnx_tests --ms_experimental --use_dml --use_winml --cmake_generator "Visual Studio 17 2022" --update --config RelWithDebInfo --enable_qspectre --enable_lto --use_telemetry --disable_rtti --enable_wcos $(BuildFlags) --cmake_extra_defines "CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" "CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO=/PROFILE" CMAKE_SYSTEM_VERSION=10.0.19041.0 --cmake_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake-3.26.3-windows-x86_64\bin\ctest.exe workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Generate cmake config' diff --git a/cmake/adjust_global_compile_flags.cmake b/cmake/adjust_global_compile_flags.cmake index 9f00c873715f4..94884a3973ef6 100644 --- a/cmake/adjust_global_compile_flags.cmake +++ b/cmake/adjust_global_compile_flags.cmake @@ -74,11 +74,6 @@ if (onnxruntime_MINIMAL_BUILD) endif() if (MSVC) - # turn on LTO (which adds some compiler flags and turns on LTCG) unless it's a Debug build to minimize binary size - if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - set(onnxruntime_ENABLE_LTO ON) - endif() - # undocumented internal flag to allow analysis of a minimal build binary size if (ADD_DEBUG_INFO_TO_MINIMAL_BUILD) string(APPEND CMAKE_CXX_FLAGS " /Zi") @@ -267,37 +262,11 @@ if (MSVC) string(APPEND CMAKE_C_FLAGS " /arch:AVX512") endif() - if (NOT GDK_PLATFORM) - add_compile_definitions(WINAPI_FAMILY=100) # Desktop app - message("Building ONNX Runtime for Windows 10 and newer") - add_compile_definitions(WINVER=0x0A00 _WIN32_WINNT=0x0A00 NTDDI_VERSION=0x0A000000) - endif() if (onnxruntime_ENABLE_LTO AND NOT onnxruntime_USE_CUDA) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Gw /GL") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /Gw /GL") set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /Gw /GL") endif() - - # The WinML build tool chain builds ARM/ARM64, and the internal tool chain does not have folders for spectre mitigation libs. - # WinML performs spectre mitigation differently. - if (NOT DEFINED onnxruntime_DISABLE_QSPECTRE_CHECK) - check_cxx_compiler_flag(-Qspectre HAS_QSPECTRE) - if (HAS_QSPECTRE) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Qspectre") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Qspectre") - endif() - endif() - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DYNAMICBASE") - check_cxx_compiler_flag(-guard:cf HAS_GUARD_CF) - if (HAS_GUARD_CF) - set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /guard:cf") - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /guard:cf") - set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} /guard:cf") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /guard:cf") - set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} /guard:cf") - set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /guard:cf") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /guard:cf") - endif() else() if (NOT APPLE) #XXX: Sometimes the value of CMAKE_SYSTEM_PROCESSOR is set but it's wrong. For example, if you run an armv7 docker @@ -378,16 +347,9 @@ else() endif() -if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - #For Mac compliance - message("Adding flags for Mac builds") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector-strong") -elseif (WIN32) - # parallel build - # These compiler opitions cannot be forwarded to NVCC, so cannot use add_compiler_options - string(APPEND CMAKE_CXX_FLAGS " /MP") +if (WIN32) # required to be set explicitly to enable Eigen-Unsupported SpecialFunctions string(APPEND CMAKE_CXX_FLAGS " -DEIGEN_HAS_C99_MATH") -else() +elseif(LINUX) add_compile_definitions("_GNU_SOURCE") endif() diff --git a/onnxruntime/test/framework/bfc_arena_test.cc b/onnxruntime/test/framework/bfc_arena_test.cc index 2d3c1521f9c03..0d3e4449da939 100644 --- a/onnxruntime/test/framework/bfc_arena_test.cc +++ b/onnxruntime/test/framework/bfc_arena_test.cc @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include #include "core/framework/bfc_arena.h" #include "core/framework/allocator_utils.h" #include "gtest/gtest.h" @@ -164,6 +165,8 @@ void TestCustomMemoryLimit_ProcessException(const OnnxRuntimeException& ex) { #endif // #ifdef GTEST_USES_POSIX_RE } +// Address Sanitizer would report allocation-size-too-big if we don't disable this test. +#ifndef ABSL_HAVE_ADDRESS_SANITIZER TEST(BFCArenaTest, TestCustomMemoryLimit) { { // Configure a 1MiB byte limit @@ -214,6 +217,7 @@ TEST(BFCArenaTest, TestCustomMemoryLimit) { b.Free(first_ptr); } } +#endif TEST(BFCArenaTest, AllocationsAndDeallocationsWithGrowth) { // Max of 2GiB, but starts out small. diff --git a/onnxruntime/test/framework/tunable_op_test.cc b/onnxruntime/test/framework/tunable_op_test.cc index 0d9e557ebc813..19253e1a5bd2c 100644 --- a/onnxruntime/test/framework/tunable_op_test.cc +++ b/onnxruntime/test/framework/tunable_op_test.cc @@ -459,6 +459,8 @@ class TunableVecAddSelectFastestIfSupported : public TunableOp #include "gtest/gtest.h" // Manually initialize the Ort API object for every test. @@ -167,7 +167,13 @@ TEST_F(RealCAPITestsFixture, CApiLoggerLogMessage) { ORT_FILE, line_num, static_cast(__FUNCTION__))); } +// The code below where it tests for formatting error generates an out-of-bound memory access. Therefore we disable it +// when memory sanitizer is enabled. +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +TEST_F(RealCAPITestsFixture, DISABLED_CppApiORTCXXLOG) { +#else TEST_F(RealCAPITestsFixture, CppApiORTCXXLOG) { +#endif // Tests the output and filtering of the ORT_CXX_LOG and ORT_CXX_LOG_NOEXCEPT macros in the C++ API. // The first two calls go through, but the last two calls are filtered out due to an insufficient severity. @@ -203,7 +209,11 @@ TEST_F(RealCAPITestsFixture, CppApiORTCXXLOG) { ORT_CXX_LOG_NOEXCEPT(cpp_ort_logger, OrtLoggingLevel::ORT_LOGGING_LEVEL_INFO, "Ignored2"); } +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +TEST_F(RealCAPITestsFixture, DISABLED_CppApiORTCXXLOGF) { +#else TEST_F(RealCAPITestsFixture, CppApiORTCXXLOGF) { +#endif // Tests the output and filtering of the ORT_CXX_LOGF and ORT_CXX_LOGF_NOEXCEPT macros in the C++ API. // The first set of calls go through. The next set of calls are filtered out due to an insufficient severity. // The last calls have a formatting error and we expect an exception depending on which macro is used. diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index 35c6b308e8fea..6ffe72f81bd24 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -11,6 +11,7 @@ #include #include +#include #include "gtest/gtest.h" #include "gmock/gmock.h" @@ -402,6 +403,8 @@ TEST(CApiTest, SparseInputModel) { #endif // DISABLE_CONTRIB_OPS #endif // !defined(DISABLE_SPARSE_TENSORS) +// Memory leak +#ifndef ABSL_HAVE_ADDRESS_SANITIZER TEST(CApiTest, custom_op_handler) { std::cout << "Running custom op inference" << std::endl; @@ -435,6 +438,7 @@ TEST(CApiTest, custom_op_handler) { custom_op_domain, nullptr); #endif } +#endif #ifdef USE_CUDA TEST(CApiTest, custom_op_set_input_memory_type) { @@ -1452,7 +1456,8 @@ TEST(CApiTest, test_custom_op_library) { #endif } -#if defined(__ANDROID__) +// Has memory leak +#if defined(__ANDROID__) || defined(ABSL_HAVE_ADDRESS_SANITIZER) TEST(CApiTest, DISABLED_test_custom_op_shape_infer_attr) { // To accomodate a reduced op build pipeline #elif defined(REDUCED_OPS_BUILD) && defined(USE_CUDA) diff --git a/onnxruntime/test/shared_lib/test_ort_format_models.cc b/onnxruntime/test/shared_lib/test_ort_format_models.cc index d67c5a3048092..99a9ebc3362ae 100644 --- a/onnxruntime/test/shared_lib/test_ort_format_models.cc +++ b/onnxruntime/test/shared_lib/test_ort_format_models.cc @@ -3,7 +3,7 @@ // custom ops are only supported in a minimal build if explicitly enabled #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) - +#include #include "core/common/common.h" #include "core/graph/constants.h" #include "core/session/onnxruntime_cxx_api.h" @@ -16,10 +16,10 @@ extern std::unique_ptr ort_env; -static void TestInference(Ort::Env& env, const std::basic_string& model_uri, - const std::vector& inputs, const char* output_name, - const std::vector& expected_dims_y, const std::vector& expected_values_y, - Ort::CustomOpDomain& custom_op_domain, void* cuda_compute_stream = nullptr) { +[[maybe_unused]] static void TestInference(Ort::Env& env, const std::basic_string& model_uri, + const std::vector& inputs, const char* output_name, + const std::vector& expected_dims_y, const std::vector& expected_values_y, + Ort::CustomOpDomain& custom_op_domain, void* cuda_compute_stream = nullptr) { Ort::SessionOptions session_options; session_options.Add(custom_op_domain); @@ -27,6 +27,7 @@ static void TestInference(Ort::Env& env, const std::basic_string& mod auto cuda_options = CreateDefaultOrtCudaProviderOptionsWithCustomStream(cuda_compute_stream); session_options.AppendExecutionProvider_CUDA(cuda_options); #else + session_options.DisableCpuMemArena(); ORT_UNUSED_PARAMETER(cuda_compute_stream); #endif Ort::Session session(env, model_uri.c_str(), session_options); @@ -65,7 +66,7 @@ static void TestInference(Ort::Env& env, const std::basic_string& mod } } -#if !defined(ORT_MINIMAL_BUILD) +#if !defined(ORT_MINIMAL_BUILD) && !defined(ABSL_HAVE_ADDRESS_SANITIZER) TEST(OrtFormatCustomOpTests, ConvertOnnxModelToOrt) { const std::basic_string onnx_file = ORT_TSTR("testdata/foo_1.onnx"); const std::basic_string ort_file = ORT_TSTR("testdata/foo_1.onnx.test_output.ort"); @@ -120,7 +121,7 @@ TEST(OrtFormatCustomOpTests, ConvertOnnxModelToOrt) { // the saved ORT format model has the CPU EP assigned to the custom op node, so we only test if we're not using the // CUDA EP for the test. -#ifndef USE_CUDA +#if !defined(USE_CUDA) && !defined(ABSL_HAVE_ADDRESS_SANITIZER) TEST(OrtFormatCustomOpTests, LoadOrtModel) { const std::basic_string ort_file = ORT_TSTR("testdata/foo_1.onnx.ort"); diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 3d0ec92a7bd23..592e6d6a564fb 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -418,8 +418,18 @@ def convert_arg_line_to_args(self, arg_line): "(e.g. macOS or iOS)" "This is only supported on MacOS", ) + # A 32-bit progress doesn't have enough memory to run all the tests in onnxruntime_test_all. + # Mimalloc is incompatible with address sanitizer. + # Address sanitizer itself is also a memory leak checker, so when it is enabled we should disable_memleak_checker. parser.add_argument( - "--disable_memleak_checker", action="store_true", help="Disable memory leak checker from Windows build" + "--enable_address_sanitizer", action="store_true", help="Enable address sanitizer. Windows/Linux/MacOS only." + ) + # The following feature requires installing some special Visual Studio components that do not get installed by default. Therefore the options is default OFF. + parser.add_argument("--enable_qspectre", action="store_true", help="Enable Qspectre. Windows only.") + parser.add_argument( + "--disable_memleak_checker", + action="store_true", + help="Disable memory leak checker from Windows build. By default it is enabled in Windows Debug build. This option is Windows only.", ) # WebAssembly build @@ -600,6 +610,11 @@ def convert_arg_line_to_args(self, arg_line): "--use_telemetry", action="store_true", help="Only official builds can set this flag to enable telemetry." ) parser.add_argument("--enable_wcos", action="store_true", help="Build for Windows Core OS.") + # Do not enable LTO when the compiler is MSVC and the flag for generating debug symbols is set to /Z7 and training + # is also enabled. Because both LTO and /Zi could significantly increase *.obj/*.lib files' size, and on Windows + # there is a 4GB per file limit(ERROR LNK1248). We may solve the issue by splitting the big static libs to smaller + # ones. Before the refactoring work is done, we should avoid enabling LTO and ccache at the same time because ccache + # needs /Z7. parser.add_argument("--enable_lto", action="store_true", help="Enable Link Time Optimization") parser.add_argument("--enable_transformers_tool_test", action="store_true", help="Enable transformers tool test") parser.add_argument( @@ -1406,6 +1421,17 @@ def generate_build_tree( if args.use_lock_free_queue: add_default_definition(cmake_extra_defines, "onnxruntime_USE_LOCK_FREE_QUEUE", "ON") + if is_windows(): + if args.use_cache: + add_default_definition( + cmake_extra_defines, "CMAKE_MSVC_DEBUG_INFORMATION_FORMAT", "$<$:Embedded>" + ) + else: + # Always enable debug info even in release build. The debug information is in separated *.pdb files that + # can be easily discarded when debug symbols are not needed. We enable it by default because many auditting + # tools need to use the symbols. + add_default_definition(cmake_extra_defines, "CMAKE_MSVC_DEBUG_INFORMATION_FORMAT", "ProgramDatabase") + cmake_args += [f"-D{define}" for define in cmake_extra_defines] cmake_args += cmake_extra_args @@ -1445,8 +1471,96 @@ def generate_build_tree( f"-DVERSION_PRIVATE_PART={MM}{DD}", f"-DVERSION_STRING={ort_major}.{ort_minor}.{build_number}.{source_version[0:7]}", ] - + cflags = None + cxxflags = None + ldflags = None for config in configs: + # Setup default values for cflags/cxxflags/ldflags. + # The values set here are purely for security and compliance purposes. ONNX Runtime should work fine without these flags. + if ( + "CFLAGS" not in os.environ + and "CXXFLAGS" not in os.environ + and not args.ios + and not args.android + and not args.build_wasm + and not (is_linux() and platform.machine() != "aarch64" and platform.machine() != "x86_64") + ): + if is_windows(): + cflags = ["/guard:cf", "/DWIN32", "/D_WINDOWS"] + if args.parallel: + cflags += ["/MP"] + if not args.use_gdk: + # Target Windows 10 + cflags += [ + "/DWINAPI_FAMILY=100", + "/DWINVER=0x0A00", + "/D_WIN32_WINNT=0x0A00", + "/DNTDDI_VERSION=0x0A000000", + ] + # The "/profile" flag implies "/DEBUG:FULL /DEBUGTYPE:cv,fixup /OPT:REF /OPT:NOICF /INCREMENTAL:NO /FIXED:NO". We set it for satisfying a Microsoft internal compliance requirement. External users + # do not need to have it. + ldflags = ["/profile", "/DYNAMICBASE"] + if args.enable_qspectre: + cflags += ["/Qspectre"] + if config == "Release": + cflags += ["/O2", "/Ob2", "/DNDEBUG"] + elif config == "RelWithDebInfo": + cflags += ["/O2", "/Ob1", "/DNDEBUG"] + elif config == "Debug": + cflags += ["/Ob0", "/Od", "/RTC1"] + if args.enable_address_sanitizer: + cflags += ["/fsanitize=address"] + elif config == "MinSizeRel": + cflags += ["/O1", "/Ob1", "/DNDEBUG"] + cxxflags = cflags.copy() + if not args.disable_exceptions: + cxxflags += ["/EHsc"] + elif is_linux() or is_macOS(): + if is_linux(): + ldflags = ["-Wl,-Bsymbolic-functions", "-Wl,-z,relro", "-Wl,-z,now"] + else: + ldflags = [] + if config == "Release": + cflags = [ + "-DNDEBUG", + "-Wp,-D_FORTIFY_SOURCE=2", + "-Wp,-D_GLIBCXX_ASSERTIONS", + "-fstack-protector-strong", + "-O3", + "-pipe", + ] + if is_linux(): + ldflags += ["-Wl,--strip-all"] + elif config == "RelWithDebInfo": + cflags = [ + "-DNDEBUG", + "-Wp,-D_FORTIFY_SOURCE=2", + "-Wp,-D_GLIBCXX_ASSERTIONS", + "-fstack-protector-strong", + "-O3", + "-pipe", + "-ggdb3", + ] + elif config == "Debug": + cflags = ["-ggdb3", "-O0"] + if args.enable_address_sanitizer: + cflags += ["-fsanitize=address"] + ldflags += ["-fsanitize=address"] + elif config == "MinSizeRel": + cflags = [ + "-DNDEBUG", + "-Wp,-D_FORTIFY_SOURCE=2", + "-Wp,-D_GLIBCXX_ASSERTIONS", + "-fstack-protector-strong", + "-Os", + "-pipe", + "-ggdb3", + ] + if is_linux() and platform.machine() == "x86_64": + # The following flags needs GCC 8 and newer + cflags += ["-fstack-clash-protection", "-fcf-protection"] + cxxflags = cflags.copy() + config_build_dir = get_config_build_dir(build_dir, config) os.makedirs(config_build_dir, exist_ok=True) if args.use_tvm: @@ -1460,20 +1574,21 @@ def generate_build_tree( + os.environ["PATH"] ) preinstalled_dir = Path(build_dir) / config + temp_cmake_args = cmake_args.copy() + if cflags is not None and cxxflags is not None: + temp_cmake_args += [ + "-DCMAKE_C_FLAGS=%s" % (" ".join(cflags)), + "-DCMAKE_CXX_FLAGS=%s" % (" ".join(cxxflags)), + ] + if ldflags is not None and len(ldflags) != 0: + temp_cmake_args += [ + "-DCMAKE_EXE_LINKER_FLAGS_INIT=%s" % (" ".join(ldflags)), + "-DCMAKE_MODULE_LINKER_FLAGS_INIT=%s" % (" ".join(ldflags)), + "-DCMAKE_SHARED_LINKER_FLAGS_INIT=%s" % (" ".join(ldflags)), + ] run_subprocess( [ - *cmake_args, - "-Donnxruntime_ENABLE_MEMLEAK_CHECKER=" - + ( - "ON" - if config.lower() == "debug" - and not args.use_tvm - and not args.use_openvino - and not args.use_gdk - and not args.enable_msvc_static_runtime - and not args.disable_memleak_checker - else "OFF" - ), + *temp_cmake_args, f"-DCMAKE_BUILD_TYPE={config}", f"-DCMAKE_PREFIX_PATH={build_dir}/{config}/installed" if preinstalled_dir.exists() and not (args.arm64 or args.arm64ec or args.arm) @@ -2363,6 +2478,10 @@ def main(): cmake_extra_defines = normalize_arg_list(args.cmake_extra_defines) cross_compiling = args.arm or args.arm64 or args.arm64ec or args.android + if args.enable_address_sanitizer: + # Disable ONNX Runtime's builtin memory checker + args.disable_memleak_checker = True + # If there was no explicit argument saying what to do, default # to update, build and test (for native builds). if not (args.update or args.clean or args.build or args.test or args.gen_doc): @@ -2665,6 +2784,7 @@ def main(): # fail unexpectedly. Similar, if your packaging step forgot to copy a file into the package, we don't know it # either. if args.build: + # TODO: find asan DLL and copy it to onnxruntime/capi folder when args.enable_address_sanitizer is True and the target OS is Windows if args.build_wheel: nightly_build = bool(os.getenv("NIGHTLY_BUILD") == "1") default_training_package_device = bool(os.getenv("DEFAULT_TRAINING_PACKAGE_DEVICE") == "1") diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index f97bf2dc6987f..024fc1116954f 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -727,6 +727,7 @@ stages: dependsOn: - Setup - Windows_Packaging_gpu + - Windows_Packaging_CPU_x64_default - Windows_Packaging_tensorrt - Linux_C_API_Packaging_GPU_x64 - Linux_C_API_Packaging_GPU_TensorRT_x64 @@ -780,6 +781,7 @@ stages: SpecificArtifact: ${{ parameters.SpecificArtifact }} BuildId: ${{ parameters.BuildId }} + # The following one is from a CPU job that publishes protoc.exe - template: templates/flex-downloadPipelineArtifact.yml parameters: StepName: 'Download Pipeline Artifact - NuGet' diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 64b78dca504ca..07f672c75d029 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -38,7 +38,84 @@ stages: - stage: x64 dependsOn: [] jobs: - - job: Linux_Build + - job: Linux_Debug + timeoutInMinutes: 180 + workspace: + clean: all + variables: + skipComponentGovernanceDetection: true + ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + pool: onnxruntime-Ubuntu2004-AMD-CPU + steps: + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + + - checkout: self + clean: true + submodules: none + + - template: templates/get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/inference/x64/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x64/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=registry.access.redhat.com/ubi8/ubi" + Repository: onnxruntimecpubuildcentos8x64 + + - template: templates/linux-build-step-with-cache.yml + parameters: + WithCache: false + Today: $(TODAY) + AdditionalKey: onnxruntime_linux_debug_with_address_sanitizer + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: CmdLine@2 + displayName: 'build' + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume /data/models:/data/models:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimecpubuildcentos8x64 \ + /bin/bash -c " + set -ex; \ + python3.9 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator 'Ninja' \ + --config Debug \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_csharp \ + --enable_onnx_tests --enable_address_sanitizer \ + --update --build; + LD_PRELOAD=/usr/lib64/libasan.so.6 python3.9 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator 'Ninja' \ + --config Debug \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_csharp \ + --enable_onnx_tests --enable_address_sanitizer \ + --test;" + workingDirectory: $(Build.SourcesDirectory) + + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + - job: Linux_Release timeoutInMinutes: 180 workspace: clean: all @@ -148,7 +225,7 @@ stages: ccache -s; \ /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator 'Ninja' \ - --config Debug Release \ + --config Release \ --skip_submodule_sync \ --build_shared_lib \ --parallel \ @@ -166,61 +243,7 @@ stages: ln -s /data/models $(Build.BinariesDirectory)/models displayName: link model dir - - bash: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuild \ - /bin/bash -c " - set -ex; \ - pushd /onnxruntime_src/csharp; \ - dotnet restore /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln; \ - dotnet build /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln; \ - dotnet test /onnxruntime_src/csharp/OnnxRuntime.DesktopOnly.CSharp.sln -f net6.0 --no-build -l \"console;verbosity=normal\"; \ - popd - " - displayName: 'Dotnet build C# sln and Test' - - - bash: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuild \ - /bin/bash -c " - set -ex; \ - /bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Release && \ - /bin/bash /onnxruntime_src/tools/scripts/symbolic_shape_infer_test.sh /build - " - displayName: 'Run Release tests and symbolic shape infer test' - - - bash: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuild \ - /bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Debug - displayName: 'Run Debug tests' + - task: PublishTestResults@2 displayName: 'Publish unit test results' diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml index 18d53654e7c4d..33701fccf0c5f 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml @@ -38,6 +38,8 @@ jobs: timeoutInMinutes: 150 steps: - template: templates/use-xcode-version.yml + parameters: + xcodeVersion: 14.3 - template: templates/mac-build-step-with-cache.yml parameters: WithCache: true diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index e26e3e9abd0b9..5ee39876733e2 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -460,7 +460,8 @@ stages: architecture: "x64" - template: templates/use-xcode-version.yml - + parameters: + xcodeVersion: 14.3 - script: | pip install -r tools/ci_build/github/apple/ios_packaging.requirements.txt displayName: "Install Python requirements" diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index 62f84a9bb185c..06cca0068523d 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -24,11 +24,6 @@ parameters: type: boolean default: true -- name: enable_mac_silicon - displayName: 'Whether Mac silicon package is built.' - type: boolean - default: true - - name: enable_linux_arm displayName: 'Whether Linux ARM package is built.' type: boolean @@ -68,7 +63,6 @@ stages: enable_windows_cpu: ${{ parameters.enable_windows_cpu }} enable_windows_gpu: ${{ parameters.enable_windows_gpu }} enable_mac_cpu: ${{ parameters.enable_mac_cpu }} - enable_mac_silicon: ${{ parameters.enable_mac_silicon }} enable_linux_arm: ${{ parameters.enable_linux_arm }} build_py_parameters: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index e6025ae1b56bd..81319e07c6b17 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -107,7 +107,8 @@ stages: - template: set-version-number-variables-step.yml - template: use-xcode-version.yml - + parameters: + xcodeVersion: 14.3 - script: | /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \ $(Build.SourcesDirectory) \ diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index 15fcec0511741..8538f15e93753 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -11,12 +11,6 @@ parameters: - name: OnnxruntimeArch type: string -- name: OnnxruntimeCFlags - type: string - -- name: OnnxruntimeCXXFlags - type: string - - name: OnnxruntimeNodejsBindingArch type: string values: @@ -67,9 +61,9 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm -e CFLAGS="${{parameters.OnnxruntimeCFlags}}" -e CXXFLAGS="${{parameters.OnnxruntimeCXXFlags}}" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}} /bin/bash -c "python3.9 \ - /onnxruntime_src/tools/ci_build/build.py --build_java --build_nodejs --build_dir /build --config Release \ + /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-${{parameters.OnnxruntimeArch}}" workingDirectory: $(Build.SourcesDirectory) displayName: 'Build' diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml index 09c52f4d5ba0d..9516753d50113 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml @@ -30,6 +30,41 @@ steps: addToPath: true architecture: ${{parameters.BuildArch}} +- ${{ if eq(parameters.BuildArch, 'x64') }}: + - script: | + @echo off + set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" + for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do ( + if exist "%%i\VC\Auxiliary\Build\vcvars64.bat" ( + set vcvarsall="%%i\VC\Auxiliary\Build\vcvars64.bat" + ) + ) + + @echo %vcvarsall% will be used as the VC compiler + @echo ##vso[task.setvariable variable=vcvarsall]%vcvarsall% + displayName: 'locate vcvarsall via vswhere' + +- ${{ if eq(parameters.BuildArch, 'x86') }}: + - script: | + @echo off + set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" + for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do ( + if exist "%%i\VC\Auxiliary\Build\vcvars32.bat" ( + set vcvarsall="%%i\VC\Auxiliary\Build\vcvars32.bat" + ) + ) + + @echo %vcvarsall% will be used as the VC compiler + @echo ##vso[task.setvariable variable=vcvarsall]%vcvarsall% + displayName: 'locate vcvarsall via vswhere' + +- task: BatchScript@1 + displayName: 'Setup VC env' + inputs: + filename: '$(vcvarsall)' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' + - script: | python -m pip install --upgrade "setuptools>=68.2.2" wheel numpy flatbuffers workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-cpu-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/linux-cpu-packaging-pipeline.yml index 1cc5c48c5513c..6ad5f9f38a4db 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-cpu-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-cpu-packaging-pipeline.yml @@ -31,8 +31,6 @@ stages: AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} BaseImage: 'registry.access.redhat.com/ubi8/ubi' OnnxruntimeArch: 'x64' - OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' - OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' OnnxruntimeNodejsBindingArch: 'x64' PoolName: 'onnxruntime-Ubuntu2004-AMD-CPU' ArtifactNamePrefix: ${{ parameters.ArtifactNamePrefix }} @@ -44,8 +42,6 @@ stages: AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} BaseImage: 'arm64v8/almalinux:8' OnnxruntimeArch: 'aarch64' - OnnxruntimeCFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' - OnnxruntimeCXXFlags: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' OnnxruntimeNodejsBindingArch: 'arm64' PoolName: 'onnxruntime-linux-ARM64-CPU-2019' ArtifactNamePrefix: ${{ parameters.ArtifactNamePrefix }} diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index a3c2983b755d0..abe06e80f4f19 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -30,11 +30,6 @@ parameters: type: boolean default: true -- name: enable_mac_silicon - displayName: 'Whether Mac silicon package is built.' - type: boolean - default: true - - name: enable_linux_arm displayName: 'Whether Linux ARM package is built.' type: boolean @@ -382,6 +377,8 @@ stages: inputs: versionSpec: $(PythonVersion) + - template: use-xcode-version.yml + - script: | set -e -x export _PYTHON_HOST_PLATFORM=macosx-${{variables.MACOSX_DEPLOYMENT_TARGET}}-universal2 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index 501251eaff20f..c83e130dd26e8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -110,7 +110,7 @@ jobs: inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' arguments: > - --config RelWithDebInfo + --config RelWithDebInfo --enable_qspectre --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "$(VSGenerator)" diff --git a/tools/ci_build/github/azure-pipelines/templates/use-xcode-version.yml b/tools/ci_build/github/azure-pipelines/templates/use-xcode-version.yml index 7d767b4f4fde6..5742b6c60fec5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/use-xcode-version.yml +++ b/tools/ci_build/github/azure-pipelines/templates/use-xcode-version.yml @@ -3,7 +3,7 @@ parameters: - name: xcodeVersion type: string - default: "14.3" + default: "15.1" steps: - bash: | diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 89c481f267e64..31e41eb4bc2d7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -150,9 +150,9 @@ stages: inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' ${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}: - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --test_all_timeout 72000' + arguments: '--config RelWithDebInfo --enable_qspectre --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --test_all_timeout 72000' ${{ else }}: - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ' + arguments: '--config RelWithDebInfo --enable_qspectre --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} ' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -172,7 +172,7 @@ stages: condition: and(succeeded(), eq('${{ parameters.runTests}}', true)) inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' + arguments: '--config RelWithDebInfo --enable_qspectre --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}' workingDirectory: '$(Build.BinariesDirectory)' - script: | diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index d7ffc1828c943..71dcdf0cc76ac 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -52,6 +52,32 @@ stages: WITH_CACHE: true MachinePool: 'onnxruntime-Win-CPU-2022' + - job: build_x64_asan + pool: 'onnxruntime-Win-CPU-2022' + timeoutInMinutes: 300 + steps: + - checkout: self + clean: true + submodules: none + + - template: templates/jobs/win-ci-prebuild-steps.yml + parameters: + EnvSetupScript: setup_env.bat + DownloadCUDA: false + BuildArch: x64 + BuildConfig: Debug + MachinePool: 'onnxruntime-Win-CPU-2022' + WithCache: false + Today: $(TODAY) + + - task: PythonScript@0 + displayName: 'Build and Test' + inputs: + scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' + arguments: --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --cmake_generator "Visual Studio 17 2022" --disable_memleak_checker --enable_address_sanitizer + workingDirectory: '$(Build.BinariesDirectory)' + + - stage: x64_release dependsOn: [] jobs: @@ -127,9 +153,9 @@ stages: isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false - WITH_CACHE: false + WITH_CACHE: true MachinePool: 'onnxruntime-Win-CPU-2022' - + - stage: x86_release dependsOn: [] jobs: diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml index d0f9772da7adc..9133db79946b5 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml @@ -11,7 +11,6 @@ jobs: variables: MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary' EnvSetupScript: setup_env_cuda.bat - buildArch: x64 TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 120 workspace: @@ -21,7 +20,7 @@ jobs: parameters: EnvSetupScript: $(EnvSetupScript) DownloadCUDA: true - BuildArch: $(buildArch) + BuildArch: 'x64' BuildConfig: $(BuildConfig) MachinePool: 'onnxruntime-Win2022-GPU-T4' WithCache: true @@ -34,7 +33,7 @@ jobs: AdditionalKey: "gpu-reduced-ops | $(BuildConfig)" BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --update --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"' MsbuildArguments: $(MsbuildArguments) - BuildArch: $(buildArch) + BuildArch: 'x64' Platform: 'x64' BuildConfig: $(BuildConfig) diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml index 658c358aa4523..2c4e4eb011783 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml @@ -33,8 +33,6 @@ jobs: variables: MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary' EnvSetupScript: setup_env_trt.bat - buildArch: x64 - BuildConfig: 'RelWithDebInfo' skipComponentGovernanceDetection: true TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 150 @@ -45,8 +43,8 @@ jobs: parameters: EnvSetupScript: $(EnvSetupScript) DownloadCUDA: true - BuildArch: $(buildArch) - BuildConfig: $(BuildConfig) + BuildArch: 'x64' + BuildConfig: RelWithDebInfo MachinePool: 'onnxruntime-Win2022-GPU-T4' WithCache: true Today: $(Today) @@ -55,28 +53,28 @@ jobs: parameters: WithCache: True Today: $(TODAY) - AdditionalKey: "gpu-tensorrt | $(BuildConfig)" - BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75' + AdditionalKey: "gpu-tensorrt | RelWithDebInfo" + BuildPyArguments: '--config RelWithDebInfo --enable_qspectre --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75' MsbuildArguments: $(MsbuildArguments) - BuildArch: $(buildArch) + BuildArch: 'x64' Platform: 'x64' - BuildConfig: $(BuildConfig) + BuildConfig: RelWithDebInfo - task: PythonScript@0 displayName: 'Build wheel' inputs: scriptPath: '$(Build.SourcesDirectory)\setup.py' arguments: 'bdist_wheel' - workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - script: | - mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models $(Build.BinariesDirectory)\models + mklink /D /J $(Build.BinariesDirectory)\RelWithDebInfo\models $(Build.BinariesDirectory)\models DIR dist\ /S /B > wheel_filename_file set /p WHEEL_FILENAME==68.2.2", "wheel", "numpy", "protobuf==$protobuf_version" &"python.exe" $pip_args @@ -661,8 +687,8 @@ function Install-ONNX { $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF -DCMAKE_PREFIX_PATH=$install_prefix" python.exe "setup.py" "bdist_wheel" - - + + Write-Host "Installing the newly built ONNX python package" Get-ChildItem -Path dist/*.whl | foreach { $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "--disable-pip-version-check", "install", "--upgrade", $_.fullname diff --git a/tools/ci_build/github/windows/install_third_party_deps.ps1 b/tools/ci_build/github/windows/install_third_party_deps.ps1 index c30b576953114..54507cd40cc47 100644 --- a/tools/ci_build/github/windows/install_third_party_deps.ps1 +++ b/tools/ci_build/github/windows/install_third_party_deps.ps1 @@ -93,11 +93,11 @@ if(-not (Test-Path $vshwere_path -PathType Leaf)){ $msbuild_path = &$vshwere_path -latest -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe | select-object -first 1 -Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path +Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path -cpu_arch $cpu_arch -Install-Abseil -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path +Install-Abseil -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path -cpu_arch $cpu_arch -Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path +Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path -cpu_arch $cpu_arch $protobuf_version="4.21.12"