From 35aa2cf830516622652110ff07a5e3c624e33346 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Mon, 29 Jul 2024 17:18:28 +0000 Subject: [PATCH 1/6] Load the ort shared lib in python --- src/python/CMakeLists.txt | 3 --- src/python/py/_dll_directory.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt index 189f4135c..6472fe3a8 100644 --- a/src/python/CMakeLists.txt +++ b/src/python/CMakeLists.txt @@ -13,9 +13,6 @@ if(NOT (CMAKE_SYSTEM_NAME STREQUAL "Android" OR CMAKE_SYSTEM_NAME STREQUAL "Linu target_link_libraries(python PRIVATE ${ONNXRUNTIME_LIB}) endif() -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - set_property(TARGET python APPEND_STRING PROPERTY LINK_FLAGS " -Xlinker -rpath=\\$ORIGIN/../onnxruntime/capi/") -endif() set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") if(CMAKE_GENERATOR_TOOLSET MATCHES "Visual Studio") diff --git a/src/python/py/_dll_directory.py b/src/python/py/_dll_directory.py index 9479003ff..026a8be8c 100644 --- a/src/python/py/_dll_directory.py +++ b/src/python/py/_dll_directory.py @@ -8,6 +8,10 @@ def _is_windows(): return sys.platform.startswith("win") +def _is_linux(): + return sys.platform.startswith("linux") + + def add_onnxruntime_dependency(): """Add the onnxruntime DLL directory to the DLL search path. @@ -20,6 +24,20 @@ def add_onnxruntime_dependency(): raise ImportError("Could not find the onnxruntime package.") ort_package_path = ort_package.submodule_search_locations[0] os.add_dll_directory(os.path.join(ort_package_path, "capi")) + elif _is_linux(): + import importlib.util + import ctypes + import glob + + ort_package = importlib.util.find_spec("onnxruntime") + if not ort_package: + raise ImportError("Could not find the onnxruntime package.") + + # Load the onnxruntime shared library here since we can find the path in python with ease. + # This avoids needing to know the exact path of the shared library from native code. + ort_package_path = ort_package.submodule_search_locations[0] + ort_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime.so*"))[0] + _ = ctypes.CDLL(ort_lib_path) def add_cuda_dependency(): From ccc4dfb02ced9ccd80cdd8f09c8ebe31283040ac Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Mon, 29 Jul 2024 17:54:55 +0000 Subject: [PATCH 2/6] Remove change of ort shared lib name --- .github/workflows/linux-cpu-x64-build.yml | 3 --- .github/workflows/linux-gpu-x64-build.yml | 4 ---- 2 files changed, 7 deletions(-) diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml index 7b7e08c7b..c405f1983 100644 --- a/.github/workflows/linux-cpu-x64-build.yml +++ b/.github/workflows/linux-cpu-x64-build.yml @@ -51,15 +51,12 @@ jobs: ls -R ${{ env.ORT_PACKAGE_NAME }} continue-on-error: true -# TODO: Find out why do we need to to have libonnxruntime.so.$ort_version - name: Extract OnnxRuntime library and header files run: | set -e -x mkdir -p ort/lib mv microsoft.ml.onnxruntime/${{ env.ORT_NIGHTLY_VERSION }}/build/native/include ort/ mv microsoft.ml.onnxruntime/${{ env.ORT_NIGHTLY_VERSION }}/runtimes/linux-x64/native/* ort/lib/ - ort_version=$(echo ${{ env.ORT_NIGHTLY_VERSION }} | cut -d- -f1-1) - cp ort/lib/libonnxruntime.so ort/lib/libonnxruntime.so.$ort_version - name: Build with CMake and GCC run: | diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 3006b750f..b36b8a316 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -65,16 +65,12 @@ jobs: ls -R ${{ env.ORT_PACKAGE_NAME }} continue-on-error: true -# TODO: Find out why do we need to to have libonnxruntime.so.$ort_version - name: Extract OnnxRuntime library and header files run: | set -e -x mkdir -p ort/lib mv microsoft.ml.onnxruntime.gpu.linux/${{ env.ORT_NIGHTLY_VERSION }}/buildTransitive/native/include ort/ mv microsoft.ml.onnxruntime.gpu.linux/${{ env.ORT_NIGHTLY_VERSION }}/runtimes/linux-x64/native/* ort/lib/ - ort_version=$(echo ${{ env.ORT_NIGHTLY_VERSION }} | cut -d- -f1-1) - cp ort/lib/libonnxruntime.so ort/lib/libonnxruntime.so.$ort_version - - name: Get Docker Image run: | From 9a2a03e61b03af98c720eb14e7e963e858559c68 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Mon, 29 Jul 2024 18:23:14 +0000 Subject: [PATCH 3/6] Use lib.so.1 --- .github/workflows/linux-cpu-arm64-build.yml | 3 +-- .github/workflows/linux-cpu-x64-build.yml | 2 ++ .github/workflows/linux-gpu-x64-build.yml | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-cpu-arm64-build.yml b/.github/workflows/linux-cpu-arm64-build.yml index 26b749c5e..5d8fe12b4 100644 --- a/.github/workflows/linux-cpu-arm64-build.yml +++ b/.github/workflows/linux-cpu-arm64-build.yml @@ -59,8 +59,7 @@ jobs: mkdir -p ort/lib mv microsoft.ml.onnxruntime/**/build/native/include ort/ mv microsoft.ml.onnxruntime/**/runtimes/linux-arm64/native/* ort/lib/ - ort_version=$(echo ${{ env.ORT_NIGHTLY_VERSION }} | cut -d- -f1-1) - cp ort/lib/libonnxruntime.so ort/lib/libonnxruntime.so.$ort_version + cp ort/lib/libonnxruntime.so ort/lib/libonnxruntime.so.1 - name: Download Docker Image run: | diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml index c405f1983..924604930 100644 --- a/.github/workflows/linux-cpu-x64-build.yml +++ b/.github/workflows/linux-cpu-x64-build.yml @@ -51,12 +51,14 @@ jobs: ls -R ${{ env.ORT_PACKAGE_NAME }} continue-on-error: true +# TODO: Find out why do we need to to have libonnxruntime.so.$ort_version - name: Extract OnnxRuntime library and header files run: | set -e -x mkdir -p ort/lib mv microsoft.ml.onnxruntime/${{ env.ORT_NIGHTLY_VERSION }}/build/native/include ort/ mv microsoft.ml.onnxruntime/${{ env.ORT_NIGHTLY_VERSION }}/runtimes/linux-x64/native/* ort/lib/ + cp ort/lib/libonnxruntime.so ort/lib/libonnxruntime.so.1 - name: Build with CMake and GCC run: | diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index b36b8a316..c7d7efd7f 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -65,12 +65,15 @@ jobs: ls -R ${{ env.ORT_PACKAGE_NAME }} continue-on-error: true +# TODO: Find out why do we need to to have libonnxruntime.so.$ort_version - name: Extract OnnxRuntime library and header files run: | set -e -x mkdir -p ort/lib mv microsoft.ml.onnxruntime.gpu.linux/${{ env.ORT_NIGHTLY_VERSION }}/buildTransitive/native/include ort/ mv microsoft.ml.onnxruntime.gpu.linux/${{ env.ORT_NIGHTLY_VERSION }}/runtimes/linux-x64/native/* ort/lib/ + cp ort/lib/libonnxruntime.so ort/lib/libonnxruntime.so.1 + - name: Get Docker Image run: | From 4ca6fe0a4a473245631c6ef83249e9c34e88cbb8 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Mon, 29 Jul 2024 19:24:10 +0000 Subject: [PATCH 4/6] preload providers shared --- src/python/py/_dll_directory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python/py/_dll_directory.py b/src/python/py/_dll_directory.py index 026a8be8c..125afd5e6 100644 --- a/src/python/py/_dll_directory.py +++ b/src/python/py/_dll_directory.py @@ -38,6 +38,9 @@ def add_onnxruntime_dependency(): ort_package_path = ort_package.submodule_search_locations[0] ort_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime.so*"))[0] _ = ctypes.CDLL(ort_lib_path) + providers_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime_providers_shared.so*")) + if providers_lib_path: + _ = [ctypes.CDLL(providers_lib_path[i]) for i in range(len(providers_lib_path))] def add_cuda_dependency(): From 7c436edc827b41b2e63afc972fdf7d68bc47f223 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Mon, 29 Jul 2024 19:45:53 +0000 Subject: [PATCH 5/6] add other providers as well --- src/python/py/_dll_directory.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/python/py/_dll_directory.py b/src/python/py/_dll_directory.py index 125afd5e6..d82c6a7c4 100644 --- a/src/python/py/_dll_directory.py +++ b/src/python/py/_dll_directory.py @@ -13,9 +13,11 @@ def _is_linux(): def add_onnxruntime_dependency(): - """Add the onnxruntime DLL directory to the DLL search path. + """Add the onnxruntime shared library dependency. - This function is a no-op on non-Windows platforms. + On Windows, this function adds the onnxruntime DLL directory to the DLL search path. + On Linux, this function loads the onnxruntime shared library and its dependencies + so that they can be found by the dynamic linker. """ if _is_windows(): import importlib.util @@ -36,11 +38,14 @@ def add_onnxruntime_dependency(): # Load the onnxruntime shared library here since we can find the path in python with ease. # This avoids needing to know the exact path of the shared library from native code. ort_package_path = ort_package.submodule_search_locations[0] - ort_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime.so*"))[0] - _ = ctypes.CDLL(ort_lib_path) - providers_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime_providers_shared.so*")) - if providers_lib_path: - _ = [ctypes.CDLL(providers_lib_path[i]) for i in range(len(providers_lib_path))] + ort_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime.so*")) + if not ort_lib_path: + raise ImportError("Could not find the onnxruntime shared library.") + + _ = ctypes.CDLL(ort_lib_path[0]) + + providers_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime_providers_*.so")) + _ = [ctypes.CDLL(providers_lib_path[i]) for i in range(len(providers_lib_path))] def add_cuda_dependency(): From 728e332c25edb0098f0da1f2680abf2b242a343e Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Mon, 5 Aug 2024 06:29:02 +0000 Subject: [PATCH 6/6] Use soname --- src/models/onnxruntime_api.h | 27 +-------------------------- src/python/py/_dll_directory.py | 3 --- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/models/onnxruntime_api.h b/src/models/onnxruntime_api.h index 2b7f3371e..e89613b66 100644 --- a/src/models/onnxruntime_api.h +++ b/src/models/onnxruntime_api.h @@ -213,32 +213,7 @@ inline void InitApi() { #if !defined(__ANDROID__) if (ort_lib_handle == nullptr) { - const std::array target_libraries = { - std::string("libonnxruntime.so"), - std::string("libonnxruntime.so.1.18.0"), - std::string("libonnxruntime.so.1.19.0"), - std::string("libonnxruntime.so.1.20.0")}; - - // Search parent directory - std::string current_module_dir = GetCurrentModuleDir(); - for (const std::string& lib_name : target_libraries) { - std::string pip_path{current_module_dir + "/" + lib_name}; - ort_lib_handle = LoadDynamicLibraryIfExists(pip_path); - if (ort_lib_handle != nullptr) { - break; - } - } - - if (ort_lib_handle == nullptr) { - // Search for pip installation - for (const std::string& lib_name : target_libraries) { - std::string pip_path{current_module_dir + "/../onnxruntime/capi/" + lib_name}; - ort_lib_handle = LoadDynamicLibraryIfExists(pip_path); - if (ort_lib_handle != nullptr) { - break; - } - } - } + ort_lib_handle = LoadDynamicLibraryIfExists("libonnxruntime.so.1"); } #endif diff --git a/src/python/py/_dll_directory.py b/src/python/py/_dll_directory.py index d82c6a7c4..d8fe005cd 100644 --- a/src/python/py/_dll_directory.py +++ b/src/python/py/_dll_directory.py @@ -44,9 +44,6 @@ def add_onnxruntime_dependency(): _ = ctypes.CDLL(ort_lib_path[0]) - providers_lib_path = glob.glob(os.path.join(ort_package_path, "capi", "libonnxruntime_providers_*.so")) - _ = [ctypes.CDLL(providers_lib_path[i]) for i in range(len(providers_lib_path))] - def add_cuda_dependency(): """Add the CUDA DLL directory to the DLL search path.