From 695af57b0177d461850cb38de65c99208e5647f5 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 10:21:05 -0800 Subject: [PATCH 01/21] Enable shared libraries --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a418ca303..8773f8e7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,7 +95,7 @@ endif() # set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) #endif() -add_library (onnxruntime-genai ${generator_srcs}) +add_library (onnxruntime-genai SHARED ${generator_srcs}) target_include_directories(onnxruntime-genai PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) if(USE_TOKENIZER) @@ -116,7 +116,7 @@ if(USE_TOKENIZER) FetchContent_Declare(simdjson URL https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip URL_HASH SHA1=2b063a2e81f74a5d1cb937fadf3d2fca0f1edb09) FetchContent_MakeAvailable (simdjson) - add_library (tokenizer ${tokenizer_srcs}) + add_library (tokenizer SHARED ${tokenizer_srcs}) target_include_directories(tokenizer PUBLIC ${TOKENIZER_ROOT} ${GSL_SOURCE_DIR}/include From 9da119a513294fc30d4af545f01045f4e2f54db1 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 10:38:47 -0800 Subject: [PATCH 02/21] Enable shared libraries --- CMakeLists.txt | 232 +++++++++++++++++++++++++------------------------ 1 file changed, 117 insertions(+), 115 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8773f8e7e..33b682a54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,11 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -cmake_minimum_required (VERSION 3.26) +cmake_minimum_required(VERSION 3.26) include(FetchContent) include(cmake/external/onnxruntime_external_deps.cmake) -project (Generators LANGUAGES C CXX) +project(Generators LANGUAGES C CXX) set(USE_CUDA 1) # "Build with CUDA support" set(USE_TOKENIZER 1) # "Build with Tokenizer support" set(BUILD_WHEEL 1) @@ -14,23 +14,23 @@ set(BUILD_WHEEL 1) include(CheckLanguage) check_language(CUDA) if (CMAKE_CUDA_COMPILER) - message(STATUS "CUDA found CUDAToolkit_VERSION ${CUDAToolkit_VERSION} with CMAKE_CUDA_COMPILER ${CMAKE_CUDA_COMPILER}" ) -# set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) -else() + message(STATUS "CUDA found CUDAToolkit_VERSION ${CUDAToolkit_VERSION} with CMAKE_CUDA_COMPILER ${CMAKE_CUDA_COMPILER}") + # set(CMAKE_CUDA_RUNTIME_LIBRARY Shared) +else () message(STATUS "CUDA not found") -endif() +endif () -if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9) - message(FATAL_ERROR "GCC version must be greater than or equal to 9") -endif() +if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9) + message(FATAL_ERROR "GCC version must be greater than or equal to 9") +endif () -if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 10) - add_compile_definitions(USE_CXX17=1) - message("USE C++17") - set(CMAKE_CXX_STANDARD 17) -else() - set(CMAKE_CXX_STANDARD 20) -endif() +if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 10) + add_compile_definitions(USE_CXX17=1) + message("USE C++17") + set(CMAKE_CXX_STANDARD 17) +else () + set(CMAKE_CXX_STANDARD 20) +endif () set(GENERATORS_ROOT ${PROJECT_SOURCE_DIR}/src) set(MODELS_ROOT ${PROJECT_SOURCE_DIR}/src/models) @@ -40,117 +40,117 @@ set(TOKENIZER_ROOT ${PROJECT_SOURCE_DIR}/src/tokenizer) # CUDA Being enabled will make it not a debug build without this option, so all of the C++ headers will complain # about a mismatch with the actual debug headers and it'll fail to link. I don't know why this happens, or if this is the best fix. -if(USE_CUDA AND CMAKE_CUDA_COMPILER AND CMAKE_BUILD_TYPE STREQUAL "Debug") - add_compile_definitions(_DEBUG=1) -endif() +if (USE_CUDA AND CMAKE_CUDA_COMPILER AND CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_definitions(_DEBUG=1) +endif () message(STATUS "Adding source files") file(GLOB generator_srcs CONFIGURE_DEPENDS - "${GENERATORS_ROOT}/*.h" - "${GENERATORS_ROOT}/*.cpp" - "${MODELS_ROOT}/*.h" - "${MODELS_ROOT}/*.cpp" + "${GENERATORS_ROOT}/*.h" + "${GENERATORS_ROOT}/*.cpp" + "${MODELS_ROOT}/*.h" + "${MODELS_ROOT}/*.cpp" ) file(GLOB test_srcs CONFIGURE_DEPENDS - "${TESTS_ROOT}/*.h" - "${TESTS_ROOT}/*.cpp" + "${TESTS_ROOT}/*.h" + "${TESTS_ROOT}/*.cpp" ) file(GLOB python_srcs CMAKE_CONFIGURE_DEPENDS - "${PYTHON_ROOT}/*.h" - "${PYTHON_ROOT}/*.cpp" + "${PYTHON_ROOT}/*.h" + "${PYTHON_ROOT}/*.cpp" ) file(GLOB tokenizer_srcs CONFIGURE_DEPENDS - "${TOKENIZER_ROOT}/*.cc" - "${TOKENIZER_ROOT}/utils/*.cc" - "${TOKENIZER_ROOT}/c_api/*.cc" + "${TOKENIZER_ROOT}/*.cc" + "${TOKENIZER_ROOT}/utils/*.cc" + "${TOKENIZER_ROOT}/c_api/*.cc" ) -if(USE_CUDA AND CMAKE_CUDA_COMPILER) +if (USE_CUDA AND CMAKE_CUDA_COMPILER) # Don't let cmake set a default value for CMAKE_CUDA_ARCHITECTURES cmake_policy(SET CMP0104 OLD) enable_language(CUDA) - message( STATUS "CMAKE_CUDA_COMPILER_VERSION: ${CMAKE_CUDA_COMPILER_VERSION}") + message(STATUS "CMAKE_CUDA_COMPILER_VERSION: ${CMAKE_CUDA_COMPILER_VERSION}") # set(CUDA_PROPAGATE_HOST_FLAGS ON) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcudafe --diag_suppress=2803 --expt-relaxed-constexpr") file(GLOB generator_cuda_srcs CONFIGURE_DEPENDS - "${GENERATORS_ROOT}/*.cu" - "${GENERATORS_ROOT}/*.cuh" - "${MODELS_ROOT}/*.cu" - "${MODELS_ROOT}/*.cuh" + "${GENERATORS_ROOT}/*.cu" + "${GENERATORS_ROOT}/*.cuh" + "${MODELS_ROOT}/*.cu" + "${MODELS_ROOT}/*.cuh" ) list(APPEND generator_srcs ${generator_cuda_srcs}) add_compile_definitions(USE_CUDA=1) include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") -else() +else () file(GLOB generator_cuda_srcs "${GENERATORS_ROOT}/*_cuda*.*") list(REMOVE_ITEM generator_srcs ${generator_cuda_srcs}) -endif() +endif () ## TODO: this is a hack to get the shared library to export .lib files on windows. We should find a better way when we need to compile dynamic libraries #if(WIN32) # set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) #endif() -add_library (onnxruntime-genai SHARED ${generator_srcs}) +add_library(onnxruntime-genai SHARED ${generator_srcs}) target_include_directories(onnxruntime-genai PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) -if(USE_TOKENIZER) +if (USE_TOKENIZER) FetchContent_Declare(spm URL https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.zip) set(SPM_USE_EXTERNAL_ABSL OFF CACHE BOOL "Use external absl") set(SPM_USE_BUILTIN_PROTOBUF ON CACHE BOOL "Use built-in protobuf") set(SPM_ENABLE_SHARED OFF) FetchContent_GetProperties(spm) - if(NOT spm_POPULATED) - FetchContent_Populate(spm) - add_subdirectory(${spm_SOURCE_DIR} ${spm_BINARY_DIR} EXCLUDE_FROM_ALL) - endif() + if (NOT spm_POPULATED) + FetchContent_Populate(spm) + add_subdirectory(${spm_SOURCE_DIR} ${spm_BINARY_DIR} EXCLUDE_FROM_ALL) + endif () FetchContent_Declare(GSL URL https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip) FetchContent_MakeAvailable(GSL) FetchContent_Declare(simdjson URL https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip URL_HASH SHA1=2b063a2e81f74a5d1cb937fadf3d2fca0f1edb09) - FetchContent_MakeAvailable (simdjson) + FetchContent_MakeAvailable(simdjson) - add_library (tokenizer SHARED ${tokenizer_srcs}) + add_library(tokenizer SHARED ${tokenizer_srcs}) target_include_directories(tokenizer PUBLIC - ${TOKENIZER_ROOT} - ${GSL_SOURCE_DIR}/include - ${simdjson_SOURCE_DIR}/singleheader/ - ${spm_SOURCE_DIR}/src - ${spm_SOURCE_DIR}/src/builtin_pb - ${spm_SOURCE_DIR}/third_party/absl - ${spm_SOURCE_DIR}/third_party/protobuf-lite + ${TOKENIZER_ROOT} + ${GSL_SOURCE_DIR}/include + ${simdjson_SOURCE_DIR}/singleheader/ + ${spm_SOURCE_DIR}/src + ${spm_SOURCE_DIR}/src/builtin_pb + ${spm_SOURCE_DIR}/third_party/absl + ${spm_SOURCE_DIR}/third_party/protobuf-lite ) target_compile_definitions(tokenizer PRIVATE _SILENCE_ALL_CXX20_DEPRECATION_WARNINGS) add_dependencies(tokenizer sentencepiece-static) target_link_libraries(tokenizer PUBLIC sentencepiece-static simdjson) -endif() +endif () -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") set_target_properties(onnxruntime-genai PROPERTIES POSITION_INDEPENDENT_CODE ON) -endif() - -if(WIN32) - set(ONNXRUNTIME_LIB "onnxruntime.lib") - set(ONNXRUNTIME_FILES "onnxruntime*.dll") - set(ONNXRUNTIME_EXTENSIONS_LIB "tfmtok_c.lib") - set(ONNXRUNTIME_EXTENSIONS_FILES "tfmtok_c.dll") -elseif(APPLE) - set(ONNXRUNTIME_LIB "libonnxruntime.dylib") - set(ONNXRUNTIME_FILES "libonnxruntime*.dylib") -else() - set(ONNXRUNTIME_LIB "libonnxruntime.so") - set(ONNXRUNTIME_FILES "libonnxruntime*.so*") - set(ONNXRUNTIME_EXTENSIONS_LIB "tfmtok_c.so") -endif() - -add_executable (Tests ${test_srcs}) +endif () + +if (WIN32) + set(ONNXRUNTIME_LIB "onnxruntime.lib") + set(ONNXRUNTIME_FILES "onnxruntime*.dll") + set(ONNXRUNTIME_EXTENSIONS_LIB "tfmtok_c.lib") + set(ONNXRUNTIME_EXTENSIONS_FILES "tfmtok_c.dll") +elseif (APPLE) + set(ONNXRUNTIME_LIB "libonnxruntime.dylib") + set(ONNXRUNTIME_FILES "libonnxruntime*.dylib") +else () + set(ONNXRUNTIME_LIB "libonnxruntime.so") + set(ONNXRUNTIME_FILES "libonnxruntime*.so*") + set(ONNXRUNTIME_EXTENSIONS_LIB "tfmtok_c.so") +endif () + +add_executable(Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) @@ -160,16 +160,18 @@ target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(python PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") - +if (WIN32) + SET(WINDOWS_EXPORT_ALL_SYMBOLS ON) +endif () # TODO: Windows is statically linking CUDA libraries, so remove the 'NOT WIN32' once it's using shared libraries -if(NOT WIN32 AND USE_CUDA AND CMAKE_CUDA_COMPILER) +if (NOT WIN32 AND USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) -endif() +endif () # Visual C++ - /MP Multiple process build, /WX Warnings as errors @@ -177,71 +179,71 @@ endif() # C5038: data member 'member1' will be initialized after data member 'member2' # Disable: # C4100: 'this_': unreferenced formal parameter -if(MSVC) +if (MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP /w15038 /wd4100 /W4 /WX") -endif() +endif () # Copy the onnxruntime binaries into the build folder so it's found on launch file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/${ONNXRUNTIME_FILES}") foreach(DLL_FILE ${onnxruntime_libs}) add_custom_command( - TARGET onnxruntime-genai POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${DLL_FILE} - ${CMAKE_BINARY_DIR}/${DLL_FILE_NAME} + TARGET onnxruntime-genai POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${DLL_FILE} + ${CMAKE_BINARY_DIR}/${DLL_FILE_NAME} ) endforeach() - if(USE_TOKENIZER) +if (USE_TOKENIZER) add_compile_definitions(USE_TOKENIZER=1) target_include_directories(onnxruntime-genai PRIVATE ${TOKENIZER_ROOT}) target_include_directories(Tests PRIVATE ${TOKENIZER_ROOT}) target_include_directories(python PRIVATE ${TOKENIZER_ROOT}) target_link_libraries(Tests PRIVATE tokenizer) target_link_libraries(python PRIVATE tokenizer) -endif() +endif () -if(BUILD_WHEEL) - set(WHEEL_FILES_DIR "${CMAKE_BINARY_DIR}/wheel") - message("Setting up wheel files in : ${WHEEL_FILES_DIR}") +if (BUILD_WHEEL) + set(WHEEL_FILES_DIR "${CMAKE_BINARY_DIR}/wheel") + message("Setting up wheel files in : ${WHEEL_FILES_DIR}") - set(TARGET_NAME "onnxruntime_genai") - - configure_file(${PYTHON_ROOT}/setup.py.in ${WHEEL_FILES_DIR}/setup.py @ONLY) - configure_file(${PYTHON_ROOT}/py/__init__.py.in ${WHEEL_FILES_DIR}/${TARGET_NAME}/__init__.py @ONLY) + set(TARGET_NAME "onnxruntime_genai") - file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/${ONNXRUNTIME_FILES}") - foreach(DLL_FILE ${onnxruntime_libs}) - add_custom_command( - TARGET onnxruntime-genai - COMMAND ${CMAKE_COMMAND} -E copy + configure_file(${PYTHON_ROOT}/setup.py.in ${WHEEL_FILES_DIR}/setup.py @ONLY) + configure_file(${PYTHON_ROOT}/py/__init__.py.in ${WHEEL_FILES_DIR}/${TARGET_NAME}/__init__.py @ONLY) + + file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/${ONNXRUNTIME_FILES}") + foreach(DLL_FILE ${onnxruntime_libs}) + add_custom_command( + TARGET onnxruntime-genai + COMMAND ${CMAKE_COMMAND} -E copy ${DLL_FILE} ${WHEEL_FILES_DIR}/${TARGET_NAME}/${DLL_FILE_NAME} + ) + endforeach() + + + # Copy over any additional python files + file(GLOB pyfiles "${PYTHON_ROOT}/py/*.py") + foreach(filename ${pyfiles}) + get_filename_component(target "${filename}" NAME) + message(STATUS "Copying ${filename} to ${target}") + configure_file("${filename}" "${WHEEL_FILES_DIR}/${TARGET_NAME}" COPYONLY) + endforeach(filename) + + add_custom_target(PyPackageBuild + COMMAND ${CMAKE_COMMAND} -E copy $ ${WHEEL_FILES_DIR}/${TARGET_NAME} + COMMAND "${PYTHON_EXECUTABLE}" -m pip wheel . + WORKING_DIRECTORY "${WHEEL_FILES_DIR}" + COMMENT "Building wheel" ) - endforeach() - - - # Copy over any additional python files - file(GLOB pyfiles "${PYTHON_ROOT}/py/*.py") - foreach(filename ${pyfiles}) - get_filename_component(target "${filename}" NAME) - message(STATUS "Copying ${filename} to ${target}") - configure_file("${filename}" "${WHEEL_FILES_DIR}/${TARGET_NAME}" COPYONLY) - endforeach (filename) - - add_custom_target(PyPackageBuild - COMMAND ${CMAKE_COMMAND} -E copy $ ${WHEEL_FILES_DIR}/${TARGET_NAME} - COMMAND "${PYTHON_EXECUTABLE}" -m pip wheel . - WORKING_DIRECTORY "${WHEEL_FILES_DIR}" - COMMENT "Building wheel" - ) - add_dependencies(PyPackageBuild python) -endif() + add_dependencies(PyPackageBuild python) +endif () # Have visual studio put all files into one single folder vs the default split of header files into a separate folder source_group(TREE ${GENERATORS_ROOT} FILES ${generator_srcs}) source_group("Sources" FILES ${test_srcs}) source_group("Sources" FILES ${python_srcs}) -set_property (DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT Tests) +set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT Tests) From 3efb6ef2b3257e79533b708bd13f3bc7465c0ec5 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 10:49:40 -0800 Subject: [PATCH 03/21] Enable shared libraries --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33b682a54..f557d9ebd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,7 +161,7 @@ target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(python PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") if (WIN32) - SET(WINDOWS_EXPORT_ALL_SYMBOLS ON) + SET(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) endif () # TODO: Windows is statically linking CUDA libraries, so remove the 'NOT WIN32' once it's using shared libraries if (NOT WIN32 AND USE_CUDA AND CMAKE_CUDA_COMPILER) From 109ad6e273428d7ec880e44fd344711892e0c536 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 16:04:20 -0800 Subject: [PATCH 04/21] Make it shared library --- CMakeLists.txt | 18 +++++++++++------- src/tokenizer/CMakeLists.txt | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d65654974..651757c33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,9 +80,11 @@ endif() #if(WIN32) # set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) #endif() - -add_library(onnxruntime-genai ${generator_srcs}) +add_library(onnxruntime-genai SHARED ${generator_srcs}) +add_library(onnxruntime-genai-static ${generator_srcs}) target_include_directories(onnxruntime-genai PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) +target_include_directories(onnxruntime-genai-static PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) + if(USE_TOKENIZER) add_subdirectory("${CMAKE_SOURCE_DIR}/src/tokenizer") @@ -91,6 +93,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set_target_properties(onnxruntime-genai PROPERTIES POSITION_INDEPENDENT_CODE ON) + set_target_properties(onnxruntime-genai-static PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() if(WIN32) @@ -115,15 +118,17 @@ target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) -target_link_libraries(python PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) +target_link_libraries(python PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") # TODO: Windows is statically linking CUDA libraries, so remove the 'NOT WIN32' once it's using shared libraries if(NOT WIN32 AND USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) + target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) endif() @@ -152,8 +157,10 @@ endforeach() if(USE_TOKENIZER) add_compile_definitions(USE_TOKENIZER=1) target_include_directories(onnxruntime-genai PRIVATE ${TOKENIZER_ROOT}) + target_include_directories(onnxruntime-genai-static PRIVATE ${TOKENIZER_ROOT}) target_include_directories(Tests PRIVATE ${TOKENIZER_ROOT}) target_include_directories(python PRIVATE ${TOKENIZER_ROOT}) + target_link_libraries(onnxruntime-genai PRIVATE tokenizer) target_link_libraries(Tests PRIVATE tokenizer) target_link_libraries(python PRIVATE tokenizer) message("Linking Tokenizer with TOKENIZER_ROOT: ${TOKENIZER_ROOT}") @@ -162,16 +169,13 @@ endif() if(BUILD_WHEEL) set(WHEEL_FILES_DIR "${CMAKE_BINARY_DIR}/wheel") message("Setting up wheel files in : ${WHEEL_FILES_DIR}") - set(TARGET_NAME "onnxruntime_genai") - configure_file(${PYTHON_ROOT}/setup.py.in ${WHEEL_FILES_DIR}/setup.py @ONLY) configure_file(${PYTHON_ROOT}/py/__init__.py.in ${WHEEL_FILES_DIR}/${TARGET_NAME}/__init__.py @ONLY) - file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/${ONNXRUNTIME_FILES}") foreach(DLL_FILE ${onnxruntime_libs}) add_custom_command( - TARGET onnxruntime-genai + TARGET onnxruntime-genai-static COMMAND ${CMAKE_COMMAND} -E copy ${DLL_FILE} ${WHEEL_FILES_DIR}/${TARGET_NAME}/${DLL_FILE_NAME} diff --git a/src/tokenizer/CMakeLists.txt b/src/tokenizer/CMakeLists.txt index 184b55571..d439c5d9d 100644 --- a/src/tokenizer/CMakeLists.txt +++ b/src/tokenizer/CMakeLists.txt @@ -21,7 +21,7 @@ FetchContent_MakeAvailable(GSL) FetchContent_Declare(simdjson URL https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip URL_HASH SHA1=2b063a2e81f74a5d1cb937fadf3d2fca0f1edb09) FetchContent_MakeAvailable(simdjson) -add_library(tokenizer ${tokenizer_srcs}) +add_library(tokenizer SHARED ${tokenizer_srcs}) message(STATUS "GSL_SOURCE_DIR: ${GSL_SOURCE_DIR}") target_include_directories(tokenizer PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} From d8d7dd344ef788c0abe7ad0d6e4d61a44be76d43 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 16:17:20 -0800 Subject: [PATCH 05/21] Make tokenizer static --- src/tokenizer/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenizer/CMakeLists.txt b/src/tokenizer/CMakeLists.txt index d439c5d9d..184b55571 100644 --- a/src/tokenizer/CMakeLists.txt +++ b/src/tokenizer/CMakeLists.txt @@ -21,7 +21,7 @@ FetchContent_MakeAvailable(GSL) FetchContent_Declare(simdjson URL https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip URL_HASH SHA1=2b063a2e81f74a5d1cb937fadf3d2fca0f1edb09) FetchContent_MakeAvailable(simdjson) -add_library(tokenizer SHARED ${tokenizer_srcs}) +add_library(tokenizer ${tokenizer_srcs}) message(STATUS "GSL_SOURCE_DIR: ${GSL_SOURCE_DIR}") target_include_directories(tokenizer PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} From 7c32db7ddde76c056499f28fd2b826ca79368ebe Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 16:28:42 -0800 Subject: [PATCH 06/21] linking cuda on shared library --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 651757c33..168aa180c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,14 +121,15 @@ target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(python PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") +set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) +target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) + # TODO: Windows is statically linking CUDA libraries, so remove the 'NOT WIN32' once it's using shared libraries if(NOT WIN32 AND USE_CUDA AND CMAKE_CUDA_COMPILER) - set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) + target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) - target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) - target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) endif() From d43d31f2dd03482233146ce9c377d006041ca67b Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 16:33:30 -0800 Subject: [PATCH 07/21] linking cuda on shared library --- CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 168aa180c..6ecf8ab75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,11 +121,11 @@ target_link_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) target_link_libraries(python PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") -set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) -target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) - -# TODO: Windows is statically linking CUDA libraries, so remove the 'NOT WIN32' once it's using shared libraries -if(NOT WIN32 AND USE_CUDA AND CMAKE_CUDA_COMPILER) +if(USE_CUDA AND CMAKE_CUDA_COMPILER) + set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) + target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) + # onnxruntime-genai-static is statically linked under Windows + if(NOT WIN32) set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) @@ -199,8 +199,8 @@ if(BUILD_WHEEL) COMMENT "Building wheel" ) - add_dependencies(PyPackageBuild python) -endif () + add_dependencies(PyPackageBuild python) +endif() # Have visual studio put all files into one single folder vs the default split of header files into a separate folder source_group(TREE ${GENERATORS_ROOT} FILES ${generator_srcs}) From 4aa11d896800ae105c88259a18130850f347efc0 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 17:01:37 -0800 Subject: [PATCH 08/21] Nesting if --- CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ecf8ab75..186130de6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,15 +126,15 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) # onnxruntime-genai-static is statically linked under Windows if(NOT WIN32) - set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) - target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) - set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) - set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) - target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) - target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) + set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) + target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) + set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) + target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) + target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) + endif() endif() - # Visual C++ - /MP Multiple process build, /WX Warnings as errors # Enable: # C5038: data member 'member1' will be initialized after data member 'member2' From b15985a0455646df913a0e20af44c098dc361e6f Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 17:13:14 -0800 Subject: [PATCH 09/21] Link test with static libraries --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 186130de6..8624d84aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,7 @@ endif() add_executable(Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) -target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) +target_link_libraries(Tests PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) From bb47d7a8591ecb6523458642fdad22bc3ff1a9f6 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 17:19:14 -0800 Subject: [PATCH 10/21] set_target_properties( PROPERTIES LINKER_LANGUAGE CUDA) --- CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8624d84aa..18a6ac76e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,8 @@ endif() add_executable(Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) -target_link_libraries(Tests PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) +target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) + # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) @@ -123,13 +124,14 @@ set_target_properties(python PROPERTIES OUTPUT_NAME "onnxruntime_genai") if(USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(onnxruntime-genai PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) + set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) + # onnxruntime-genai-static is statically linked under Windows if(NOT WIN32) set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) - set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) - set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) endif() From 1d113ec8ba7d9aac7ab11b897aaaf5c6eb41725a Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 17:29:46 -0800 Subject: [PATCH 11/21] Link test with static libraries --- CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 18a6ac76e..83b9a03b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,7 @@ endif() add_executable(Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) -target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) +target_link_libraries(Tests PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) # pybind11_add_module(python ${python_srcs}) @@ -127,13 +127,12 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER) set_target_properties(Tests PROPERTIES LINKER_LANGUAGE CUDA) set_target_properties(python PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai PRIVATE cublasLt cublas cudnn curand cufft cudart) - + target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) + target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) # onnxruntime-genai-static is statically linked under Windows if(NOT WIN32) set_target_properties(onnxruntime-genai-static PROPERTIES LINKER_LANGUAGE CUDA) target_link_libraries(onnxruntime-genai-static PRIVATE cublasLt cublas cudnn curand cufft cudart) - target_link_libraries(Tests PRIVATE cublasLt cublas cudnn curand cufft cudart) - target_link_libraries(python PRIVATE cublasLt cublas cudnn curand cufft cudart) endif() endif() From 405c6610e42000d36c129a8230373f5a89758695 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 17:46:56 -0800 Subject: [PATCH 12/21] Remove cmake enale build share flag/ --- .github/workflows/linux-cpu-arm64-build.yml | 2 +- .github/workflows/linux-cpu-x64-build.yml | 4 ++-- .github/workflows/linux-gpu-x64-build.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux-cpu-arm64-build.yml b/.github/workflows/linux-cpu-arm64-build.yml index ba524b82b..50670c93c 100644 --- a/.github/workflows/linux-cpu-arm64-build.yml +++ b/.github/workflows/linux-cpu-arm64-build.yml @@ -41,7 +41,7 @@ jobs: --docker-build-args "--build-arg BUILD_UID=$( id -u )" \ --container-registry onnxruntimebuildcache \ --repository onnxruntimecpubuild - docker run --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimecpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release && ninja" + docker run --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src --volume $RUNNER_TEMP:/build -w /build onnxruntimecpubuild bash -c "echo $PATH && /usr/bin/cmake -G Ninja /onnxruntime_src -DCMAKE_BUILD_TYPE=Release && ninja" diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml index 6f0437f6b..9d35e975e 100644 --- a/.github/workflows/linux-cpu-x64-build.yml +++ b/.github/workflows/linux-cpu-x64-build.yml @@ -38,7 +38,7 @@ jobs: set -e -x mkdir build cd build - cmake -G "Unix Makefiles" .. -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release + cmake -G "Unix Makefiles" .. -DCMAKE_BUILD_TYPE=Release cmake --build . --config Release --parallel cd .. rm -rf build @@ -48,7 +48,7 @@ jobs: set -e -x mkdir build cd build - CC=clang CXX=clang++ cmake -G "Unix Makefiles" .. -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_CLANG_TIDY=clang-tidy + CC=clang CXX=clang++ cmake -G "Unix Makefiles" .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_CLANG_TIDY=clang-tidy cmake --build . --config Release --parallel diff --git a/.github/workflows/linux-gpu-x64-build.yml b/.github/workflows/linux-gpu-x64-build.yml index 4bde267a5..906bc4d07 100644 --- a/.github/workflows/linux-gpu-x64-build.yml +++ b/.github/workflows/linux-gpu-x64-build.yml @@ -55,5 +55,5 @@ jobs: --rm \ --volume $GITHUB_WORKSPACE:/onnxruntime_src \ --volume $RUNNER_TEMP:/build \ - -w /build onnxruntimegpubuild bash -c "/usr/bin/cmake -G Ninja /onnxruntime_src -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CUDA_ARCHITECTURES=75 -DUSE_CUDA=1 -DCMAKE_BUILD_TYPE=Release && ninja" + -w /build onnxruntimegpubuild bash -c "/usr/bin/cmake -G Ninja /onnxruntime_src -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_CUDA_ARCHITECTURES=75 -DUSE_CUDA=1 -DCMAKE_BUILD_TYPE=Release && ninja" From 7d94970a9a6cc725cedd9ae32c543a78fbb5aa74 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 18:08:55 -0800 Subject: [PATCH 13/21] Remove # target_link_libraries(Tests PRIVATE tokenizer) # target_link_libraries(python PRIVATE tokenizer) --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 83b9a03b6..9bf67a900 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,8 +163,7 @@ if(USE_TOKENIZER) target_include_directories(Tests PRIVATE ${TOKENIZER_ROOT}) target_include_directories(python PRIVATE ${TOKENIZER_ROOT}) target_link_libraries(onnxruntime-genai PRIVATE tokenizer) - target_link_libraries(Tests PRIVATE tokenizer) - target_link_libraries(python PRIVATE tokenizer) + target_link_libraries(onnxruntime-genai-static PRIVATE tokenizer) message("Linking Tokenizer with TOKENIZER_ROOT: ${TOKENIZER_ROOT}") endif() From 80bfa11f6b3e0a1aceefda54038d2441c7aa55ab Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 18:49:52 -0800 Subject: [PATCH 14/21] Set POSITION_INDEPENDENT_CODE on --- src/tokenizer/CMakeLists.txt | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/tokenizer/CMakeLists.txt b/src/tokenizer/CMakeLists.txt index 184b55571..616e2703f 100644 --- a/src/tokenizer/CMakeLists.txt +++ b/src/tokenizer/CMakeLists.txt @@ -1,8 +1,8 @@ set(TOKENIZER_ROOT ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE) file(GLOB tokenizer_srcs CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/*.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/utils/*.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/c_api/*.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/*.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/utils/*.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/c_api/*.cc" ) FetchContent_Declare(spm URL https://github.com/google/sentencepiece/archive/refs/tags/v0.1.99.zip) set(SPM_USE_EXTERNAL_ABSL OFF CACHE BOOL "Use external absl") @@ -10,10 +10,10 @@ set(SPM_USE_BUILTIN_PROTOBUF ON CACHE BOOL "Use built-in protobuf") set(SPM_ENABLE_SHARED OFF) FetchContent_GetProperties(spm) -if (NOT spm_POPULATED) - FetchContent_Populate(spm) - add_subdirectory(${spm_SOURCE_DIR} ${spm_BINARY_DIR} EXCLUDE_FROM_ALL) -endif () +if(NOT spm_POPULATED) + FetchContent_Populate(spm) + add_subdirectory(${spm_SOURCE_DIR} ${spm_BINARY_DIR} EXCLUDE_FROM_ALL) +endif() FetchContent_Declare(GSL URL https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip) FetchContent_MakeAvailable(GSL) @@ -24,15 +24,19 @@ FetchContent_MakeAvailable(simdjson) add_library(tokenizer ${tokenizer_srcs}) message(STATUS "GSL_SOURCE_DIR: ${GSL_SOURCE_DIR}") target_include_directories(tokenizer PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR} - ${GSL_SOURCE_DIR}/include - ${simdjson_SOURCE_DIR}/singleheader/ - ${spm_SOURCE_DIR}/src - ${spm_SOURCE_DIR}/src/builtin_pb - ${spm_SOURCE_DIR}/third_party/absl - ${spm_SOURCE_DIR}/third_party/protobuf-lite + ${CMAKE_CURRENT_SOURCE_DIR} + ${GSL_SOURCE_DIR}/include + ${simdjson_SOURCE_DIR}/singleheader/ + ${spm_SOURCE_DIR}/src + ${spm_SOURCE_DIR}/src/builtin_pb + ${spm_SOURCE_DIR}/third_party/absl + ${spm_SOURCE_DIR}/third_party/protobuf-lite ) target_compile_definitions(tokenizer PRIVATE _SILENCE_ALL_CXX20_DEPRECATION_WARNINGS) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set_target_properties(tokenizer PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() + add_dependencies(tokenizer sentencepiece-static) target_link_libraries(tokenizer PUBLIC sentencepiece-static simdjson) \ No newline at end of file From c67f642be516e34a5f30aeab2715ddac5f08a2f7 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 18:53:00 -0800 Subject: [PATCH 15/21] setting library --- CMakeLists.txt | 2 +- src/tokenizer/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bf67a900..e12bec0ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,7 +81,7 @@ endif() # set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) #endif() add_library(onnxruntime-genai SHARED ${generator_srcs}) -add_library(onnxruntime-genai-static ${generator_srcs}) +add_library(onnxruntime-genai-static STATIC ${generator_srcs}) target_include_directories(onnxruntime-genai PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_include_directories(onnxruntime-genai-static PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) diff --git a/src/tokenizer/CMakeLists.txt b/src/tokenizer/CMakeLists.txt index 616e2703f..e229b30cd 100644 --- a/src/tokenizer/CMakeLists.txt +++ b/src/tokenizer/CMakeLists.txt @@ -21,7 +21,7 @@ FetchContent_MakeAvailable(GSL) FetchContent_Declare(simdjson URL https://github.com/simdjson/simdjson/archive/refs/tags/v3.6.3.zip URL_HASH SHA1=2b063a2e81f74a5d1cb937fadf3d2fca0f1edb09) FetchContent_MakeAvailable(simdjson) -add_library(tokenizer ${tokenizer_srcs}) +add_library(tokenizer STATIC ${tokenizer_srcs}) message(STATUS "GSL_SOURCE_DIR: ${GSL_SOURCE_DIR}") target_include_directories(tokenizer PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} From 39055aba9e16b46d99569516f64ebda050d7414d Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 19:01:07 -0800 Subject: [PATCH 16/21] making simdjson POSITION_INDEPENDENT_CODE --- src/tokenizer/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tokenizer/CMakeLists.txt b/src/tokenizer/CMakeLists.txt index e229b30cd..d47aa912d 100644 --- a/src/tokenizer/CMakeLists.txt +++ b/src/tokenizer/CMakeLists.txt @@ -36,6 +36,7 @@ target_compile_definitions(tokenizer PRIVATE _SILENCE_ALL_CXX20_DEPRECATION_WARN if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set_target_properties(tokenizer PROPERTIES POSITION_INDEPENDENT_CODE ON) + set_target_properties(simdjson PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() add_dependencies(tokenizer sentencepiece-static) From b1f2568be36a42a74e80cd28ea8c509252533898 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 19:09:39 -0800 Subject: [PATCH 17/21] remove POSITION_INDEPENDENT_CODE from ort-gentai-static --- .github/workflows/linux-cpu-x64-build.yml | 3 ++- CMakeLists.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml index 9d35e975e..2dbf7fa6b 100644 --- a/.github/workflows/linux-cpu-x64-build.yml +++ b/.github/workflows/linux-cpu-x64-build.yml @@ -9,7 +9,8 @@ env: ort_url: "https://github.com/microsoft/onnxruntime/releases/download/v1.16.3/onnxruntime-linux-x64-1.16.3.tgz" jobs: job: - runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2204-AMD-CPU" ] +# runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2204-AMD-CPU" ] + runs-on: ubuntu-latest steps: - name: Checkout OnnxRuntime GenAI repo uses: actions/checkout@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index e12bec0ec..2e7f6ef3b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,7 +93,6 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set_target_properties(onnxruntime-genai PROPERTIES POSITION_INDEPENDENT_CODE ON) - set_target_properties(onnxruntime-genai-static PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() if(WIN32) From 9a6f2490b0f96b29710b32f49317b35d371de1a5 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 19:14:38 -0800 Subject: [PATCH 18/21] Merge with main --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f0644644..a359b3a0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,8 +147,8 @@ endif() file(GLOB onnxruntime_libs "${CMAKE_SOURCE_DIR}/ort/${ONNXRUNTIME_FILES}") foreach(DLL_FILE ${onnxruntime_libs}) add_custom_command( - TARGET onnxruntime-genai POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${DLL_FILE} ${CMAKE_BINARY_DIR}/$/${DLL_FILE_NAME} + TARGET onnxruntime-genai POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${DLL_FILE} ${CMAKE_BINARY_DIR}/$/${DLL_FILE_NAME} ) endforeach() From a5a34aaaf747c5335a48592034ccf0aab4460486 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 19:22:50 -0800 Subject: [PATCH 19/21] change the POSITION_INDEPENDENT_CODE to genai-static --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a359b3a0c..751ec78f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,7 +92,7 @@ if(USE_TOKENIZER) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - set_target_properties(onnxruntime-genai PROPERTIES POSITION_INDEPENDENT_CODE ON) + set_target_properties(onnxruntime-genai-static PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() if(WIN32) From 607730985d48d17a2a471bc3c2cef8cfdf6a214b Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 19:29:54 -0800 Subject: [PATCH 20/21] Final clean up --- .github/workflows/linux-cpu-x64-build.yml | 3 +-- CMakeLists.txt | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/linux-cpu-x64-build.yml b/.github/workflows/linux-cpu-x64-build.yml index 2dbf7fa6b..9d35e975e 100644 --- a/.github/workflows/linux-cpu-x64-build.yml +++ b/.github/workflows/linux-cpu-x64-build.yml @@ -9,8 +9,7 @@ env: ort_url: "https://github.com/microsoft/onnxruntime/releases/download/v1.16.3/onnxruntime-linux-x64-1.16.3.tgz" jobs: job: -# runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2204-AMD-CPU" ] - runs-on: ubuntu-latest + runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2204-AMD-CPU" ] steps: - name: Checkout OnnxRuntime GenAI repo uses: actions/checkout@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 751ec78f1..30292b8ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,10 +76,6 @@ else() list(REMOVE_ITEM generator_srcs ${generator_cuda_srcs}) endif() -## TODO: this is a hack to get the shared library to export .lib files on windows. We should find a better way when we need to compile dynamic libraries -#if(WIN32) -# set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) -#endif() add_library(onnxruntime-genai SHARED ${generator_srcs}) add_library(onnxruntime-genai-static STATIC ${generator_srcs}) target_include_directories(onnxruntime-genai PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) From 68a7af434b27b14ca1a8d57fed96fe66ec423a0c Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Fri, 2 Feb 2024 19:48:08 -0800 Subject: [PATCH 21/21] Tests needs static libraries too. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 30292b8ff..f5f059c72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -89,6 +89,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set_target_properties(onnxruntime-genai-static PROPERTIES POSITION_INDEPENDENT_CODE ON) + endif() if(WIN32) @@ -108,7 +109,7 @@ endif() add_executable(Tests ${test_srcs}) target_include_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/include) target_link_directories(Tests PRIVATE ${CMAKE_SOURCE_DIR}/ort/lib) -target_link_libraries(Tests PRIVATE onnxruntime-genai ${ONNXRUNTIME_LIB}) +target_link_libraries(Tests PRIVATE onnxruntime-genai-static ${ONNXRUNTIME_LIB}) # pybind11_add_module(python ${python_srcs}) target_include_directories(python PRIVATE ${CMAKE_SOURCE_DIR}/ort/include)