From 77514cb9ce3354a5d29b1680f389993b5bedbb35 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Thu, 25 May 2023 21:37:18 -0700 Subject: [PATCH 01/14] support static linking of ctranslate2 --- .gitmodules | 3 + crates/ctranslate2-bindings/.gitignore | 1 + crates/ctranslate2-bindings/CMakeLists.txt | 16 ++++ crates/ctranslate2-bindings/Cargo.toml | 1 + crates/ctranslate2-bindings/build.rs | 12 ++- crates/ctranslate2-bindings/build_cmake.sh | 12 +++ .../cmake/export_libs.cmake | 88 +++++++++++++++++++ crates/ctranslate2-bindings/src/dummy.cc | 0 crates/rust-cxx-cmake-bridge | 1 + crates/tabby/Cargo.lock | 5 ++ 10 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 crates/ctranslate2-bindings/CMakeLists.txt create mode 100755 crates/ctranslate2-bindings/build_cmake.sh create mode 100644 crates/ctranslate2-bindings/cmake/export_libs.cmake create mode 100644 crates/ctranslate2-bindings/src/dummy.cc create mode 160000 crates/rust-cxx-cmake-bridge diff --git a/.gitmodules b/.gitmodules index 17f7250cbe1f..d754dac8faf6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "crates/ctranslate2-bindings/CTranslate2"] path = crates/ctranslate2-bindings/CTranslate2 url = https://github.com/OpenNMT/CTranslate2.git +[submodule "crates/rust-cxx-cmake-bridge"] + path = crates/rust-cxx-cmake-bridge + url = https://github.com/Interstellar-Network/rust-cxx-cmake-bridge.git diff --git a/crates/ctranslate2-bindings/.gitignore b/crates/ctranslate2-bindings/.gitignore index 4fffb2f89cbd..ca58cabbc4d6 100644 --- a/crates/ctranslate2-bindings/.gitignore +++ b/crates/ctranslate2-bindings/.gitignore @@ -1,2 +1,3 @@ /target /Cargo.lock +/build diff --git a/crates/ctranslate2-bindings/CMakeLists.txt b/crates/ctranslate2-bindings/CMakeLists.txt new file mode 100644 index 000000000000..2c40151077f9 --- /dev/null +++ b/crates/ctranslate2-bindings/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.7) + +project(ctranslate2_bindings) + +add_subdirectory(CTranslate2) + +add_library(dummy + src/dummy.cc +) + +target_link_libraries(dummy + PRIVATE ctranslate2 +) + +include(cmake/export_libs.cmake) +export_all_target_libs(dummy) diff --git a/crates/ctranslate2-bindings/Cargo.toml b/crates/ctranslate2-bindings/Cargo.toml index 46f8b1544a27..91e01d606695 100644 --- a/crates/ctranslate2-bindings/Cargo.toml +++ b/crates/ctranslate2-bindings/Cargo.toml @@ -12,3 +12,4 @@ tokenizers = "0.13.3" bindgen = "0.53.1" cxx-build = "1.0" cmake = "0.1" +rust-cxx-cmake-bridge = { path = "../rust-cxx-cmake-bridge" } diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 916e1fd2153d..2d8fa26c0ea3 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -1,11 +1,12 @@ use cmake::Config; +use rust_cxx_cmake_bridge::read_cmake_generated; fn main() { - let mut config = Config::new("CTranslate2"); + let mut config = Config::new("."); config .define("CMAKE_BUILD_TYPE", "Release") .define("BUILD_CLI", "OFF") - .define("BUILD_SHARED_LIBS", "ON") + .define("BUILD_SHARED_LIBS", "OFF") .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON"); if cfg!(target_os = "macos") { @@ -29,11 +30,8 @@ fn main() { let dst = config.build(); - println!( - "cargo:rustc-link-search=native={}", - dst.join("lib").display() - ); - println!("cargo:rustc-link-lib=ctranslate2"); + let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); + read_cmake_generated(&cmake_generated_libs_str); // Tell cargo to invalidate the built crate whenever the wrapper changes println!("cargo:rerun-if-changed=include/ctranslate2.h"); diff --git a/crates/ctranslate2-bindings/build_cmake.sh b/crates/ctranslate2-bindings/build_cmake.sh new file mode 100755 index 000000000000..8757a64a7b65 --- /dev/null +++ b/crates/ctranslate2-bindings/build_cmake.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +set -e +set -x + +rm -rf build +mkdir build && cd build + +CMAKE_EXTRA_OPTIONS='-DCMAKE_OSX_ARCHITECTURES=arm64 -DWITH_ACCELERATE=ON -DWITH_MKL=OFF -DOPENMP_RUNTIME=NONE -DWITH_RUY=ON' +cmake -DBULID_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_CLI=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON $CMAKE_EXTRA_OPTIONS .. + +make -j4 diff --git a/crates/ctranslate2-bindings/cmake/export_libs.cmake b/crates/ctranslate2-bindings/cmake/export_libs.cmake new file mode 100644 index 000000000000..1c9787fee3ac --- /dev/null +++ b/crates/ctranslate2-bindings/cmake/export_libs.cmake @@ -0,0 +1,88 @@ +################################################################################ + +# WARNING: to list the system libraries(ie IMPORTED) you MUST set: +# set_target_properties(your_lib PROPERTIES IMPORTED_GLOBAL TRUE) +# just after the find_package call +# cf https://gitlab.kitware.com/cmake/cmake/-/issues/17256 +# +# https://stackoverflow.com/questions/32756195/recursive-list-of-link-libraries-in-cmake +# https://stackoverflow.com/questions/32197663/how-can-i-remove-the-the-location-property-may-not-be-read-from-target-error-i +function(_get_link_libraries OUTPUT_LIST TARGET) + list(APPEND VISITED_TARGETS ${TARGET}) + + # DO NOT switch on IMPORTED or not + # An INTERFACE library CAN have LINK_LIBRARIES! + # get_target_property(IMPORTED ${TARGET} IMPORTED) + set(LIBS "") + get_target_property(LIBS_1 ${TARGET} INTERFACE_LINK_LIBRARIES) + get_target_property(LIBS_2 ${TARGET} LINK_LIBRARIES) + list(APPEND LIBS ${LIBS_1} ${LIBS_2}) + + set(LIB_FILES "") + + foreach(LIB ${LIBS}) + if (TARGET ${LIB}) + list(FIND VISITED_TARGETS ${LIB} VISITED) + if (${VISITED} EQUAL -1) + # OLD: get_target_property(LIB_FILE ${LIB} LOCATION) + # NEW: + _get_link_libraries(LINK_LIB_FILES ${LIB}) + set(LIB_FILE ${LIB}) + list(APPEND LIB_FILES ${LINK_LIB_FILES}) + list(APPEND LIB_FILES ${LIB_FILE}) + endif() + endif() + endforeach() + + set(VISITED_TARGETS ${VISITED_TARGETS} PARENT_SCOPE) + set(${OUTPUT_LIST} ${LIB_FILES} PARENT_SCOPE) +endfunction() + +################################################################################ + +function(export_all_target_libs TARGET) + # NOTE: get_target_property(CIRCUIT_LIB_LINK_LIBRARIES a_target LINK_LIBRARIES) is NOT transitive + # This function will return eg: "$;$;" + # b/c generator expression are evaluated LATER + # cf https://stackoverflow.com/questions/59226127/cmake-generator-expression-how-to-get-target-file-property-on-list-of-targets + set(ALL_LINK_LIBRARIES "") + _get_link_libraries(ALL_LINK_LIBRARIES ${TARGET}) + + message(STATUS "ALL_LINK_LIBRARIES : ${ALL_LINK_LIBRARIES}") + + set(ALL_LIBS "") + # TODO move that back into get_link_libraries + # NOTE: we MUST do it in 2 steps: + # - collect all the LINK_LIBRARIES recursively + # - loop on those and get their TARGET_FILE (if not INTERFACE_LIBRARY) + # That is b/c in get_link_libraries a INTERFACE_LIBRARY CAN have link_libraries + # but we CAN NOT evaluate generator expressions at this time. + foreach(LIB ${ALL_LINK_LIBRARIES}) + # MUST skip INTERFACE else: + # CMake Error at src/CMakeLists.txt:136 (add_custom_command): + # Error evaluating generator expression: + # $ + # Target "rust_cxx" is not an executable or library. + # SHARED_LIBRARY,INTERFACE_LIBRARY,STATIC_LIBRARY + # + get_target_property(LIB_TYPE ${LIB} TYPE) + message(STATUS "LIB_TYPE : ${LIB} = ${LIB_TYPE}") + + if(NOT ${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") + set(LIB_FILE $) + list(APPEND ALL_LIBS ${LIB_FILE}) + endif() + endforeach() # LIB ${ALL_LIBS} + + message(STATUS "ALL_LIBS : ${ALL_LIBS}") + + # add_custom_command(ie echoing only to stdout) works but more difficult to get from build.rs + # b/c when there is "ninja: no work to do" it will NOT echo on the console + add_custom_command( + TARGET ${TARGET} + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo ${ALL_LIBS} > ${CMAKE_CURRENT_BINARY_DIR}/cmake_generated_libs + # OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cmake_generated_libs + VERBATIM + ) +endfunction(export_all_target_libs) diff --git a/crates/ctranslate2-bindings/src/dummy.cc b/crates/ctranslate2-bindings/src/dummy.cc new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/crates/rust-cxx-cmake-bridge b/crates/rust-cxx-cmake-bridge new file mode 160000 index 000000000000..2236064d88a1 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge @@ -0,0 +1 @@ +Subproject commit 2236064d88a1e72897f77b2ecbeecab3402c4c6d diff --git a/crates/tabby/Cargo.lock b/crates/tabby/Cargo.lock index 2d80655a7869..4bfae11af128 100644 --- a/crates/tabby/Cargo.lock +++ b/crates/tabby/Cargo.lock @@ -538,6 +538,7 @@ dependencies = [ "cxx", "cxx-build", "derive_builder", + "rust-cxx-cmake-bridge", "tokenizers", ] @@ -1804,6 +1805,10 @@ dependencies = [ "winreg", ] +[[package]] +name = "rust-cxx-cmake-bridge" +version = "0.1.0" + [[package]] name = "rust-embed" version = "6.6.1" From 59f0a22caf45ea98e2ae8d24975d155ab6ffcb9e Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Thu, 25 May 2023 21:40:23 -0700 Subject: [PATCH 02/14] update --- .../ctranslate2-bindings/{build_cmake.sh => cmake/debug_cmake.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename crates/ctranslate2-bindings/{build_cmake.sh => cmake/debug_cmake.sh} (100%) diff --git a/crates/ctranslate2-bindings/build_cmake.sh b/crates/ctranslate2-bindings/cmake/debug_cmake.sh similarity index 100% rename from crates/ctranslate2-bindings/build_cmake.sh rename to crates/ctranslate2-bindings/cmake/debug_cmake.sh From 3dc934011e0e8233d877550269573e1510d271e5 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 12:31:37 -0700 Subject: [PATCH 03/14] remove submodule rust-cxx-cmake-bridge --- crates/ctranslate2-bindings/build.rs | 2 ++ crates/rust-cxx-cmake-bridge | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 160000 crates/rust-cxx-cmake-bridge diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 2d8fa26c0ea3..31cd6e0e54fc 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -16,6 +16,8 @@ fn main() { .define("WITH_MKL", "OFF") .define("OPENMP_RUNTIME", "NONE") .define("WITH_RUY", "ON"); + + println!("cargo:rustc-link-lib=framework=Accelerate") } else if cfg!(target_os = "linux") { config .define("WITH_CUDA", "ON") diff --git a/crates/rust-cxx-cmake-bridge b/crates/rust-cxx-cmake-bridge deleted file mode 160000 index 2236064d88a1..000000000000 --- a/crates/rust-cxx-cmake-bridge +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2236064d88a1e72897f77b2ecbeecab3402c4c6d From 5f66ca193a5a8c7c8cbd305990300288ffd952a8 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 12:41:25 -0700 Subject: [PATCH 04/14] support alwayslink with whole-archive --- crates/ctranslate2-bindings/build.rs | 2 +- crates/rust-cxx-cmake-bridge/.gitignore | 1 + crates/rust-cxx-cmake-bridge/Cargo.lock | 7 + crates/rust-cxx-cmake-bridge/Cargo.toml | 8 + crates/rust-cxx-cmake-bridge/README.md | 1 + crates/rust-cxx-cmake-bridge/src/lib.rs | 243 ++++++++++++++++++++++++ 6 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 crates/rust-cxx-cmake-bridge/.gitignore create mode 100644 crates/rust-cxx-cmake-bridge/Cargo.lock create mode 100644 crates/rust-cxx-cmake-bridge/Cargo.toml create mode 100644 crates/rust-cxx-cmake-bridge/README.md create mode 100644 crates/rust-cxx-cmake-bridge/src/lib.rs diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 31cd6e0e54fc..66531080a1fa 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -33,7 +33,7 @@ fn main() { let dst = config.build(); let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); - read_cmake_generated(&cmake_generated_libs_str); + read_cmake_generated(&cmake_generated_libs_str, true); // Tell cargo to invalidate the built crate whenever the wrapper changes println!("cargo:rerun-if-changed=include/ctranslate2.h"); diff --git a/crates/rust-cxx-cmake-bridge/.gitignore b/crates/rust-cxx-cmake-bridge/.gitignore new file mode 100644 index 000000000000..eb5a316cbd19 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/.gitignore @@ -0,0 +1 @@ +target diff --git a/crates/rust-cxx-cmake-bridge/Cargo.lock b/crates/rust-cxx-cmake-bridge/Cargo.lock new file mode 100644 index 000000000000..40374196acab --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "rust-cxx-cmake-bridge" +version = "0.1.0" diff --git a/crates/rust-cxx-cmake-bridge/Cargo.toml b/crates/rust-cxx-cmake-bridge/Cargo.toml new file mode 100644 index 000000000000..c0edcbee1975 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rust-cxx-cmake-bridge" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/rust-cxx-cmake-bridge/README.md b/crates/rust-cxx-cmake-bridge/README.md new file mode 100644 index 000000000000..e176a02275c5 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/README.md @@ -0,0 +1 @@ +Forked from https://github.com/Interstellar-Network/rust-cxx-cmake-bridge diff --git a/crates/rust-cxx-cmake-bridge/src/lib.rs b/crates/rust-cxx-cmake-bridge/src/lib.rs new file mode 100644 index 000000000000..ac027e0729f3 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/src/lib.rs @@ -0,0 +1,243 @@ +use std::io::Write; +use std::path::PathBuf; + +// ## lib_name: "protobufd" +// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/protobuf_fetch-build" +// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/protobuf_fetch-build +// rustc-link-lib=protobufd +// ## lib_name: "libyosys" +// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/yosys_fetch-build" +// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/yosys_fetch-build +// rustc-link-lib=libyosys +// ## lib_name: "xxhash" +// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/xxhash-build" +// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/xxhash-build +// rustc-link-lib=xxhash +fn parse_lib_path_dir_and_name(static_lib_str: &str) -> (PathBuf, String, bool, bool) { + let static_lib_path = std::path::Path::new(static_lib_str); + + // NOTE: file_stem only split eg "libprotobufd.so.3.19.4.0" -> "libprotobufd.so.3.19.4" + // but that is NOT what we want (ie "libprotobufd") + // TODO use "file_prefix" https://github.com/rust-lang/rust/issues/86319 + let liblib_name = static_lib_path.my_file_prefix().unwrap(); + let liblib_name_str: String = liblib_name.to_str().unwrap().into(); + let lib_name_str = liblib_name_str.strip_prefix("lib").unwrap(); + + // basically: + // - input = /.../target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/glog-build/libglogd.so.0.6.0 + // - get the extension: a (or "so.3.19.4" or "so" etc) + // NOTE: extension DOES NOT work(same issue than file_stem) + // eg ".../libglogd.so.0.6.0".extension() == "0" (ie the part after the last dot) + // and we NEED "so" (ie the part after the FIRST dot) + let file_with_ext = static_lib_path.file_name().unwrap(); + let full_ext = file_with_ext + .to_str() + .unwrap() + .trim_start_matches(&liblib_name_str); + let is_static = full_ext.starts_with(".a"); + + let dir = static_lib_path.parent().unwrap(); + + // COULD probably have a more foolproof system by using the IMPORTED property in CMake + // and writing that to a different file(or the same one with a prefix/suffix?) + // NOTE: be sure that the prefix does not conflict with the Dockerfile WORKDIR /usr/src/app + let is_system = dir.starts_with("/usr/lib/"); + + return ( + dir.to_path_buf(), + lib_name_str.to_string(), + is_static, + is_system, + ); +} + +// Parse the content of "cmake_generated_rust_wrapper_libs" which SHOULD have +// been generated by our CMake function. +// It is expected to contain a list of space separated libraries eg: +// "/full/path/build/liblib1.so /full/path/build/liblib2.a /usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0" +// etc +fn read_cmake_generated_to_output( + cmake_generated_rust_wrapper_libs_str: &str, + output: &mut impl Write, + is_alwayslink: bool +) { + // Previous version was globing all .a and .so in the build dir but it only worked for SHARED dependencies. + // That is b/c when linking STATIC libs order matters! So we must get a proper list from CMake. + for static_lib_str in cmake_generated_rust_wrapper_libs_str + .split(&[' ', '\n'][..]) + .filter(|&x| !x.is_empty()) + { + let (dir, lib_name_str, is_static, is_system) = parse_lib_path_dir_and_name(static_lib_str); + // WARNING: we MUST add to the linker path: + // - NON system libs (obviously) wether SHARED or STATIC + // - system STATIC libs eg /usr/lib/x86_64-linux-gnu/libboost_filesystem.a else + // "error: could not find native static library `boost_filesystem`, perhaps an -L flag is missing?" + if !is_system || is_static { + writeln!(output, "cargo:rustc-link-search=native={}", dir.display()).unwrap(); + } + + writeln!( + output, + "cargo:rustc-link-lib={}{}={}", + if is_static { "static" } else { "dylib" }, + if is_alwayslink { ":+whole-archive,-bundle" } else { "" }, + lib_name_str + ) + .unwrap(); + } +} + +pub fn read_cmake_generated(cmake_generated_rust_wrapper_libs_str: &str, alwayslink: bool) { + read_cmake_generated_to_output( + &cmake_generated_rust_wrapper_libs_str, + &mut std::io::stdout(), + alwayslink + ) +} + +//////////////////////////////////////////////////////////////////////////////// +/// TEMP +/// Implement "file_prefix" +/// copy pasted from https://github.com/rust-lang/rust/issues/86319 +use std::ffi::OsStr; + +trait HasMyFilePrefix { + fn my_file_prefix(&self) -> Option<&OsStr>; +} + +impl HasMyFilePrefix for std::path::Path { + fn my_file_prefix(&self) -> Option<&OsStr> { + self.file_name() + .map(split_file_at_dot) + .and_then(|(before, _after)| Some(before)) + } +} + +fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) { + let slice = os_str_as_u8_slice(file); + if slice == b".." { + return (file, None); + } + + // The unsafety here stems from converting between &OsStr and &[u8] + // and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced + // only from ASCII-bounded slices of existing &OsStr values. + let i = match slice[1..].iter().position(|b| *b == b'.') { + Some(i) => i + 1, + None => return (file, None), + }; + let before = &slice[..i]; + let after = &slice[i + 1..]; + unsafe { (u8_slice_as_os_str(before), Some(u8_slice_as_os_str(after))) } +} + +fn os_str_as_u8_slice(s: &OsStr) -> &[u8] { + unsafe { &*(s as *const OsStr as *const [u8]) } +} +unsafe fn u8_slice_as_os_str(s: &[u8]) -> &OsStr { + // SAFETY: see the comment of `os_str_as_u8_slice` + { + &*(s as *const [u8] as *const OsStr) + } +} + +//////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod tests { + use crate::{parse_lib_path_dir_and_name, read_cmake_generated_to_output}; + + #[test] + fn parse_local_lib_static_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/some/path/liblibstatic.a"); + assert_eq!(dir.as_os_str(), "/some/path"); + assert_eq!(lib_name_str, "libstatic"); + assert_eq!(is_static, true); + assert_eq!(is_system, false); + } + + #[test] + fn parse_local_lib_shared_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/some/path/liblibshared.so"); + assert_eq!(dir.as_os_str(), "/some/path"); + assert_eq!(lib_name_str, "libshared"); + assert_eq!(is_static, false); + assert_eq!(is_system, false); + } + + #[test] + fn parse_local_lib_shared_with_soversion_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/some/path/liblibshared.so.1.2.3"); + assert_eq!(dir.as_os_str(), "/some/path"); + assert_eq!(lib_name_str, "libshared"); + assert_eq!(is_static, false); + assert_eq!(is_system, false); + } + + #[test] + fn parse_system_lib_static_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/usr/lib/libsystem1.a"); + assert_eq!(dir.as_os_str(), "/usr/lib"); + assert_eq!(lib_name_str, "system1"); + assert_eq!(is_static, true); + assert_eq!(is_system, true); + } + + #[test] + fn parse_system_lib_shared_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/usr/lib/libsystem2.so"); + assert_eq!(dir.as_os_str(), "/usr/lib"); + assert_eq!(lib_name_str, "system2"); + assert_eq!(is_static, false); + assert_eq!(is_system, true); + } + + #[test] + fn test_read_cmake_generated_to_output() { + let input = "/some/libA.a /some/libB.so"; + let mut stdout = Vec::new(); + read_cmake_generated_to_output(input, &mut stdout, false); + + assert_eq!( + std::str::from_utf8(&stdout).unwrap(), + "cargo:rustc-link-search=native=/some\n\ + cargo:rustc-link-lib=static=A\n\ + cargo:rustc-link-search=native=/some\n\ + cargo:rustc-link-lib=dylib=B\n" + ); + } + + // no need to touch "rustc-link-search" to link with eg "/usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0" + // simply "cargo:rustc-link-lib=dylib=png16.so" is OK + #[test] + fn test_read_cmake_generated_to_output_system_shared_no_rustc_link_search() { + let input = "/usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0"; + let mut stdout = Vec::new(); + read_cmake_generated_to_output(input, &mut stdout, false); + + assert_eq!( + std::str::from_utf8(&stdout).unwrap(), + "cargo:rustc-link-lib=dylib=png16\n" + ); + } + + // BUT system STATIC libs require "rustc-link-search"?? + #[test] + fn test_read_cmake_generated_to_output_system_static_rustc_link_search() { + let input = "/usr/lib/x86_64-linux-gnu/libpng16.a"; + let mut stdout = Vec::new(); + read_cmake_generated_to_output(input, &mut stdout, false); + + assert_eq!( + std::str::from_utf8(&stdout).unwrap(), + "cargo:rustc-link-search=native=/usr/lib/x86_64-linux-gnu\n\ + cargo:rustc-link-lib=static=png16\n" + ); + } +} From 26b445252557cc910292f818f97be1f3d4073dc9 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 12:52:50 -0700 Subject: [PATCH 05/14] update --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index d754dac8faf6..17f7250cbe1f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "crates/ctranslate2-bindings/CTranslate2"] path = crates/ctranslate2-bindings/CTranslate2 url = https://github.com/OpenNMT/CTranslate2.git -[submodule "crates/rust-cxx-cmake-bridge"] - path = crates/rust-cxx-cmake-bridge - url = https://github.com/Interstellar-Network/rust-cxx-cmake-bridge.git From 936ff81d4cefb83cdd5980945a7d146d95521ff6 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 12:58:14 -0700 Subject: [PATCH 06/14] move export_libs --- crates/ctranslate2-bindings/CMakeLists.txt | 2 +- .../cmake/export_libs.cmake | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename crates/{ctranslate2-bindings => rust-cxx-cmake-bridge}/cmake/export_libs.cmake (100%) diff --git a/crates/ctranslate2-bindings/CMakeLists.txt b/crates/ctranslate2-bindings/CMakeLists.txt index 2c40151077f9..aa8e420b222f 100644 --- a/crates/ctranslate2-bindings/CMakeLists.txt +++ b/crates/ctranslate2-bindings/CMakeLists.txt @@ -12,5 +12,5 @@ target_link_libraries(dummy PRIVATE ctranslate2 ) -include(cmake/export_libs.cmake) +include(../rust-cxx-cmake-bridge/cmake/export_libs.cmake) export_all_target_libs(dummy) diff --git a/crates/ctranslate2-bindings/cmake/export_libs.cmake b/crates/rust-cxx-cmake-bridge/cmake/export_libs.cmake similarity index 100% rename from crates/ctranslate2-bindings/cmake/export_libs.cmake rename to crates/rust-cxx-cmake-bridge/cmake/export_libs.cmake From ead23696f1cbbbff9db92e82233a7b96459b0657 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 13:00:34 -0700 Subject: [PATCH 07/14] update docker config --- Dockerfile.rust | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile.rust b/Dockerfile.rust index 22dd136a4b7b..2501cb6c76b3 100644 --- a/Dockerfile.rust +++ b/Dockerfile.rust @@ -48,14 +48,12 @@ COPY crates crates WORKDIR /root/crates/tabby RUN mkdir -p /opt/tabby/bin -RUN mkdir -p /opt/tabby/lib RUN mkdir -p target RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/root/crates/tabby/target \ cargo build --release && \ - cp target/release/tabby /opt/tabby/bin/ && \ - cp $(dirname $(find target | grep lib/libctranslate2 | head -1))/libctranslate2* /opt/tabby/lib + cp target/release/tabby /opt/tabby/bin/ FROM nvidia/cuda:11.2.2-base-ubuntu20.04 From c9aee0f46bcd3dea3f26e9e32815fd46a2393718 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 17:41:28 -0700 Subject: [PATCH 08/14] update ctranslate2 --- crates/ctranslate2-bindings/CTranslate2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ctranslate2-bindings/CTranslate2 b/crates/ctranslate2-bindings/CTranslate2 index 692fb607ab67..45af5ebcb643 160000 --- a/crates/ctranslate2-bindings/CTranslate2 +++ b/crates/ctranslate2-bindings/CTranslate2 @@ -1 +1 @@ -Subproject commit 692fb607ab67573fa5cf6e410aec24e8655844f8 +Subproject commit 45af5ebcb643f205a6709e0bf6c09157d1ecba52 From 77711a8b2a29547329ab922419682a9cb9e52257 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 18:27:23 -0700 Subject: [PATCH 09/14] remove --- Dockerfile.rust | 4 +- crates/ctranslate2-bindings/build.rs | 76 +++--- crates/rust-cxx-cmake-bridge/.gitignore | 1 - crates/rust-cxx-cmake-bridge/Cargo.lock | 7 - crates/rust-cxx-cmake-bridge/Cargo.toml | 8 - crates/rust-cxx-cmake-bridge/README.md | 1 - .../cmake/export_libs.cmake | 88 ------- crates/rust-cxx-cmake-bridge/src/lib.rs | 243 ------------------ 8 files changed, 48 insertions(+), 380 deletions(-) delete mode 100644 crates/rust-cxx-cmake-bridge/.gitignore delete mode 100644 crates/rust-cxx-cmake-bridge/Cargo.lock delete mode 100644 crates/rust-cxx-cmake-bridge/Cargo.toml delete mode 100644 crates/rust-cxx-cmake-bridge/README.md delete mode 100644 crates/rust-cxx-cmake-bridge/cmake/export_libs.cmake delete mode 100644 crates/rust-cxx-cmake-bridge/src/lib.rs diff --git a/Dockerfile.rust b/Dockerfile.rust index 2501cb6c76b3..22dd136a4b7b 100644 --- a/Dockerfile.rust +++ b/Dockerfile.rust @@ -48,12 +48,14 @@ COPY crates crates WORKDIR /root/crates/tabby RUN mkdir -p /opt/tabby/bin +RUN mkdir -p /opt/tabby/lib RUN mkdir -p target RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/root/crates/tabby/target \ cargo build --release && \ - cp target/release/tabby /opt/tabby/bin/ + cp target/release/tabby /opt/tabby/bin/ && \ + cp $(dirname $(find target | grep lib/libctranslate2 | head -1))/libctranslate2* /opt/tabby/lib FROM nvidia/cuda:11.2.2-base-ubuntu20.04 diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 66531080a1fa..906de02f18d1 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -1,39 +1,15 @@ +use std::path::PathBuf; use cmake::Config; use rust_cxx_cmake_bridge::read_cmake_generated; fn main() { - let mut config = Config::new("."); - config - .define("CMAKE_BUILD_TYPE", "Release") - .define("BUILD_CLI", "OFF") - .define("BUILD_SHARED_LIBS", "OFF") - .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON"); - - if cfg!(target_os = "macos") { - config - .define("CMAKE_OSX_ARCHITECTURES", "arm64") - .define("WITH_ACCELERATE", "ON") - .define("WITH_MKL", "OFF") - .define("OPENMP_RUNTIME", "NONE") - .define("WITH_RUY", "ON"); - - println!("cargo:rustc-link-lib=framework=Accelerate") + let dst = if cfg!(target_os = "macos") { + ctranslate2_build_macos_static() } else if cfg!(target_os = "linux") { - config - .define("WITH_CUDA", "ON") - .define("WITH_CUDNN", "ON") - .define("WITH_MKL", "ON") - .define("WITH_DNNL", "ON") - .define("OPENMP_RUNTIME", "COMP") - .cxxflag("-msse4.1") - .define("CUDA_NVCC_FLAGS", "-Xfatbin=-compress-all") - .define("CUDA_ARCH_LIST", "Common"); - } - - let dst = config.build(); - - let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); - read_cmake_generated(&cmake_generated_libs_str, true); + ctranslate2_build_linux_shared() + } else { + panic!("Invalid target") + }; // Tell cargo to invalidate the built crate whenever the wrapper changes println!("cargo:rerun-if-changed=include/ctranslate2.h"); @@ -46,3 +22,41 @@ fn main() { .flag_if_supported(&format!("-I{}", dst.join("include").display())) .compile("cxxbridge"); } + +fn ctranslate2_build_linux_shared() -> PathBuf { + Config::new("CTranslate2") + .define("BUILD_CLI", "OFF") + .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") + + .define("BUILD_CLI", "OFF") + .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") + .define("WITH_CUDA", "ON") + .define("WITH_CUDNN", "ON") + .define("WITH_MKL", "ON") + .define("WITH_DNNL", "ON") + .define("OPENMP_RUNTIME", "COMP") + .cxxflag("-msse4.1") + .define("CUDA_NVCC_FLAGS", "-Xfatbin=-compress-all") + .define("CUDA_ARCH_LIST", "Common") + .build() +} + +fn ctranslate2_build_macos_static() -> PathBuf { + let dst = Config::new(".") + .define("BUILD_CLI", "OFF") + .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") + .define("BUILD_SHARED_LIBS", "OFF") + + .define("CMAKE_OSX_ARCHITECTURES", "arm64") + .define("WITH_ACCELERATE", "ON") + .define("WITH_MKL", "OFF") + .define("OPENMP_RUNTIME", "NONE") + .define("WITH_RUY", "ON") + .build(); + + let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); + read_cmake_generated(&cmake_generated_libs_str); + println!("cargo:rustc-link-lib=framework=Accelerate"); + + return dst; +} diff --git a/crates/rust-cxx-cmake-bridge/.gitignore b/crates/rust-cxx-cmake-bridge/.gitignore deleted file mode 100644 index eb5a316cbd19..000000000000 --- a/crates/rust-cxx-cmake-bridge/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target diff --git a/crates/rust-cxx-cmake-bridge/Cargo.lock b/crates/rust-cxx-cmake-bridge/Cargo.lock deleted file mode 100644 index 40374196acab..000000000000 --- a/crates/rust-cxx-cmake-bridge/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "rust-cxx-cmake-bridge" -version = "0.1.0" diff --git a/crates/rust-cxx-cmake-bridge/Cargo.toml b/crates/rust-cxx-cmake-bridge/Cargo.toml deleted file mode 100644 index c0edcbee1975..000000000000 --- a/crates/rust-cxx-cmake-bridge/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "rust-cxx-cmake-bridge" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] diff --git a/crates/rust-cxx-cmake-bridge/README.md b/crates/rust-cxx-cmake-bridge/README.md deleted file mode 100644 index e176a02275c5..000000000000 --- a/crates/rust-cxx-cmake-bridge/README.md +++ /dev/null @@ -1 +0,0 @@ -Forked from https://github.com/Interstellar-Network/rust-cxx-cmake-bridge diff --git a/crates/rust-cxx-cmake-bridge/cmake/export_libs.cmake b/crates/rust-cxx-cmake-bridge/cmake/export_libs.cmake deleted file mode 100644 index 1c9787fee3ac..000000000000 --- a/crates/rust-cxx-cmake-bridge/cmake/export_libs.cmake +++ /dev/null @@ -1,88 +0,0 @@ -################################################################################ - -# WARNING: to list the system libraries(ie IMPORTED) you MUST set: -# set_target_properties(your_lib PROPERTIES IMPORTED_GLOBAL TRUE) -# just after the find_package call -# cf https://gitlab.kitware.com/cmake/cmake/-/issues/17256 -# -# https://stackoverflow.com/questions/32756195/recursive-list-of-link-libraries-in-cmake -# https://stackoverflow.com/questions/32197663/how-can-i-remove-the-the-location-property-may-not-be-read-from-target-error-i -function(_get_link_libraries OUTPUT_LIST TARGET) - list(APPEND VISITED_TARGETS ${TARGET}) - - # DO NOT switch on IMPORTED or not - # An INTERFACE library CAN have LINK_LIBRARIES! - # get_target_property(IMPORTED ${TARGET} IMPORTED) - set(LIBS "") - get_target_property(LIBS_1 ${TARGET} INTERFACE_LINK_LIBRARIES) - get_target_property(LIBS_2 ${TARGET} LINK_LIBRARIES) - list(APPEND LIBS ${LIBS_1} ${LIBS_2}) - - set(LIB_FILES "") - - foreach(LIB ${LIBS}) - if (TARGET ${LIB}) - list(FIND VISITED_TARGETS ${LIB} VISITED) - if (${VISITED} EQUAL -1) - # OLD: get_target_property(LIB_FILE ${LIB} LOCATION) - # NEW: - _get_link_libraries(LINK_LIB_FILES ${LIB}) - set(LIB_FILE ${LIB}) - list(APPEND LIB_FILES ${LINK_LIB_FILES}) - list(APPEND LIB_FILES ${LIB_FILE}) - endif() - endif() - endforeach() - - set(VISITED_TARGETS ${VISITED_TARGETS} PARENT_SCOPE) - set(${OUTPUT_LIST} ${LIB_FILES} PARENT_SCOPE) -endfunction() - -################################################################################ - -function(export_all_target_libs TARGET) - # NOTE: get_target_property(CIRCUIT_LIB_LINK_LIBRARIES a_target LINK_LIBRARIES) is NOT transitive - # This function will return eg: "$;$;" - # b/c generator expression are evaluated LATER - # cf https://stackoverflow.com/questions/59226127/cmake-generator-expression-how-to-get-target-file-property-on-list-of-targets - set(ALL_LINK_LIBRARIES "") - _get_link_libraries(ALL_LINK_LIBRARIES ${TARGET}) - - message(STATUS "ALL_LINK_LIBRARIES : ${ALL_LINK_LIBRARIES}") - - set(ALL_LIBS "") - # TODO move that back into get_link_libraries - # NOTE: we MUST do it in 2 steps: - # - collect all the LINK_LIBRARIES recursively - # - loop on those and get their TARGET_FILE (if not INTERFACE_LIBRARY) - # That is b/c in get_link_libraries a INTERFACE_LIBRARY CAN have link_libraries - # but we CAN NOT evaluate generator expressions at this time. - foreach(LIB ${ALL_LINK_LIBRARIES}) - # MUST skip INTERFACE else: - # CMake Error at src/CMakeLists.txt:136 (add_custom_command): - # Error evaluating generator expression: - # $ - # Target "rust_cxx" is not an executable or library. - # SHARED_LIBRARY,INTERFACE_LIBRARY,STATIC_LIBRARY - # - get_target_property(LIB_TYPE ${LIB} TYPE) - message(STATUS "LIB_TYPE : ${LIB} = ${LIB_TYPE}") - - if(NOT ${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") - set(LIB_FILE $) - list(APPEND ALL_LIBS ${LIB_FILE}) - endif() - endforeach() # LIB ${ALL_LIBS} - - message(STATUS "ALL_LIBS : ${ALL_LIBS}") - - # add_custom_command(ie echoing only to stdout) works but more difficult to get from build.rs - # b/c when there is "ninja: no work to do" it will NOT echo on the console - add_custom_command( - TARGET ${TARGET} - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E echo ${ALL_LIBS} > ${CMAKE_CURRENT_BINARY_DIR}/cmake_generated_libs - # OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cmake_generated_libs - VERBATIM - ) -endfunction(export_all_target_libs) diff --git a/crates/rust-cxx-cmake-bridge/src/lib.rs b/crates/rust-cxx-cmake-bridge/src/lib.rs deleted file mode 100644 index ac027e0729f3..000000000000 --- a/crates/rust-cxx-cmake-bridge/src/lib.rs +++ /dev/null @@ -1,243 +0,0 @@ -use std::io::Write; -use std::path::PathBuf; - -// ## lib_name: "protobufd" -// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/protobuf_fetch-build" -// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/protobuf_fetch-build -// rustc-link-lib=protobufd -// ## lib_name: "libyosys" -// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/yosys_fetch-build" -// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/yosys_fetch-build -// rustc-link-lib=libyosys -// ## lib_name: "xxhash" -// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/xxhash-build" -// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/xxhash-build -// rustc-link-lib=xxhash -fn parse_lib_path_dir_and_name(static_lib_str: &str) -> (PathBuf, String, bool, bool) { - let static_lib_path = std::path::Path::new(static_lib_str); - - // NOTE: file_stem only split eg "libprotobufd.so.3.19.4.0" -> "libprotobufd.so.3.19.4" - // but that is NOT what we want (ie "libprotobufd") - // TODO use "file_prefix" https://github.com/rust-lang/rust/issues/86319 - let liblib_name = static_lib_path.my_file_prefix().unwrap(); - let liblib_name_str: String = liblib_name.to_str().unwrap().into(); - let lib_name_str = liblib_name_str.strip_prefix("lib").unwrap(); - - // basically: - // - input = /.../target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/glog-build/libglogd.so.0.6.0 - // - get the extension: a (or "so.3.19.4" or "so" etc) - // NOTE: extension DOES NOT work(same issue than file_stem) - // eg ".../libglogd.so.0.6.0".extension() == "0" (ie the part after the last dot) - // and we NEED "so" (ie the part after the FIRST dot) - let file_with_ext = static_lib_path.file_name().unwrap(); - let full_ext = file_with_ext - .to_str() - .unwrap() - .trim_start_matches(&liblib_name_str); - let is_static = full_ext.starts_with(".a"); - - let dir = static_lib_path.parent().unwrap(); - - // COULD probably have a more foolproof system by using the IMPORTED property in CMake - // and writing that to a different file(or the same one with a prefix/suffix?) - // NOTE: be sure that the prefix does not conflict with the Dockerfile WORKDIR /usr/src/app - let is_system = dir.starts_with("/usr/lib/"); - - return ( - dir.to_path_buf(), - lib_name_str.to_string(), - is_static, - is_system, - ); -} - -// Parse the content of "cmake_generated_rust_wrapper_libs" which SHOULD have -// been generated by our CMake function. -// It is expected to contain a list of space separated libraries eg: -// "/full/path/build/liblib1.so /full/path/build/liblib2.a /usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0" -// etc -fn read_cmake_generated_to_output( - cmake_generated_rust_wrapper_libs_str: &str, - output: &mut impl Write, - is_alwayslink: bool -) { - // Previous version was globing all .a and .so in the build dir but it only worked for SHARED dependencies. - // That is b/c when linking STATIC libs order matters! So we must get a proper list from CMake. - for static_lib_str in cmake_generated_rust_wrapper_libs_str - .split(&[' ', '\n'][..]) - .filter(|&x| !x.is_empty()) - { - let (dir, lib_name_str, is_static, is_system) = parse_lib_path_dir_and_name(static_lib_str); - // WARNING: we MUST add to the linker path: - // - NON system libs (obviously) wether SHARED or STATIC - // - system STATIC libs eg /usr/lib/x86_64-linux-gnu/libboost_filesystem.a else - // "error: could not find native static library `boost_filesystem`, perhaps an -L flag is missing?" - if !is_system || is_static { - writeln!(output, "cargo:rustc-link-search=native={}", dir.display()).unwrap(); - } - - writeln!( - output, - "cargo:rustc-link-lib={}{}={}", - if is_static { "static" } else { "dylib" }, - if is_alwayslink { ":+whole-archive,-bundle" } else { "" }, - lib_name_str - ) - .unwrap(); - } -} - -pub fn read_cmake_generated(cmake_generated_rust_wrapper_libs_str: &str, alwayslink: bool) { - read_cmake_generated_to_output( - &cmake_generated_rust_wrapper_libs_str, - &mut std::io::stdout(), - alwayslink - ) -} - -//////////////////////////////////////////////////////////////////////////////// -/// TEMP -/// Implement "file_prefix" -/// copy pasted from https://github.com/rust-lang/rust/issues/86319 -use std::ffi::OsStr; - -trait HasMyFilePrefix { - fn my_file_prefix(&self) -> Option<&OsStr>; -} - -impl HasMyFilePrefix for std::path::Path { - fn my_file_prefix(&self) -> Option<&OsStr> { - self.file_name() - .map(split_file_at_dot) - .and_then(|(before, _after)| Some(before)) - } -} - -fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) { - let slice = os_str_as_u8_slice(file); - if slice == b".." { - return (file, None); - } - - // The unsafety here stems from converting between &OsStr and &[u8] - // and back. This is safe to do because (1) we only look at ASCII - // contents of the encoding and (2) new &OsStr values are produced - // only from ASCII-bounded slices of existing &OsStr values. - let i = match slice[1..].iter().position(|b| *b == b'.') { - Some(i) => i + 1, - None => return (file, None), - }; - let before = &slice[..i]; - let after = &slice[i + 1..]; - unsafe { (u8_slice_as_os_str(before), Some(u8_slice_as_os_str(after))) } -} - -fn os_str_as_u8_slice(s: &OsStr) -> &[u8] { - unsafe { &*(s as *const OsStr as *const [u8]) } -} -unsafe fn u8_slice_as_os_str(s: &[u8]) -> &OsStr { - // SAFETY: see the comment of `os_str_as_u8_slice` - { - &*(s as *const [u8] as *const OsStr) - } -} - -//////////////////////////////////////////////////////////////////////////////// - -#[cfg(test)] -mod tests { - use crate::{parse_lib_path_dir_and_name, read_cmake_generated_to_output}; - - #[test] - fn parse_local_lib_static_ok() { - let (dir, lib_name_str, is_static, is_system) = - parse_lib_path_dir_and_name("/some/path/liblibstatic.a"); - assert_eq!(dir.as_os_str(), "/some/path"); - assert_eq!(lib_name_str, "libstatic"); - assert_eq!(is_static, true); - assert_eq!(is_system, false); - } - - #[test] - fn parse_local_lib_shared_ok() { - let (dir, lib_name_str, is_static, is_system) = - parse_lib_path_dir_and_name("/some/path/liblibshared.so"); - assert_eq!(dir.as_os_str(), "/some/path"); - assert_eq!(lib_name_str, "libshared"); - assert_eq!(is_static, false); - assert_eq!(is_system, false); - } - - #[test] - fn parse_local_lib_shared_with_soversion_ok() { - let (dir, lib_name_str, is_static, is_system) = - parse_lib_path_dir_and_name("/some/path/liblibshared.so.1.2.3"); - assert_eq!(dir.as_os_str(), "/some/path"); - assert_eq!(lib_name_str, "libshared"); - assert_eq!(is_static, false); - assert_eq!(is_system, false); - } - - #[test] - fn parse_system_lib_static_ok() { - let (dir, lib_name_str, is_static, is_system) = - parse_lib_path_dir_and_name("/usr/lib/libsystem1.a"); - assert_eq!(dir.as_os_str(), "/usr/lib"); - assert_eq!(lib_name_str, "system1"); - assert_eq!(is_static, true); - assert_eq!(is_system, true); - } - - #[test] - fn parse_system_lib_shared_ok() { - let (dir, lib_name_str, is_static, is_system) = - parse_lib_path_dir_and_name("/usr/lib/libsystem2.so"); - assert_eq!(dir.as_os_str(), "/usr/lib"); - assert_eq!(lib_name_str, "system2"); - assert_eq!(is_static, false); - assert_eq!(is_system, true); - } - - #[test] - fn test_read_cmake_generated_to_output() { - let input = "/some/libA.a /some/libB.so"; - let mut stdout = Vec::new(); - read_cmake_generated_to_output(input, &mut stdout, false); - - assert_eq!( - std::str::from_utf8(&stdout).unwrap(), - "cargo:rustc-link-search=native=/some\n\ - cargo:rustc-link-lib=static=A\n\ - cargo:rustc-link-search=native=/some\n\ - cargo:rustc-link-lib=dylib=B\n" - ); - } - - // no need to touch "rustc-link-search" to link with eg "/usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0" - // simply "cargo:rustc-link-lib=dylib=png16.so" is OK - #[test] - fn test_read_cmake_generated_to_output_system_shared_no_rustc_link_search() { - let input = "/usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0"; - let mut stdout = Vec::new(); - read_cmake_generated_to_output(input, &mut stdout, false); - - assert_eq!( - std::str::from_utf8(&stdout).unwrap(), - "cargo:rustc-link-lib=dylib=png16\n" - ); - } - - // BUT system STATIC libs require "rustc-link-search"?? - #[test] - fn test_read_cmake_generated_to_output_system_static_rustc_link_search() { - let input = "/usr/lib/x86_64-linux-gnu/libpng16.a"; - let mut stdout = Vec::new(); - read_cmake_generated_to_output(input, &mut stdout, false); - - assert_eq!( - std::str::from_utf8(&stdout).unwrap(), - "cargo:rustc-link-search=native=/usr/lib/x86_64-linux-gnu\n\ - cargo:rustc-link-lib=static=png16\n" - ); - } -} From 0e4ca4775604d8bd3a5f0702c17bdfd0d011d1b8 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 19:06:35 -0700 Subject: [PATCH 10/14] update --- .gitmodules | 3 + crates/ctranslate2-bindings/CMakeLists.txt | 5 +- crates/ctranslate2-bindings/build.rs | 20 ++++- .../ctranslate2-bindings/cmake/debug_cmake.sh | 2 +- .../cmake/export_libs.cmake | 88 +++++++++++++++++++ crates/rust-cxx-cmake-bridge | 1 + 6 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 crates/ctranslate2-bindings/cmake/export_libs.cmake create mode 160000 crates/rust-cxx-cmake-bridge diff --git a/.gitmodules b/.gitmodules index 17f7250cbe1f..d754dac8faf6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "crates/ctranslate2-bindings/CTranslate2"] path = crates/ctranslate2-bindings/CTranslate2 url = https://github.com/OpenNMT/CTranslate2.git +[submodule "crates/rust-cxx-cmake-bridge"] + path = crates/rust-cxx-cmake-bridge + url = https://github.com/Interstellar-Network/rust-cxx-cmake-bridge.git diff --git a/crates/ctranslate2-bindings/CMakeLists.txt b/crates/ctranslate2-bindings/CMakeLists.txt index aa8e420b222f..91b6922ae8ba 100644 --- a/crates/ctranslate2-bindings/CMakeLists.txt +++ b/crates/ctranslate2-bindings/CMakeLists.txt @@ -12,5 +12,8 @@ target_link_libraries(dummy PRIVATE ctranslate2 ) -include(../rust-cxx-cmake-bridge/cmake/export_libs.cmake) +get_target_property(OUT ctranslate2 LINK_LIBRARIES) +message(STATUS ${OUT}) + +include(cmake/export_libs.cmake) export_all_target_libs(dummy) diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 906de02f18d1..71e36bc329ce 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -41,6 +41,25 @@ fn ctranslate2_build_linux_shared() -> PathBuf { .build() } +fn ctranslate2_build_linux_static() -> PathBuf { + Config::new(".") + .define("BUILD_CLI", "OFF") + .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") + .define("BUILD_SHARED_LIBS", "OFF") + + .define("BUILD_CLI", "OFF") + .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") + .define("WITH_CUDA", "ON") + .define("WITH_CUDNN", "ON") + .define("WITH_MKL", "ON") + .define("WITH_DNNL", "ON") + .define("OPENMP_RUNTIME", "COMP") + .cxxflag("-msse4.1") + .define("CUDA_NVCC_FLAGS", "-Xfatbin=-compress-all") + .define("CUDA_ARCH_LIST", "Common") + .build() +} + fn ctranslate2_build_macos_static() -> PathBuf { let dst = Config::new(".") .define("BUILD_CLI", "OFF") @@ -56,7 +75,6 @@ fn ctranslate2_build_macos_static() -> PathBuf { let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); read_cmake_generated(&cmake_generated_libs_str); - println!("cargo:rustc-link-lib=framework=Accelerate"); return dst; } diff --git a/crates/ctranslate2-bindings/cmake/debug_cmake.sh b/crates/ctranslate2-bindings/cmake/debug_cmake.sh index 8757a64a7b65..9acbfdb8c833 100755 --- a/crates/ctranslate2-bindings/cmake/debug_cmake.sh +++ b/crates/ctranslate2-bindings/cmake/debug_cmake.sh @@ -9,4 +9,4 @@ mkdir build && cd build CMAKE_EXTRA_OPTIONS='-DCMAKE_OSX_ARCHITECTURES=arm64 -DWITH_ACCELERATE=ON -DWITH_MKL=OFF -DOPENMP_RUNTIME=NONE -DWITH_RUY=ON' cmake -DBULID_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_CLI=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON $CMAKE_EXTRA_OPTIONS .. -make -j4 +"$@" diff --git a/crates/ctranslate2-bindings/cmake/export_libs.cmake b/crates/ctranslate2-bindings/cmake/export_libs.cmake new file mode 100644 index 000000000000..1c9787fee3ac --- /dev/null +++ b/crates/ctranslate2-bindings/cmake/export_libs.cmake @@ -0,0 +1,88 @@ +################################################################################ + +# WARNING: to list the system libraries(ie IMPORTED) you MUST set: +# set_target_properties(your_lib PROPERTIES IMPORTED_GLOBAL TRUE) +# just after the find_package call +# cf https://gitlab.kitware.com/cmake/cmake/-/issues/17256 +# +# https://stackoverflow.com/questions/32756195/recursive-list-of-link-libraries-in-cmake +# https://stackoverflow.com/questions/32197663/how-can-i-remove-the-the-location-property-may-not-be-read-from-target-error-i +function(_get_link_libraries OUTPUT_LIST TARGET) + list(APPEND VISITED_TARGETS ${TARGET}) + + # DO NOT switch on IMPORTED or not + # An INTERFACE library CAN have LINK_LIBRARIES! + # get_target_property(IMPORTED ${TARGET} IMPORTED) + set(LIBS "") + get_target_property(LIBS_1 ${TARGET} INTERFACE_LINK_LIBRARIES) + get_target_property(LIBS_2 ${TARGET} LINK_LIBRARIES) + list(APPEND LIBS ${LIBS_1} ${LIBS_2}) + + set(LIB_FILES "") + + foreach(LIB ${LIBS}) + if (TARGET ${LIB}) + list(FIND VISITED_TARGETS ${LIB} VISITED) + if (${VISITED} EQUAL -1) + # OLD: get_target_property(LIB_FILE ${LIB} LOCATION) + # NEW: + _get_link_libraries(LINK_LIB_FILES ${LIB}) + set(LIB_FILE ${LIB}) + list(APPEND LIB_FILES ${LINK_LIB_FILES}) + list(APPEND LIB_FILES ${LIB_FILE}) + endif() + endif() + endforeach() + + set(VISITED_TARGETS ${VISITED_TARGETS} PARENT_SCOPE) + set(${OUTPUT_LIST} ${LIB_FILES} PARENT_SCOPE) +endfunction() + +################################################################################ + +function(export_all_target_libs TARGET) + # NOTE: get_target_property(CIRCUIT_LIB_LINK_LIBRARIES a_target LINK_LIBRARIES) is NOT transitive + # This function will return eg: "$;$;" + # b/c generator expression are evaluated LATER + # cf https://stackoverflow.com/questions/59226127/cmake-generator-expression-how-to-get-target-file-property-on-list-of-targets + set(ALL_LINK_LIBRARIES "") + _get_link_libraries(ALL_LINK_LIBRARIES ${TARGET}) + + message(STATUS "ALL_LINK_LIBRARIES : ${ALL_LINK_LIBRARIES}") + + set(ALL_LIBS "") + # TODO move that back into get_link_libraries + # NOTE: we MUST do it in 2 steps: + # - collect all the LINK_LIBRARIES recursively + # - loop on those and get their TARGET_FILE (if not INTERFACE_LIBRARY) + # That is b/c in get_link_libraries a INTERFACE_LIBRARY CAN have link_libraries + # but we CAN NOT evaluate generator expressions at this time. + foreach(LIB ${ALL_LINK_LIBRARIES}) + # MUST skip INTERFACE else: + # CMake Error at src/CMakeLists.txt:136 (add_custom_command): + # Error evaluating generator expression: + # $ + # Target "rust_cxx" is not an executable or library. + # SHARED_LIBRARY,INTERFACE_LIBRARY,STATIC_LIBRARY + # + get_target_property(LIB_TYPE ${LIB} TYPE) + message(STATUS "LIB_TYPE : ${LIB} = ${LIB_TYPE}") + + if(NOT ${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") + set(LIB_FILE $) + list(APPEND ALL_LIBS ${LIB_FILE}) + endif() + endforeach() # LIB ${ALL_LIBS} + + message(STATUS "ALL_LIBS : ${ALL_LIBS}") + + # add_custom_command(ie echoing only to stdout) works but more difficult to get from build.rs + # b/c when there is "ninja: no work to do" it will NOT echo on the console + add_custom_command( + TARGET ${TARGET} + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo ${ALL_LIBS} > ${CMAKE_CURRENT_BINARY_DIR}/cmake_generated_libs + # OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cmake_generated_libs + VERBATIM + ) +endfunction(export_all_target_libs) diff --git a/crates/rust-cxx-cmake-bridge b/crates/rust-cxx-cmake-bridge new file mode 160000 index 000000000000..2236064d88a1 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge @@ -0,0 +1 @@ +Subproject commit 2236064d88a1e72897f77b2ecbeecab3402c4c6d From d6806c3d5fec3c2ac74cd192a646da15b8b13672 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 19:53:07 -0700 Subject: [PATCH 11/14] update build.rs --- crates/ctranslate2-bindings/build.rs | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 71e36bc329ce..3a53818f5328 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -6,7 +6,7 @@ fn main() { let dst = if cfg!(target_os = "macos") { ctranslate2_build_macos_static() } else if cfg!(target_os = "linux") { - ctranslate2_build_linux_shared() + ctranslate2_build_linux_static() } else { panic!("Invalid target") }; @@ -23,24 +23,6 @@ fn main() { .compile("cxxbridge"); } -fn ctranslate2_build_linux_shared() -> PathBuf { - Config::new("CTranslate2") - .define("BUILD_CLI", "OFF") - .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") - - .define("BUILD_CLI", "OFF") - .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") - .define("WITH_CUDA", "ON") - .define("WITH_CUDNN", "ON") - .define("WITH_MKL", "ON") - .define("WITH_DNNL", "ON") - .define("OPENMP_RUNTIME", "COMP") - .cxxflag("-msse4.1") - .define("CUDA_NVCC_FLAGS", "-Xfatbin=-compress-all") - .define("CUDA_ARCH_LIST", "Common") - .build() -} - fn ctranslate2_build_linux_static() -> PathBuf { Config::new(".") .define("BUILD_CLI", "OFF") @@ -75,6 +57,7 @@ fn ctranslate2_build_macos_static() -> PathBuf { let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); read_cmake_generated(&cmake_generated_libs_str); + println!("cargo:rustc-link-lib=framework=Accelerate"); return dst; } From abbda65aa4105ea1c582c5eaabcee72cf1db4da2 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 21:24:39 -0700 Subject: [PATCH 12/14] parse external libs --- .gitmodules | 3 - crates/ctranslate2-bindings/CMakeLists.txt | 3 - crates/ctranslate2-bindings/build.rs | 1 - .../ctranslate2-bindings/cmake/debug_cmake.sh | 13 + .../cmake/export_libs.cmake | 18 +- crates/rust-cxx-cmake-bridge | 1 - crates/rust-cxx-cmake-bridge/Cargo.toml | 8 + crates/rust-cxx-cmake-bridge/src/lib.rs | 243 ++++++++++++++++++ 8 files changed, 278 insertions(+), 12 deletions(-) delete mode 160000 crates/rust-cxx-cmake-bridge create mode 100644 crates/rust-cxx-cmake-bridge/Cargo.toml create mode 100644 crates/rust-cxx-cmake-bridge/src/lib.rs diff --git a/.gitmodules b/.gitmodules index d754dac8faf6..17f7250cbe1f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "crates/ctranslate2-bindings/CTranslate2"] path = crates/ctranslate2-bindings/CTranslate2 url = https://github.com/OpenNMT/CTranslate2.git -[submodule "crates/rust-cxx-cmake-bridge"] - path = crates/rust-cxx-cmake-bridge - url = https://github.com/Interstellar-Network/rust-cxx-cmake-bridge.git diff --git a/crates/ctranslate2-bindings/CMakeLists.txt b/crates/ctranslate2-bindings/CMakeLists.txt index 91b6922ae8ba..2c40151077f9 100644 --- a/crates/ctranslate2-bindings/CMakeLists.txt +++ b/crates/ctranslate2-bindings/CMakeLists.txt @@ -12,8 +12,5 @@ target_link_libraries(dummy PRIVATE ctranslate2 ) -get_target_property(OUT ctranslate2 LINK_LIBRARIES) -message(STATUS ${OUT}) - include(cmake/export_libs.cmake) export_all_target_libs(dummy) diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index 3a53818f5328..f8d6884dfa94 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -57,7 +57,6 @@ fn ctranslate2_build_macos_static() -> PathBuf { let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); read_cmake_generated(&cmake_generated_libs_str); - println!("cargo:rustc-link-lib=framework=Accelerate"); return dst; } diff --git a/crates/ctranslate2-bindings/cmake/debug_cmake.sh b/crates/ctranslate2-bindings/cmake/debug_cmake.sh index 9acbfdb8c833..0f4b5a8ed35d 100755 --- a/crates/ctranslate2-bindings/cmake/debug_cmake.sh +++ b/crates/ctranslate2-bindings/cmake/debug_cmake.sh @@ -3,10 +3,23 @@ set -e set -x +UNAME="$(uname -s)" +case "${UNAME}" in + Linux*) MACHINE=linux;; + Darwin*) MACHINE=macos;; + *) exit 1;; +esac + rm -rf build mkdir build && cd build +if [[ "$MACHINE" == "macos" ]]; then CMAKE_EXTRA_OPTIONS='-DCMAKE_OSX_ARCHITECTURES=arm64 -DWITH_ACCELERATE=ON -DWITH_MKL=OFF -DOPENMP_RUNTIME=NONE -DWITH_RUY=ON' +elif [[ "$MACHINE" == "linux" ]]; then +CMAKE_EXTRA_OPTIONS='-DWITH_CUDA=ON -DWITH_CUDNN=ON -DWITH_MKL=ON -DWITH_DNNL=ON -DOPENMP_RUNTIME=COMP -DCUDA_NVCC_FLAGS=-Xfatbin=-compress-all -DCUDA_ARCH_LIST=Common -DCXXFLAGS=-msse4.1' +fi + + cmake -DBULID_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_CLI=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON $CMAKE_EXTRA_OPTIONS .. "$@" diff --git a/crates/ctranslate2-bindings/cmake/export_libs.cmake b/crates/ctranslate2-bindings/cmake/export_libs.cmake index 1c9787fee3ac..9d30a01425d7 100644 --- a/crates/ctranslate2-bindings/cmake/export_libs.cmake +++ b/crates/ctranslate2-bindings/cmake/export_libs.cmake @@ -31,6 +31,9 @@ function(_get_link_libraries OUTPUT_LIST TARGET) list(APPEND LIB_FILES ${LINK_LIB_FILES}) list(APPEND LIB_FILES ${LIB_FILE}) endif() + elseif(EXISTS ${LIB}) + set(LIB_FILE ${LIB}) + list(APPEND LIB_FILES ${LIB_FILE}) endif() endforeach() @@ -51,6 +54,7 @@ function(export_all_target_libs TARGET) message(STATUS "ALL_LINK_LIBRARIES : ${ALL_LINK_LIBRARIES}") set(ALL_LIBS "") + set(ALL_EXTERNAL_LIBS "") # TODO move that back into get_link_libraries # NOTE: we MUST do it in 2 steps: # - collect all the LINK_LIBRARIES recursively @@ -65,11 +69,17 @@ function(export_all_target_libs TARGET) # Target "rust_cxx" is not an executable or library. # SHARED_LIBRARY,INTERFACE_LIBRARY,STATIC_LIBRARY # - get_target_property(LIB_TYPE ${LIB} TYPE) - message(STATUS "LIB_TYPE : ${LIB} = ${LIB_TYPE}") + if (TARGET ${LIB}) + get_target_property(LIB_TYPE ${LIB} TYPE) + message(STATUS "LIB_TYPE : ${LIB} = ${LIB_TYPE}") - if(NOT ${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") - set(LIB_FILE $) + if(NOT ${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") + set(LIB_FILE $) + list(APPEND ALL_LIBS ${LIB_FILE}) + endif() + elseif(EXISTS ${LIB}) + set(LIB_FILE ${LIB}) + message(STATUS "LIB_TYPE : ${LIB} = EXTERNAL") list(APPEND ALL_LIBS ${LIB_FILE}) endif() endforeach() # LIB ${ALL_LIBS} diff --git a/crates/rust-cxx-cmake-bridge b/crates/rust-cxx-cmake-bridge deleted file mode 160000 index 2236064d88a1..000000000000 --- a/crates/rust-cxx-cmake-bridge +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2236064d88a1e72897f77b2ecbeecab3402c4c6d diff --git a/crates/rust-cxx-cmake-bridge/Cargo.toml b/crates/rust-cxx-cmake-bridge/Cargo.toml new file mode 100644 index 000000000000..c0edcbee1975 --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "rust-cxx-cmake-bridge" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/rust-cxx-cmake-bridge/src/lib.rs b/crates/rust-cxx-cmake-bridge/src/lib.rs new file mode 100644 index 000000000000..4f9b8f47f4ab --- /dev/null +++ b/crates/rust-cxx-cmake-bridge/src/lib.rs @@ -0,0 +1,243 @@ +use std::io::Write; +use std::path::PathBuf; + +// ## lib_name: "protobufd" +// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/protobuf_fetch-build" +// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/protobuf_fetch-build +// rustc-link-lib=protobufd +// ## lib_name: "libyosys" +// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/yosys_fetch-build" +// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/yosys_fetch-build +// rustc-link-lib=libyosys +// ## lib_name: "xxhash" +// ## dir: "..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/xxhash-build" +// rustc-link-search=native=..../api_circuits/target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/xxhash-build +// rustc-link-lib=xxhash +fn parse_lib_path_dir_and_name(static_lib_str: &str) -> (PathBuf, String, bool, bool, bool) { + let static_lib_path = std::path::Path::new(static_lib_str); + + // NOTE: file_stem only split eg "libprotobufd.so.3.19.4.0" -> "libprotobufd.so.3.19.4" + // but that is NOT what we want (ie "libprotobufd") + // TODO use "file_prefix" https://github.com/rust-lang/rust/issues/86319 + let liblib_name = static_lib_path.my_file_prefix().unwrap(); + let liblib_name_str: String = liblib_name.to_str().unwrap().into(); + let lib_name_str = liblib_name_str.trim_start_matches("lib"); + + // basically: + // - input = /.../target/debug/build/lib-circuits-wrapper-49025516ce40925e/out/build/_deps/glog-build/libglogd.so.0.6.0 + // - get the extension: a (or "so.3.19.4" or "so" etc) + // NOTE: extension DOES NOT work(same issue than file_stem) + // eg ".../libglogd.so.0.6.0".extension() == "0" (ie the part after the last dot) + // and we NEED "so" (ie the part after the FIRST dot) + let file_with_ext = static_lib_path.file_name().unwrap(); + let full_ext = file_with_ext + .to_str() + .unwrap() + .trim_start_matches(&liblib_name_str); + let is_static = full_ext.starts_with(".a"); + + let dir = static_lib_path.parent().unwrap(); + + // COULD probably have a more foolproof system by using the IMPORTED property in CMake + // and writing that to a different file(or the same one with a prefix/suffix?) + // NOTE: be sure that the prefix does not conflict with the Dockerfile WORKDIR /usr/src/app + let is_system = dir.starts_with("/usr/lib/"); + + let is_framework = static_lib_str.ends_with(".framework"); + + return ( + dir.to_path_buf(), + lib_name_str.to_string(), + is_static, + is_system, + is_framework + ); +} + +// Parse the content of "cmake_generated_rust_wrapper_libs" which SHOULD have +// been generated by our CMake function. +// It is expected to contain a list of space separated libraries eg: +// "/full/path/build/liblib1.so /full/path/build/liblib2.a /usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0" +// etc +fn read_cmake_generated_to_output( + cmake_generated_rust_wrapper_libs_str: &str, + output: &mut impl Write, +) { + // Previous version was globing all .a and .so in the build dir but it only worked for SHARED dependencies. + // That is b/c when linking STATIC libs order matters! So we must get a proper list from CMake. + for static_lib_str in cmake_generated_rust_wrapper_libs_str + .split(&[' ', '\n'][..]) + .filter(|&x| !x.is_empty()) + { + let (dir, lib_name_str, is_static, is_system, is_framework) = parse_lib_path_dir_and_name(static_lib_str); + // WARNING: we MUST add to the linker path: + // - NON system libs (obviously) wether SHARED or STATIC + // - system STATIC libs eg /usr/lib/x86_64-linux-gnu/libboost_filesystem.a else + // "error: could not find native static library `boost_filesystem`, perhaps an -L flag is missing?" + if !is_system || is_static || !is_framework { + writeln!(output, "cargo:rustc-link-search=native={}", dir.display()).unwrap(); + } + + writeln!( + output, + "cargo:rustc-link-lib={}={}", + if is_framework { "framework" } else if is_static { "static" } else { "dylib" }, + lib_name_str + ) + .unwrap(); + } +} + +pub fn read_cmake_generated(cmake_generated_rust_wrapper_libs_str: &str) { + read_cmake_generated_to_output( + &cmake_generated_rust_wrapper_libs_str, + &mut std::io::stdout(), + ) +} + +//////////////////////////////////////////////////////////////////////////////// +/// TEMP +/// Implement "file_prefix" +/// copy pasted from https://github.com/rust-lang/rust/issues/86319 +use std::ffi::OsStr; + +trait HasMyFilePrefix { + fn my_file_prefix(&self) -> Option<&OsStr>; +} + +impl HasMyFilePrefix for std::path::Path { + fn my_file_prefix(&self) -> Option<&OsStr> { + self.file_name() + .map(split_file_at_dot) + .and_then(|(before, _after)| Some(before)) + } +} + +fn split_file_at_dot(file: &OsStr) -> (&OsStr, Option<&OsStr>) { + let slice = os_str_as_u8_slice(file); + if slice == b".." { + return (file, None); + } + + // The unsafety here stems from converting between &OsStr and &[u8] + // and back. This is safe to do because (1) we only look at ASCII + // contents of the encoding and (2) new &OsStr values are produced + // only from ASCII-bounded slices of existing &OsStr values. + let i = match slice[1..].iter().position(|b| *b == b'.') { + Some(i) => i + 1, + None => return (file, None), + }; + let before = &slice[..i]; + let after = &slice[i + 1..]; + unsafe { (u8_slice_as_os_str(before), Some(u8_slice_as_os_str(after))) } +} + +fn os_str_as_u8_slice(s: &OsStr) -> &[u8] { + unsafe { &*(s as *const OsStr as *const [u8]) } +} +unsafe fn u8_slice_as_os_str(s: &[u8]) -> &OsStr { + // SAFETY: see the comment of `os_str_as_u8_slice` + { + &*(s as *const [u8] as *const OsStr) + } +} + +//////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod tests { + use crate::{parse_lib_path_dir_and_name, read_cmake_generated_to_output}; + + #[test] + fn parse_local_lib_static_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/some/path/liblibstatic.a"); + assert_eq!(dir.as_os_str(), "/some/path"); + assert_eq!(lib_name_str, "libstatic"); + assert_eq!(is_static, true); + assert_eq!(is_system, false); + } + + #[test] + fn parse_local_lib_shared_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/some/path/liblibshared.so"); + assert_eq!(dir.as_os_str(), "/some/path"); + assert_eq!(lib_name_str, "libshared"); + assert_eq!(is_static, false); + assert_eq!(is_system, false); + } + + #[test] + fn parse_local_lib_shared_with_soversion_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/some/path/liblibshared.so.1.2.3"); + assert_eq!(dir.as_os_str(), "/some/path"); + assert_eq!(lib_name_str, "libshared"); + assert_eq!(is_static, false); + assert_eq!(is_system, false); + } + + #[test] + fn parse_system_lib_static_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/usr/lib/libsystem1.a"); + assert_eq!(dir.as_os_str(), "/usr/lib"); + assert_eq!(lib_name_str, "system1"); + assert_eq!(is_static, true); + assert_eq!(is_system, true); + } + + #[test] + fn parse_system_lib_shared_ok() { + let (dir, lib_name_str, is_static, is_system) = + parse_lib_path_dir_and_name("/usr/lib/libsystem2.so"); + assert_eq!(dir.as_os_str(), "/usr/lib"); + assert_eq!(lib_name_str, "system2"); + assert_eq!(is_static, false); + assert_eq!(is_system, true); + } + + #[test] + fn test_read_cmake_generated_to_output() { + let input = "/some/libA.a /some/libB.so"; + let mut stdout = Vec::new(); + read_cmake_generated_to_output(input, &mut stdout); + + assert_eq!( + std::str::from_utf8(&stdout).unwrap(), + "cargo:rustc-link-search=native=/some\n\ + cargo:rustc-link-lib=static=A\n\ + cargo:rustc-link-search=native=/some\n\ + cargo:rustc-link-lib=dylib=B\n" + ); + } + + // no need to touch "rustc-link-search" to link with eg "/usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0" + // simply "cargo:rustc-link-lib=dylib=png16.so" is OK + #[test] + fn test_read_cmake_generated_to_output_system_shared_no_rustc_link_search() { + let input = "/usr/lib/x86_64-linux-gnu/libpng16.so.16.37.0"; + let mut stdout = Vec::new(); + read_cmake_generated_to_output(input, &mut stdout); + + assert_eq!( + std::str::from_utf8(&stdout).unwrap(), + "cargo:rustc-link-lib=dylib=png16\n" + ); + } + + // BUT system STATIC libs require "rustc-link-search"?? + #[test] + fn test_read_cmake_generated_to_output_system_static_rustc_link_search() { + let input = "/usr/lib/x86_64-linux-gnu/libpng16.a"; + let mut stdout = Vec::new(); + read_cmake_generated_to_output(input, &mut stdout); + + assert_eq!( + std::str::from_utf8(&stdout).unwrap(), + "cargo:rustc-link-search=native=/usr/lib/x86_64-linux-gnu\n\ + cargo:rustc-link-lib=static=png16\n" + ); + } +} From 44c7358a166a010dc76dbfc8f88590a4ee35140e Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 21:31:51 -0700 Subject: [PATCH 13/14] cleanup --- crates/ctranslate2-bindings/build.rs | 76 ++++++++++++---------------- 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/crates/ctranslate2-bindings/build.rs b/crates/ctranslate2-bindings/build.rs index f8d6884dfa94..e42936e891ae 100644 --- a/crates/ctranslate2-bindings/build.rs +++ b/crates/ctranslate2-bindings/build.rs @@ -1,16 +1,44 @@ -use std::path::PathBuf; use cmake::Config; use rust_cxx_cmake_bridge::read_cmake_generated; fn main() { - let dst = if cfg!(target_os = "macos") { - ctranslate2_build_macos_static() - } else if cfg!(target_os = "linux") { - ctranslate2_build_linux_static() + let mut config = Config::new("."); + config + .define("CMAKE_BUILD_TYPE", "Release") + .define("BUILD_CLI", "OFF") + .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") + .define("BUILD_SHARED_LIBS", "OFF"); + + if cfg!(target_os = "linux") { + config + .define("WITH_CUDA", "ON") + .define("WITH_CUDNN", "ON") + .define("WITH_MKL", "ON") + .define("WITH_DNNL", "ON") + .define("OPENMP_RUNTIME", "COMP") + .cxxflag("-msse4.1") + .define("CUDA_NVCC_FLAGS", "-Xfatbin=-compress-all") + .define("CUDA_ARCH_LIST", "Common") + } else if cfg!(target_os = "macos") { + config + .define("CMAKE_OSX_ARCHITECTURES", "arm64") + .define("WITH_ACCELERATE", "ON") + .define("WITH_MKL", "OFF") + .define("OPENMP_RUNTIME", "NONE") + .define("WITH_RUY", "ON") } else { panic!("Invalid target") }; + let dst = config.build(); + + // Read static lib from generated deps. + let cmake_generated_libs_str = std::fs::read_to_string( + &format!("/{}/build/cmake_generated_libs", dst.display()).to_string(), + ) + .unwrap(); + read_cmake_generated(&cmake_generated_libs_str); + // Tell cargo to invalidate the built crate whenever the wrapper changes println!("cargo:rerun-if-changed=include/ctranslate2.h"); println!("cargo:rerun-if-changed=src/ctranslate2.cc"); @@ -22,41 +50,3 @@ fn main() { .flag_if_supported(&format!("-I{}", dst.join("include").display())) .compile("cxxbridge"); } - -fn ctranslate2_build_linux_static() -> PathBuf { - Config::new(".") - .define("BUILD_CLI", "OFF") - .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") - .define("BUILD_SHARED_LIBS", "OFF") - - .define("BUILD_CLI", "OFF") - .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") - .define("WITH_CUDA", "ON") - .define("WITH_CUDNN", "ON") - .define("WITH_MKL", "ON") - .define("WITH_DNNL", "ON") - .define("OPENMP_RUNTIME", "COMP") - .cxxflag("-msse4.1") - .define("CUDA_NVCC_FLAGS", "-Xfatbin=-compress-all") - .define("CUDA_ARCH_LIST", "Common") - .build() -} - -fn ctranslate2_build_macos_static() -> PathBuf { - let dst = Config::new(".") - .define("BUILD_CLI", "OFF") - .define("CMAKE_INSTALL_RPATH_USE_LINK_PATH", "ON") - .define("BUILD_SHARED_LIBS", "OFF") - - .define("CMAKE_OSX_ARCHITECTURES", "arm64") - .define("WITH_ACCELERATE", "ON") - .define("WITH_MKL", "OFF") - .define("OPENMP_RUNTIME", "NONE") - .define("WITH_RUY", "ON") - .build(); - - let cmake_generated_libs_str = std::fs::read_to_string(&format!("/{}/build/cmake_generated_libs", dst.display()).to_string()).unwrap(); - read_cmake_generated(&cmake_generated_libs_str); - - return dst; -} From 463dec0b2c1a739b5df8b0625a13f37ab10a42e2 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 26 May 2023 21:33:39 -0700 Subject: [PATCH 14/14] add cargo fmt --- crates/rust-cxx-cmake-bridge/src/lib.rs | 13 ++++++++++--- crates/tabby/src/serve/mod.rs | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/rust-cxx-cmake-bridge/src/lib.rs b/crates/rust-cxx-cmake-bridge/src/lib.rs index 4f9b8f47f4ab..94bb7a1de325 100644 --- a/crates/rust-cxx-cmake-bridge/src/lib.rs +++ b/crates/rust-cxx-cmake-bridge/src/lib.rs @@ -50,7 +50,7 @@ fn parse_lib_path_dir_and_name(static_lib_str: &str) -> (PathBuf, String, bool, lib_name_str.to_string(), is_static, is_system, - is_framework + is_framework, ); } @@ -69,7 +69,8 @@ fn read_cmake_generated_to_output( .split(&[' ', '\n'][..]) .filter(|&x| !x.is_empty()) { - let (dir, lib_name_str, is_static, is_system, is_framework) = parse_lib_path_dir_and_name(static_lib_str); + let (dir, lib_name_str, is_static, is_system, is_framework) = + parse_lib_path_dir_and_name(static_lib_str); // WARNING: we MUST add to the linker path: // - NON system libs (obviously) wether SHARED or STATIC // - system STATIC libs eg /usr/lib/x86_64-linux-gnu/libboost_filesystem.a else @@ -81,7 +82,13 @@ fn read_cmake_generated_to_output( writeln!( output, "cargo:rustc-link-lib={}={}", - if is_framework { "framework" } else if is_static { "static" } else { "dylib" }, + if is_framework { + "framework" + } else if is_static { + "static" + } else { + "dylib" + }, lib_name_str ) .unwrap(); diff --git a/crates/tabby/src/serve/mod.rs b/crates/tabby/src/serve/mod.rs index 4739be375960..ee27e389b612 100644 --- a/crates/tabby/src/serve/mod.rs +++ b/crates/tabby/src/serve/mod.rs @@ -49,7 +49,7 @@ pub struct ServeArgs { #[clap(long)] model: String, - #[clap(long, default_value_t=8080)] + #[clap(long, default_value_t = 8080)] port: u16, #[clap(long, default_value_t=Device::CPU)]