merge with gmaster branch

marian-nmt · Feb 26, 2021 · c2d5fcc · c2d5fcc
2 parents c0a84a4 + 8336bbd
commit c2d5fcc
Show file tree

Hide file tree

Showing 225 changed files with 229,153 additions and 69,881 deletions.
diff --git a/.gitignore b/.gitignore
@@ -30,6 +30,10 @@ CMakeFiles
 CMakeCache.txt
 config.h
 sentencepiece.pc
+CPackConfig.cmake
+CTestTestfile.cmake
+CPackSourceConfig.cmake
+DartConfiguration.tcl
 
 *.o
 *.lo
@@ -53,9 +57,6 @@ spm_train
 spm_normalize
 spm_test
 
-*.pb.cc
-*.pb.h
-
 .DS_Store
 *.egg-info/
 dist/
@@ -65,3 +66,9 @@ dist/
 
 m.model
 m.vocab
+
+cmake_install.cmake
+libsentencepiece.so*
+libsentencepiece_train.so*
+python/bundled
+_sentencepiece.*.so
diff --git a/.travis.yml b/.travis.yml
@@ -5,13 +5,16 @@ matrix:
  env: IMAGE=ubuntu:rolling COMMAND=build_linux_gcc_coverall_ubuntu RELEASE_FILES="$TRAVIS_BUILD_DIR/build/*.xz"
  services: docker
  - os: linux
- env: IMAGE=i386/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu_i386
+ env: IMAGE=i386/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+ services: docker
+ - os: linux
+ env: IMAGE=ubuntu:bionic COMMAND=build_linux_gcc_ubuntu
  services: docker
  - os: linux
  env: IMAGE=ubuntu:xenial COMMAND=build_linux_gcc_ubuntu
  services: docker
  - os: linux
- env: IMAGE=ubuntu:trusty COMMAND=build_linux_gcc_ubuntu_trusty
+ env: IMAGE=ubuntu:trusty COMMAND=build_linux_gcc_ubuntu
  services: docker
  - os: linux
  env: IMAGE=debian:stable COMMAND=build_linux_gcc_debian
@@ -22,6 +25,18 @@ matrix:
  - os: linux
  env: IMAGE=ubuntu:rolling COMMAND=build_linux_clang_ubuntu
  services: docker
+ - os: linux
+ arch: arm64
+ env: IMAGE=arm64v8/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+ services: docker
+ - os: linux
+ arch: ppc64le
+ env: IMAGE=ppc64le/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+ services: docker
+ - os: linux
+ arch: s390x
+ env: IMAGE=s390x/ubuntu:rolling COMMAND=build_linux_gcc_ubuntu
+ services: docker
  - os: linux
  env: IMAGE=x86_64 COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
  script:
@@ -35,32 +50,35 @@ matrix:
  - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi
  services: docker
  - os: linux
- env: IMAGE=x86_64 COMMAND=make_py_wheel_tf RELEASE_FILES="$TRAVIS_BUILD_DIR/tensorflow/dist/*.whl"
+ arch: arm64
+ env: IMAGE=aarch64 COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
  script:
- - $TRAVIS_BUILD_DIR/tensorflow/make_py_wheel.sh
+ - $TRAVIS_BUILD_DIR/python/make_py_wheel.sh ${IMAGE}
+ - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi
+ services: docker
+ - os: linux
+ arch: ppc64le
+ env: IMAGE=ppc64le COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
+ script:
+ - $TRAVIS_BUILD_DIR/python/make_py_wheel.sh ${IMAGE}
+ - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi
+ services: docker
+ - os: linux
+ arch: s390x
+ env: IMAGE=s390x COMMAND=make_py_wheel_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/*manylinux*.whl"
+ script:
+ - $TRAVIS_BUILD_DIR/python/make_py_wheel.sh ${IMAGE}
  - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi
  services: docker
- - os: osx
- osx_image: xcode9.3
- env: IMAGE=native COMMAND=build_osx
  - os: osx
  osx_image: xcode9.4
  env: IMAGE=native COMMAND=build_osx
- - os: osx
- osx_image: xcode9.2
- env: IMAGE=native COMMAND=build_osx
  - os: osx
  osx_image: xcode9.4
  env: IMAGE=native COMMAND=make_py_wheel_mac_py RELEASE_FILES="$TRAVIS_BUILD_DIR/python/dist/delocated_wheel/*.whl"
  script:
  - $TRAVIS_BUILD_DIR/python/make_py_wheel_mac.sh
  - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi
- - os: osx
- osx_image: xcode9.4
- env: IMAGE=native COMMAND=make_py_wheel_mac_tf RELEASE_FILES="$TRAVIS_BUILD_DIR/tensorflow/dist/*.whl"
- script:
- - $TRAVIS_BUILD_DIR/tensorflow/make_py_wheel_mac.sh
- - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi
 script:
  - $TRAVIS_BUILD_DIR/test.sh ${IMAGE} ${COMMAND}
  - if [[ "$RELEASE_FILES" != "" ]]; then ls -l $RELEASE_FILES ; fi

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -15,7 +15,7 @@
 cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
 file(STRINGS "VERSION" SPM_VERSION)
 message(STATUS "VERSION: ${SPM_VERSION}")
-project(sentencepiece VERSION ${SPM_VERSION} LANGUAGES CXX)
+project(sentencepiece VERSION ${SPM_VERSION} LANGUAGES C CXX)
 
 option(SPM_ENABLE_NFKC_COMPILE "Enables NFKC compile" OFF)
 option(SPM_ENABLE_SHARED "Builds shared libaries in addition to static libraries." ON)
@@ -25,24 +25,38 @@ option(SPM_ENABLE_TENSORFLOW_SHARED "Makes a tensorflow compatible shared file."
 option(SPM_ENABLE_TCMALLOC "Enable TCMalloc if available." ON)
 option(SPM_TCMALLOC_STATIC "Link static library of TCMALLOC." OFF)
 option(SPM_NO_THREADLOCAL "Disable thread_local operator" OFF)
+option(SPM_USE_BUILTIN_PROTOBUF "Use built-in protobuf" ON)
 
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-
-set(prefix ${CMAKE_INSTALL_PREFIX})
-set(exec_prefix "\${prefix}")
-set(libdir "\${exec_prefix}/lib")
-set(includedir "\${prefix}/include")
+if (UNIX)
+ include(GNUInstallDirs)
+ set(prefix ${CMAKE_INSTALL_PREFIX})
+ set(exec_prefix "\${prefix}")
+ set(libdir "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}")
+ set(includedir "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}")
+else()
+ set(prefix ${CMAKE_INSTALL_PREFIX})
+ set(exec_prefix "\${prefix}")
+ set(libdir "\${exec_prefix}/lib")
+ set(includedir "\${prefix}/include")
+endif()
 set(GNUCXX_STD_SUPPORT_VERSION "4.3")
 
-if(MSVC)
- string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
- string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_MINSIZEREL ${CMAKE_CXX_FLAGS_MINSIZEREL})
- string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
- string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
- add_definitions("/wd4267 /wd4244 /wd4305 /Zc:strictStrings /utf-8")
-endif(MSVC)
+if (SPM_USE_BUILTIN_PROTOBUF)
+ set(libprotobuf_lite "")
+else()
+ set(libprotobuf_lite "-lprotobuf-lite")
+endif()
+
+if (MSVC)
+ string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
+ string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_MINSIZEREL ${CMAKE_CXX_FLAGS_MINSIZEREL})
+ string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
+ string(REPLACE "/MD" "/MT" CMAKE_CXX_FLAGS_RELWITHDEBINFO ${CMAKE_CXX_FLAGS_RELWITHDEBINFO})
+ add_definitions("/wd4267 /wd4244 /wd4305 /Zc:strictStrings /utf-8")
+endif()
 
 if (APPLE)
  set(CMAKE_MACOSX_RPATH ON)
@@ -64,10 +78,6 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
  set(CMAKE_INSTALL_LIBDIR lib)
 endif()
 
-if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
- set(CMAKE_INSTALL_LIBDIR lib)
-endif()
-
 if (NOT DEFINED CMAKE_INSTALL_INCDIR)
  set(CMAKE_INSTALL_INCDIR include)
 endif()
@@ -79,13 +89,14 @@ if (NOT MSVC)
  install(FILES "${CMAKE_CURRENT_BINARY_DIR}/sentencepiece.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 endif()
 
-include_directories("." ${CMAKE_SOURCE_DIR} ${PROJECT_BINARY_DIR})
+include_directories("." ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_BINARY_DIR})
 
 if (SPM_BUILD_TEST)
  enable_testing()
 endif()
 
 add_subdirectory(src)
+add_subdirectory(third_party)
 
 set(CPACK_SOURCE_GENERATOR "TXZ")
 set(CPACK_GENERATOR "7Z")