Merge stable 1.1.0 release of MXNet

https://github.com/apache/incubator-mxnet/releases/tag/1.1.0

A quite extended amount of manual work was needed here to resolve all
the conflicts, almost none due to the tsunami fork!

Some diffs seemed to survive the merge, and were fixed via manually
checking out the files in question from the upstream 1.1.0 tag.

* 3rdparty/cub ()...05eb57fa(05eb57fa) (1 commits)
  > Merge pull request #1 from ptrendx/update

* 3rdparty/googletest ()...release-1.8.0(ec44c6c) (1 commits)
  > Merge pull request apache#821 from mazong1123/master

* 3rdparty/openmp ()...37c7212(37c7212) (1 commits)
  > Merging r317115:

* 3rdparty/cub ()...05eb57fa(05eb57fa) (1 commits)
  > Merge pull request #1 from ptrendx/update

* 3rdparty/googletest ()...release-1.8.0(ec44c6c) (1 commits)
  > Merge pull request apache#821 from mazong1123/master

* 3rdparty/openmp ()...37c7212(37c7212) (1 commits)
  > Merging r317115:

* dmlc-core 87b7ffa(87b7ffa)...6389c10(6389c10) (20 commits)
  > Fix symbol demangling on stacktraces under Linux. (apache#356)
  > Add more environment variables to support 3rd-party S3 implementations (apache#354)
  > add qid as ranklib format (apache#317)
  > Make choice of archiver configurable (apache#352)
  > Fix warnings in unittest_lockfree.cc, fix travis CI (apache#347)
  (...)

* mshadow 2d7780c(2d7780c)...16ac8cd(16ac8cd) (5 commits)
  > Add fully qualified type on define default_real_t (apache#317)
  > fix AddTakeGradLargeBatch for CPU. close apache#235 (apache#316)
  > fix float16 min and max values (apache#315)
  > [Windows] make Packet::size constexpr (apache#313)
  > add GetRndEngine (apache#308)

* nnvm e4a138a(e4a138a)...7a052d6(7a052d6) (3 commits)
  > [SYMBOL] Add __iter__ and GetChildren for symbol (apache#268)
  > API call to get symbol output count (apache#270)
  > [FRONTEND] Fix mxnet multi outputs (apache#271)

.gitignore

@@ -100,6 +100,10 @@ input.txt*
 # ctags
 tags
 
+# cscope
+cscope.out
+cscope.files
+
 # Scala package
 *.class
 scala-package/*/target/

.gitmodules

@@ -14,8 +14,14 @@
 	path = dlpack
 	url = https://github.com/dmlc/dlpack
 [submodule "cub"]
-	path = cub
+	path = 3rdparty/cub
 	url = https://github.com/dmlc/cub
+[submodule "3rdparty/openmp"]
+	path = 3rdparty/openmp
+	url = https://github.com/llvm-mirror/openmp
+[submodule "3rdparty/googletest"]
+	path = 3rdparty/googletest
+	url = https://github.com/google/googletest.git
 [submodule "beaver"]
 	path = beaver
 	url = https://github.com/sociomantic-tsunami/beaver.git

CMakeLists.txt

@@ -38,26 +38,31 @@ mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and foun
 mxnet_option(USE_OPERATOR_TUNING  "Enable auto-tuning of operators" ON AND NOT MSVC)
 mxnet_option(USE_GPERFTOOLS       "Build with GPerfTools support (if found)" ON)
 mxnet_option(USE_JEMALLOC         "Build with Jemalloc support"   ON)
-mxnet_option(USE_PROFILER         "Build with Profiler support"   OFF)
+mxnet_option(USE_PROFILER         "Build with Profiler support"   ON)
 mxnet_option(USE_DIST_KVSTORE     "Build with DIST_KVSTORE support" OFF)
 mxnet_option(USE_PLUGINS_WARPCTC  "Use WARPCTC Plugins" OFF)
 mxnet_option(USE_PLUGIN_CAFFE     "Use Caffe Plugin" OFF)
 mxnet_option(USE_CPP_PACKAGE      "Build C++ Package" OFF)
 mxnet_option(USE_MXNET_LIB_NAMING "Use MXNet library naming conventions." ON)
 mxnet_option(USE_GPROF            "Compile with gprof (profiling) flag" OFF)
 mxnet_option(USE_VTUNE            "Enable use of Intel Amplifier XE (VTune)" OFF) # one could set VTUNE_ROOT for search path
+mxnet_option(ENABLE_CUDA_RTC      "Build with CUDA runtime compilation support" ON)
 mxnet_option(INSTALL_EXAMPLES     "Install the example source files." OFF)
+mxnet_option(USE_SIGNAL_HANDLER   "Print stack traces on segfaults." OFF)
 
 
 
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
   include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
 endif()
 
-set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
-
-
+if(MSVC)
+  set(SYSTEM_ARCHITECTURE x86_64)
+else()
+  EXECUTE_PROCESS( COMMAND uname -m COMMAND tr -d '\n' OUTPUT_VARIABLE SYSTEM_ARCHITECTURE)
+endif()
 
+set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}")
 
 SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH")
 
@@ -84,6 +89,9 @@ else(MSVC)
   check_cxx_compiler_flag("-std=c++0x"   SUPPORT_CXX0X)
   check_cxx_compiler_flag("-msse2"       SUPPORT_MSSE2)
   set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -fPIC -Wno-sign-compare")
+  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
+    set(CMAKE_C_FLAGS "-Wno-braced-scalar-init")
+  endif()
   if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0 -g")
   elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
@@ -179,12 +187,12 @@ endif()
 list(APPEND mxnet_LINKER_LIBS ${mshadow_LINKER_LIBS})
 
 foreach(var ${C_CXX_INCLUDE_DIRECTORIES})
- 	include_directories(${var})
+    include_directories(${var})
 endforeach()
 
 include_directories("include")
 include_directories("mshadow")
-include_directories("cub")
+include_directories("3rdparty/cub")
 include_directories("nnvm/include")
 include_directories("dmlc-core/include")
 include_directories("dlpack/include")
@@ -194,9 +202,13 @@ include_directories("dlpack/include")
 #  add_subdirectory(dlpack)
 #endif()
 
+# Prevent stripping out symbols (operator registrations, for example)
 if(NOT MSVC AND NOT APPLE)
   set(BEGIN_WHOLE_ARCHIVE -Wl,--whole-archive)
   set(END_WHOLE_ARCHIVE -Wl,--no-whole-archive)
+elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  # using regular Clang or AppleClang
+  set(BEGIN_WHOLE_ARCHIVE -Wl,-force_load)
 endif()
 
 if(UNIX)
@@ -248,13 +260,6 @@ if(USE_OPENCV)
   message(STATUS " OpenCV_LIBS=${OpenCV_LIBS}")
   message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
   add_definitions(-DMXNET_USE_OPENCV=1)
-  if(NOT MSVC)
-    if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-undefined,error")
-    else()
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-undefined")
-    endif()
-  endif()
 else(USE_OPENCV)
   message(STATUS "OpenCV Disabled")
   add_definitions(-DMXNET_USE_OPENCV=0)
@@ -263,11 +268,13 @@ endif()
 # ---[ OpenMP
 if(USE_OPENMP)
   find_package(OpenMP REQUIRED)
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/openmp/CMakeLists.txt)
+  # This should build on Windows, but there's some problem and I don't have a Windows box, so
+  # could a Windows user please fix?
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT MSVC)
     # Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
     set(OPENMP_STANDALONE_BUILD TRUE)
-    set(LIBOMP_ENABLE_SHARED FALSE)
-    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/openmp)
+    set(LIBOMP_ENABLE_SHARED TRUE)
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp)
     list(REMOVE_ITEM mxnet_LINKER_LIBS iomp5)
     list(APPEND mxnet_LINKER_LIBS omp)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
@@ -306,6 +313,15 @@ if(USE_JEMALLOC)
 endif()
 
 include(CTest)
+set(GTEST_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/googletest/googletest")
+set(GTEST_INCLUDE_DIR ${GTEST_ROOT}/include)
+#set(GTEST_BOTH_LIBRARIES gtest gtest_main)
+set(GTEST_LIBRARIES gtest gtest_main)
+set(GTEST_MAIN_LIBRARY gtest_main)
+set(GTEST_LIBRARY gtest)
+
+add_subdirectory(${GTEST_ROOT})
+find_package(GTest REQUIRED)
 
 # cudnn detection
 if(USE_CUDNN AND USE_CUDA)
@@ -314,7 +330,7 @@ if(USE_CUDNN AND USE_CUDA)
     add_definitions(-DUSE_CUDNN)
     include_directories(SYSTEM ${CUDNN_INCLUDE})
     list(APPEND mxnet_LINKER_LIBS ${CUDNN_LIBRARY})
-	  add_definitions(-DMSHADOW_USE_CUDNN=1)
+      add_definitions(-DMSHADOW_USE_CUDNN=1)
   endif()
 endif()
 
@@ -354,20 +370,20 @@ assign_source_group("Include" ${GROUP_Include})
 assign_source_group("CUDA" ${GROUP_CUDA})
 
 if(USE_PLUGINS_WARPCTC)
-	set(WARPCTC_INCLUDE  "" CACHE PATH "WARPCTC include")
+    set(WARPCTC_INCLUDE  "" CACHE PATH "WARPCTC include")
     set(WARPCTC_LIB_DEBUG  "" CACHE FILEPATH "WARPCTC lib")
     set(WARPCTC_LIB_RELEASE  "" CACHE FILEPATH "WARPCTC lib")
 
 
-	include_directories(SYSTEM ${WARPCTC_INCLUDE})
-	list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})
-	FILE(GLOB_RECURSE PLUGINS_SOURCE "plugin/warpctc/*.cc" "plugin/warpctc/*.h")
-	FILE(GLOB_RECURSE PLUGINS_CUSRC "plugin/warpctc/*.cu")
-	list(APPEND SOURCE ${PLUGINS_SOURCE})
-	list(APPEND CUDA ${PLUGINS_CUSRC})
+    include_directories(SYSTEM ${WARPCTC_INCLUDE})
+    list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})
+    FILE(GLOB_RECURSE PLUGINS_SOURCE "plugin/warpctc/*.cc" "plugin/warpctc/*.h")
+    FILE(GLOB_RECURSE PLUGINS_CUSRC "plugin/warpctc/*.cu")
+    list(APPEND SOURCE ${PLUGINS_SOURCE})
+    list(APPEND CUDA ${PLUGINS_CUSRC})
 endif()
 
-if(USE_OPERATOR_TUNING)
+if(USE_OPERATOR_TUNING AND USE_OPENMP)
   add_definitions(-DMXNET_USE_OPERATOR_TUNING=1)
 endif()
 
@@ -407,11 +423,11 @@ if(USE_PLUGIN_CAFFE)
 endif()
 
 if (NOT (EXTRA_OPERATORS STREQUAL ""))
-	mxnet_source_group("Extra"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cc")
-	mxnet_source_group("Extra\\Cuda"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cu")
-	FILE(GLOB_RECURSE EXTRA_SRC "${EXTRA_OPERATORS}/*.cc")
-	FILE(GLOB_RECURSE EXTRA_CUSRC "${EXTRA_OPERATORS}/*.cu")
-	list(APPEND SOURCE ${EXTRA_SRC} ${EXTRA_CUSRC})
+    mxnet_source_group("Extra"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cc")
+    mxnet_source_group("Extra\\Cuda"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cu")
+    FILE(GLOB_RECURSE EXTRA_SRC "${EXTRA_OPERATORS}/*.cc")
+    FILE(GLOB_RECURSE EXTRA_CUSRC "${EXTRA_OPERATORS}/*.cu")
+    list(APPEND SOURCE ${EXTRA_SRC} ${EXTRA_CUSRC})
 endif()
 
 if(MSVC)
@@ -430,24 +446,35 @@ if(USE_CUDA)
     string(REPLACE ";" " " NVCC_FLAGS_ARCH "${NVCC_FLAGS_ARCH}")
     set(CMAKE_CUDA_FLAGS "${NVCC_FLAGS_ARCH}")
     set(CMAKE_CUDA_FLAGS_RELEASE "${NVCC_FLAGS_ARCH} -use_fast_math")
-    list(APPEND mxnet_LINKER_LIBS nvrtc cuda cublas cufft cusolver curand)
+    list(APPEND mxnet_LINKER_LIBS cublas cufft cusolver curand)
+    if(ENABLE_CUDA_RTC)
+        list(APPEND mxnet_LINKER_LIBS nvrtc cuda)
+        add_definitions(-DMXNET_ENABLE_CUDA_RTC=1)
+    endif()
     list(APPEND SOURCE ${CUDA})
     add_definitions(-DMXNET_USE_CUDA=1)
   else()
     list(APPEND CUDA_INCLUDE_DIRS ${INCLUDE_DIRECTORIES})
     # define preprocessor macro so that we will not include the generated forcelink header
     mshadow_cuda_compile(cuda_objs ${CUDA})
     if(MSVC)
-        FIND_LIBRARY(CUDA_nvrtc_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64"  "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
-        list(APPEND mxnet_LINKER_LIBS ${CUDA_nvrtc_LIBRARY})
-        set(CUDA_cuda_LIBRARY "${CUDA_nvrtc_LIBRARY}/../cuda.lib")
-        list(APPEND mxnet_LINKER_LIBS ${CUDA_cuda_LIBRARY})
+        if(ENABLE_CUDA_RTC)
+            FIND_LIBRARY(CUDA_nvrtc_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64"  "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
+            list(APPEND mxnet_LINKER_LIBS ${CUDA_nvrtc_LIBRARY})
+            set(CUDA_cuda_LIBRARY "${CUDA_nvrtc_LIBRARY}/../cuda.lib")
+            list(APPEND mxnet_LINKER_LIBS ${CUDA_cuda_LIBRARY})
+            add_definitions(-DMXNET_ENABLE_CUDA_RTC=1)
+        endif()
         FIND_LIBRARY(CUDA_cufft_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64"  "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
         list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") # For fft operator
         FIND_LIBRARY(CUDA_cusolver_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64"  "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32")
         list(APPEND mxnet_LINKER_LIBS "${CUDA_cusolver_LIBRARY}/../cusolver.lib") # For cusolver
     else(MSVC)
-        list(APPEND mxnet_LINKER_LIBS nvrtc cuda cufft cusolver)
+        list(APPEND mxnet_LINKER_LIBS cufft cusolver)
+        if(ENABLE_CUDA_RTC)
+            list(APPEND mxnet_LINKER_LIBS nvrtc cuda)
+            add_definitions(-DMXNET_ENABLE_CUDA_RTC=1)
+        endif()
         link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
     endif()
     list(APPEND SOURCE ${cuda_objs} ${CUDA})
@@ -491,7 +518,7 @@ endif()
 
 set(MXNET_INSTALL_TARGETS mxnet)
 if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" AND USE_MXNET_LIB_NAMING)
-  add_library(mxnet MODULE ${SOURCE})
+  add_library(mxnet SHARED ${SOURCE})
   add_library(mxnet_static STATIC ${SOURCE})
 else()
   if(UNIX)
@@ -508,7 +535,7 @@ else()
 endif()
 
 if(USE_CUDA)
-  if(FIRST_CUDA)
+  if(FIRST_CUDA AND MSVC)
     target_compile_options(mxnet PUBLIC "$<$<CONFIG:DEBUG>:-Xcompiler=-MTd>")
     target_compile_options(mxnet PUBLIC "$<$<CONFIG:RELEASE>:-Xcompiler=-MT>")
   endif()
@@ -542,14 +569,20 @@ if(USE_PLUGINS_WARPCTC)
   target_link_libraries(mxnet PUBLIC optimized ${WARPCTC_LIB_RELEASE})
 endif()
 
+
+if(USE_OPENCV)
+  add_executable(im2rec "tools/im2rec.cc")
+  target_link_libraries(im2rec ${BEGIN_WHOLE_ARCHIVE} mxnet ${END_WHOLE_ARCHIVE} ${mxnet_LINKER_LIBS} ${OpenCV_LIBS} dmlc)
+endif()
+
 target_link_libraries(mxnet PUBLIC dmlc)
 
 if(MSVC AND USE_MXNET_LIB_NAMING)
   set_target_properties(mxnet PROPERTIES OUTPUT_NAME "libmxnet")
 endif()
 
 if(USE_PROFILER)
-	add_definitions(-DMXNET_USE_PROFILER)
+    add_definitions(-DMXNET_USE_PROFILER)
 endif()
 
 add_subdirectory(tests)
@@ -566,6 +599,10 @@ if (INSTALL_EXAMPLES)
   install(DIRECTORY example  DESTINATION ${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME})
 endif()
 
+if (USE_SIGNAL_HANDLER)
+    add_definitions(-DMXNET_USE_SIGNAL_HANDLER=1)
+endif()
+
 # AUTO_INSTALL_DIR -> Optional: specify post-build install direcory
 if(AUTO_INSTALL_DIR)
   # ---[ Install Includes

CODEOWNERS

@@ -1,14 +1,47 @@
-# Owners of Apache MXNet
+# Watchers and contributors to Apache MXNet repo directories/packages/files
+# Please see documentation of use of CODEOWNERS file at
+# https://help.github.com/articles/about-codeowners/ and
+# https://github.com/blog/2392-introducing-code-owners
+#
+# Anybody can add themselves or a team as additional watcher or contributor 
+# to get notified about changes in a specific package.
+# See https://help.github.com/articles/about-teams how to setup teams.
+
 
 # Global owners
 *			@apache/mxnet-committers
 
-# Owners of language bindings
-R-package/*		@thirdwing
-scala-package/*		@javelinjs
-perl-package/*		@sergeykolychev
+# Language bindings
+/R-package/       @thirdwing
+/scala-package/   @yzhliu
+/perl-package/    @sergeykolychev
+/python/          @szha
+
+# CMake
+CMakeLists.txt    @cjolivier01 @szha
+/cmake/           @cjolivier01 @szha
+
+# MXNet CI
+/tests/ci_build/    @marcoabreu
+Jenkinsfile         @marcoabreu
+.travis.yml         @marcoabreu
+appveyor.yml        @marcoabreu
+
+# Build logic
+Makefile          @szha
+prepare_mkl.sh    @szha
+
+# Docs
+/docs/            @szha
+
+# Submodules
+.gitmodules       @szha
+
+# Examples
+/example/         @szha
 
-# CMake owners
-CMakeLists.txt		@cjolivier01
-cmake/*			@cjolivier01
+# Tools
+/tools/           @szha
 
+# Github templates
+/.github/         @szha