clab · bskaggs · Oct 27, 2016 · Oct 27, 2016 · Oct 27, 2016 · Oct 27, 2016
diff --git a/.gitignore b/.gitignore
@@ -1,19 +1,25 @@
 # cmake stuff
+build*/
 Testing/
-cnn/Testing/
-cnn/tests.bin/
+dynet/Testing/
+dynet/tests.bin/
 CTestTestfile.cmake
 config.h
 Makefile
 CMakeCache.txt
 CMakeFiles
 cmake_install.cmake
+python/dynet.cpp
+python/dist/
+python/dyNET.egg-info/
 
 # binaries
-examples/xor
-examples/xor-xent
-examples/rnnlm
-examples/nlm
+
+#data
+rnnlm/ptb-mikolov/
+
+# Python temporary files
+*.pyc
 
 # Compiled Object files
 *.slo
@@ -44,5 +50,12 @@ examples/nlm
 *.out
 *.app
 
+# Editor stuff
+*.swp
+
+# Doc stuff
+doc/doxygen/xml
+doc/source/tutorials_notebooks
+
 .RData
 .RHistory
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "external/easyloggingpp"]
+	path = external/easyloggingpp
+	url = https://github.com/easylogging/easyloggingpp.git
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,27 @@
+language: cpp
+compiler:
+  - gcc
+
+install:
+- if [ "$CXX" = "g++" ]; then export CXX="g++-4.8" CC="gcc-4.8"; fi
+addons:
+  apt:
+    sources:
+    - ubuntu-toolchain-r-test
+    packages:
+    - gcc-4.8
+    - g++-4.8
+
+before_script:
+  - sudo add-apt-repository -y ppa:boost-latest/ppa
+  - sudo apt-get update -qq
+  - sudo apt-get install libboost-filesystem1.55-dev libboost-program-options1.55-dev libboost-serialization1.55-dev libboost-test1.55-dev libboost-regex1.55-dev
+  - hg clone https://bitbucket.org/eigen/eigen
+  - mkdir build
+  - cd build
+  - cmake .. -DEIGEN3_INCLUDE_DIR=eigen
+
+script:
+  - make
+  - make test
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,12 +1,83 @@
-project(cnn)
+project(dynet)
 cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
-set(CMAKE_CXX_FLAGS "-Wall -std=c++11 -O3 -g")
+
+# DYNET uses Eigen which exploits modern CPU architectures. To get the
+# best possible performance, the following are recommended:
+#   1. use very recent versions of gcc or Clang to build
+#   2. use very recent versions of Eigen (ideally the dev version)
+#   3. try compiler options like -march=native or other architecture
+#      flags (the compiler does not always make the best configuration
+#      decisions without help)
+
+# NOTE: This seems to be causing problems with linking before using
+#       make install. It is allegedly preferred, but probably doesn't
+#       suit our model of not installing the library most of the time.
+set(CMAKE_MACOSX_RPATH 0)
+
+function(find_mkl)
+  set(MKL_ARCH intel64)
+  find_path(MKL_INCLUDE_DIR mkl.h
+            PATHS ${MKL_ROOT} ${MKL_ROOT}/include)
+  find_library(MKL_CORE_LIB NAMES mkl_intel_lp64 mkl_intel_thread mkl_core
+               PATHS ${MKL_ROOT} ${MKL_ROOT}/lib/${MKL_ARCH}
+                     ${MKL_ROOT}/lib #OSX
+               DOC "MKL core library path")
+
+  find_library(MKL_COMPILER_LIB NAMES iomp5 libiomp5md
+               PATHS ${MKL_ROOT} ${MKL_ROOT}/../compiler/lib/${MKL_ARCH}              #Windows
+                     ${MKL_ROOT}/../compilers_and_libraries/linux/lib/${MKL_ARCH}_lin #Linux
+                     ${MKL_ROOT}/../compilers_and_libraries/mac/lib                   #OSX
+               DOC "MKL compiler lib (for threaded MKL)")
+
+  if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_COMPILER_LIB)
+    get_filename_component(MKL_CORE_LIB_DIR ${MKL_CORE_LIB} DIRECTORY)
+    get_filename_component(MKL_COMPILER_LIB_DIR ${MKL_COMPILER_LIB} DIRECTORY)
+    get_filename_component(MKL_COMPILER_LIB_FILE ${MKL_COMPILER_LIB} NAME)
+    message(STATUS "Found MKL\n   * include: ${MKL_INCLUDE_DIR},\n   * core library dir: ${MKL_CORE_LIB_DIR},\n   * compiler library: ${MKL_COMPILER_LIB}")
+
+    # Due to a conflict with /MT and /MD, MSVC needs mkl_intel_lp64 linked last, or we can change individual
+    # projects to use /MT (mkl_intel_lp64 linked with /MT, default MSVC projects use /MD), or we can instead
+    # link to the DLL versions. For now I'm opting for this solution which seems to work with projects still
+    # at their default /MD. Linux build requires the mkl_intel_lp64 to be linked first. So...:
+    if(MSVC)
+      set(LIBS ${LIBS} mkl_intel_thread mkl_core mkl_intel_lp64 ${MKL_COMPILER_LIB_FILE} PARENT_SCOPE)
+    else()
+      set(LIBS ${LIBS} mkl_intel_lp64 mkl_intel_thread mkl_core ${MKL_COMPILER_LIB_FILE} PARENT_SCOPE)
+    endif()
+    include_directories(${MKL_INCLUDE_DIR})
+    link_directories(${MKL_CORE_LIB_DIR} ${MKL_COMPILER_LIB_DIR})
+    set(MKL_LINK_DIRS ${MKL_CORE_LIB_DIR} ${MKL_COMPILER_LIB_DIR} PARENT_SCOPE) # Keeping this for python build
+  else()
+    message(FATAL_ERROR "Failed to find MKL in path: ${MKL_ROOT} (Did you set MKL_ROOT properly?)")
+  endif()
+endfunction()
+
+######## Cross-compiler, cross-platform options
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DEIGEN_FAST_MATH")
+if (MKL OR MKL_ROOT)
+  find_mkl()  # sets include/lib directories and sets ${LIBS} needed for linking
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DEIGEN_USE_MKL_ALL")
+endif()
+
+
+######## Platform-specific options
+if(WIN32)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNOMINMAX")   # Disable min/max macros in windef.h
+endif()
+
+######## Compiler-specific options
+if(MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W1 -DEIGEN_HAS_C99_MATH /MP")   # -Wall produces 20k warnings
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -funroll-loops -fno-finite-math-only -Wall -Wno-missing-braces -std=c++11 -Ofast -g -march=native")
+endif()
 
 enable_testing()
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}
+                    ${PROJECT_SOURCE_DIR}/external/easyloggingpp/src)
 
 function(find_cudnn)
   set(CUDNN_ROOT "" CACHE PATH "CUDNN root path")
@@ -29,32 +100,76 @@ function(find_cudnn)
 endfunction()
 
 # look for Boost
-find_package(Boost COMPONENTS program_options serialization unit_test_framework REQUIRED)
+if(DEFINED ENV{BOOST_ROOT})
+  set(Boost_NO_SYSTEM_PATHS ON)
+  if(DEFINED ${Boost_INCLUDE_DIR})
+    get_filename_component(Boost_INCLUDE_DIR "${Boost_INCLUDE_DIR}" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
+  endif()
+endif()
+set(Boost_REALPATH ON)
+find_package(Boost COMPONENTS program_options regex serialization REQUIRED)
+message("-- Boost dir is " ${Boost_INCLUDE_DIR})
 include_directories(${Boost_INCLUDE_DIR})
+if(MSVC)
+  # Boost does auto-linking when using a compiler like Microsoft Visual C++, we just need to help it find the libraries
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LIBPATH:${Boost_LIBRARY_DIRS}")
+  set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /LIBPATH:${Boost_LIBRARY_DIRS}")
+else()
+  set(LIBS ${LIBS} ${Boost_LIBRARIES})
+endif()
+# trouble shooting:
+# if boost library cannot be found, in addition to install boost library
+# check if environment variables are set
+#
+# to set boost root and its library root in environment variable, use
+# for example
+# echo "export BOOST_LIBRARYDIR=/usr/local/lib" >> ~/.bashrc
+# echo "export BOOST_ROOT=/cygdrive/d/tools/boost_1_58_0/boost_1_58_0" >> ~/.bashrc
+# then run source ~/.bashrc to have those environment variable effective immediately
 
-# look for Minerva
-find_package(Minerva)
-if(MINERVA_FOUND)
-  include_directories(${MINERVA_INCLUDE_DIR})
-  set(HAVE_MINERVA_H 1)
-  if (ENABLE_CUDA)
-    set(CUDA_TOOLKIT_ROOT_DIR ${CUDA_ROOT})
-    find_package(CUDA REQUIRED)
-    include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
-    add_definitions(-DHAS_CUDA)
-    find_cudnn()
-    include_directories(SYSTEM ${CUDNN_INCLUDE_DIRS})
-  else ()
-    message(STATUS "CUDA not enabled (use -DENABLE_CUDA=1)")
-  endif()
-endif(MINERVA_FOUND)
+if(BACKEND)
+  message("-- BACKEND: ${BACKEND}")
+else()
+  message("-- BACKEND not specified, defaulting to eigen.")
+  set(BACKEND "eigen")
+endif()
+
+if(BACKEND MATCHES "^eigen$")
+  set(WITH_EIGEN_BACKEND 1)
+elseif(BACKEND MATCHES "^cuda$")
+  set(WITH_CUDA_BACKEND 1)
+else()
+  message(SEND_ERROR "BACKEND must be eigen or cuda")
+endif()
+
+if (WITH_CUDA_BACKEND)
+  find_package(CUDA REQUIRED)
+  set(CUDA_TOOLKIT_ROOT_DIR ${CUDA_ROOT})
+  include_directories(SYSTEM ${CUDA_INCLUDE_DIRS})
+  #list(APPEND CUDA_LIBRARIES /usr/lib64/libpthread.so)
+  MESSAGE("CUDA_LIBRARIES: ${CUDA_LIBRARIES}")
+  list(REMOVE_ITEM CUDA_LIBRARIES -lpthread)
+  set(LIBS ${LIBS} ${CUDA_LIBRARIES})
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DEIGEN_HAS_CUDA_FP16 -DEIGEN_USE_GPU")
+  #find_cudnn()
+  #include_directories(SYSTEM ${CUDNN_INCLUDE_DIRS})
+endif()
 
 # look for Eigen
 find_package(Eigen3 REQUIRED)
+get_filename_component(EIGEN3_INCLUDE_DIR "${EIGEN3_INCLUDE_DIR}" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
+message("-- Eigen dir is " ${EIGEN3_INCLUDE_DIR})
 include_directories(${EIGEN3_INCLUDE_DIR})
 
+FIND_PACKAGE(Threads REQUIRED)
+set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT})
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-add_subdirectory(cnn)
+add_subdirectory(dynet)
+add_subdirectory(tests)
 add_subdirectory(examples)
-
+add_subdirectory(tutorial)
+add_subdirectory(python)
+enable_testing()
diff --git a/LICENSE b/LICENSE
@@ -186,7 +186,7 @@ Apache License
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2015 Chris Dyer
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

diff --git a/MEM.notes b/MEM.notes
@@ -0,0 +1,10 @@
+The code that computes the l2 norm of the gradient is going to need
+scratch space on every device that DYNET is using that has a parameter.
+1) devices should know whether they have parameters/gradients
+
+alignment code is hidden away. it's all hard coded, but it looks like
+Intel at least is getting more foregiving about alingment problems so
+we might not notice opportunities for speedups if something changes.
+GPU memory is aligned mostly by CUDA
+
+the MP stuff needs to be tested by Austin.
diff --git a/README.md b/README.md
@@ -1,18 +1,18 @@
-# cnn
-C++ neural network library
+# DyNet
+The Dynamic Neural Network Toolkit
 
-#### Building
+DyNet (formerly known as [cnn](http://github.com/clab/cnn-v1)) is a neural network library developed by Carnegie Mellon University and many others. It is written in C++ (with bindings in Python) and is designed to be efficient when run on either CPU or GPU, and to work well with networks that have dynamic structures that change for every training instance. For example, these kinds of networks are particularly important in natural language processing tasks, and DyNet has been used to build state-of-the-art systems for [syntactic parsing](https://github.com/clab/lstm-parser), [machine translation](https://github.com/neubig/lamtram), [morphological inflection](https://github.com/mfaruqui/morph-trans), and many other application areas.
 
-In `src`, you need to first use [`cmake`](http://www.cmake.org/) to generate the makefiles
+Read the [documentation](http://dynet.readthedocs.io/en/latest/) to get started, and feel free to contact the [dynet-users group](https://groups.google.com/forum/#!forum/dynet-users) group with any questions (if you want to receive email make sure to select "all email" when you sign up). We greatly appreciate any bug reports and contributions, which can be made by filing an issue or making a pull request through the [github page](http://github.com/clab/dynet).
 
-    cmake -DEIGEN3_INCLUDE_DIR=/Users/cdyer/software/eigen-eigen-36fd1ba04c12 -G 'Unix Makefiles'
+You can also read more technical details in our [technical report](https://arxiv.org/abs/1701.03980). If you use DyNet for research, please cite this report as follows:
 
-Then to compile, run
-
-    make -j 2
-    make test
-
-If you want to see the compile commands that are used, you can run
-
-    make VERBOSE=1
+    @article{dynet,
+      title={DyNet: The Dynamic Neural Network Toolkit},
+      author={Graham Neubig and Chris Dyer and Yoav Goldberg and Austin Matthews and Waleed Ammar and Antonios Anastasopoulos and Miguel Ballesteros and David Chiang and Daniel Clothiaux and Trevor Cohn and Kevin Duh and Manaal Faruqui and Cynthia Gan and Dan Garrette and Yangfeng Ji and Lingpeng Kong and Adhiguna Kuncoro and Gaurav Kumar and Chaitanya Malaviya and Paul Michel and Yusuke Oda and Matthew Richardson and Naomi Saphra and Swabha Swayamdipta and Pengcheng Yin},
+      journal={arXiv preprint arXiv:1701.03980},
+      year={2017}
+    }
 
+[![Build Status](https://travis-ci.org/clab/dynet.svg?branch=master)](https://travis-ci.org/clab/dynet)
+[![Doc build Status](https://readthedocs.org/projects/dynet/badge/?version=latest)](http://dynet.readthedocs.io/en/latest/)
diff --git a/cmake/FindTH.cmake → cmake/FindDyNet.cmake b/cmake/FindTH.cmake → cmake/FindDyNet.cmake
diff --git a/cmake/FindEigen3.cmake b/cmake/FindEigen3.cmake
@@ -9,6 +9,12 @@
 #  EIGEN3_FOUND - system has eigen lib with correct version
 #  EIGEN3_INCLUDE_DIR - the eigen include directory
 #  EIGEN3_VERSION - eigen version
+#
+# This module reads hints about search locations from 
+# the following enviroment variables:
+#
+# EIGEN3_ROOT
+# EIGEN3_ROOT_DIR
 
 # Copyright (c) 2006, 2007 Montel Laurent, <[email protected]>
 # Copyright (c) 2008, 2009 Gael Guennebaud, <[email protected]>
@@ -62,6 +68,9 @@ if (EIGEN3_INCLUDE_DIR)
 else (EIGEN3_INCLUDE_DIR)
 
   find_path(EIGEN3_INCLUDE_DIR NAMES signature_of_eigen3_matrix_library
+      HINTS
+      ENV EIGEN3_ROOT 
+      ENV EIGEN3_ROOT_DIR
       PATHS
       ${CMAKE_INSTALL_PREFIX}/include
       ${KDE4_INCLUDE_DIR}

diff --git a/cmake/FindMinerva.cmake b/cmake/FindMinerva.cmake
diff --git a/cmake/FindTH++.cmake b/cmake/FindTH++.cmake