Reusable

CMakeLists.txt

@@ -1,8 +1,12 @@
 project(lstm-parser)
 cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE RelWithDebInfo)
+endif(NOT CMAKE_BUILD_TYPE)
+
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
-set(CMAKE_CXX_FLAGS "-Wall -std=c++11 -O3 -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -std=c++14")
 
 enable_testing()
 
@@ -16,7 +20,7 @@ if(DEFINED ENV{BOOST_ROOT})
   set(Boost_NO_SYSTEM_PATHS ON)
 endif()
 set(Boost_REALPATH ON)
-find_package(Boost COMPONENTS program_options serialization iostreams REQUIRED)
+find_package(Boost COMPONENTS program_options serialization iostreams regex filesystem REQUIRED)
 include_directories(${Boost_INCLUDE_DIR})
 set(LIBS ${LIBS} ${Boost_LIBRARIES})
 
@@ -26,6 +30,6 @@ include_directories(${EIGEN3_INCLUDE_DIR})
 
 #configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)
 
-add_subdirectory(cnn/cnn)
+add_subdirectory(cnn)
 # add_subdirectory(cnn/examples)
-add_subdirectory(parser)
+add_subdirectory(parser)

README.md

@@ -25,7 +25,7 @@ Given a `training.conll` file and a `development.conll` formatted according to t
     java -jar ParserOracleArcStdWithSwap.jar -t -1 -l 1 -c training.conll > trainingOracle.txt
     java -jar ParserOracleArcStdWithSwap.jar -t -1 -l 1 -c development.conll > devOracle.txt
 
-    parser/lstm-parse -P -t trainingOracle.txt -d devOracle.txt --hidden_dim 100 --lstm_input_dim 100 -w sskip.100.vectors --rel_dim 20 --action_dim 20
+    parser/lstm-parse --train -t trainingOracle.txt -d devOracle.txt --hidden_dim 100 --lstm_input_dim 100 --words sskip.100.vectors --rel_dim 20 --action_dim 20 --use_pos_tags
 
 Link to the word vectors used in the ACL 2015 paper for English:  [sskip.100.vectors](https://drive.google.com/file/d/0B8nESzOdPhLsdWF2S1Ayb1RkTXc/view?usp=sharing).
 
@@ -43,7 +43,7 @@ There is a pretrained model for English [here](http://www.cs.cmu.edu/~jdunietz/h
 
 Given a `test.conll` file formatted according to the [CoNLL data format](http://ilk.uvt.nl/conll/#dataformat):
 
-    parser/lstm-parse -m english_pos_2_32_100_20_100_12_20.params -t test.conll
+    parser/lstm-parse -m english_pos_2_32_100_20_100_12_20.params -T test.conll -s
 
 If you are not using the pretrained model, you will need to replace the `.params` argument with the name of your own trained model file.
 

cnn/CMakeLists.txt

@@ -1,6 +1,10 @@
 project(cnn)
 cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
 
+if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CMAKE_BUILD_TYPE RelWithDebInfo)
+endif(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "Debug")
+
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 
 # CNN uses Eigen which exploits modern CPU architectures. To get the
@@ -10,7 +14,7 @@ set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
 #   3. try compiler options like -march=native or other architecture
 #      flags (the compiler does not always make the best configuration
 #      decisions without help)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -funroll-loops -Wall -std=c++11 -Ofast -g -DEIGEN_FAST_MATH -march=native")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -funroll-loops -Wall -std=c++14 -Ofast -g -DEIGEN_FAST_MATH -march=native")
 
 enable_testing()
 
@@ -63,9 +67,11 @@ else()
 endif()
 
 if(BACKEND MATCHES "^eigen$")
-  set(WITH_EIGEN_BACKEND 1)
+  set(WITH_CUDA_BACKEND 0 CACHE INTERNAL "" FORCE)
+  set(WITH_EIGEN_BACKEND 1 CACHE INTERNAL "" FORCE)
 elseif(BACKEND MATCHES "^cuda$")
-  set(WITH_CUDA_BACKEND 1)
+  set(WITH_CUDA_BACKEND 1 CACHE INTERNAL "" FORCE)
+  set(WITH_EIGEN_BACKEND 0 CACHE INTERNAL "" FORCE)
 else()
   message(SEND_ERROR "BACKEND must be eigen or cuda")
 endif()
@@ -93,8 +99,12 @@ set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT})
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
+option(CNN_CORE_ONLY "If off, won't build extra dirs like tests and examples" ON)
+
 add_subdirectory(cnn)
-add_subdirectory(tests)
-add_subdirectory(examples)
-add_subdirectory(rnnlm)
-enable_testing()
+if(NOT CNN_CORE_ONLY)
+    add_subdirectory(tests)
+    add_subdirectory(examples)
+    add_subdirectory(rnnlm)
+    enable_testing()
+endif(NOT CNN_CORE_ONLY)

cnn/cnn/CMakeLists.txt

@@ -69,6 +69,8 @@ set(cnn_library_HDRS
     training.h
 )
 
+option(CNN_SHARED "Whether to build CNN shared libs" OFF)
+
 if(WITH_CUDA_BACKEND)
   list(APPEND cnn_library_SRCS
        cuda.cc)
@@ -99,20 +101,24 @@ file(GLOB TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} tests/*.cc)
 # actual target:
 add_library(cnn STATIC ${cnn_library_SRCS} ${cnn_library_HDRS})
 target_link_libraries(cnn ${LIBS})
-if(WITH_CUDA_BACKEND)
+if(CNN_SHARED)
+  if(WITH_CUDA_BACKEND)
 	add_library(gcnn_shared SHARED ${cnn_library_SRCS} ${cnn_library_HDRS})
 	target_link_libraries(gcnn_shared ${LIBS})
-else()
+  else()
 	add_library(cnn_shared SHARED ${cnn_library_SRCS} ${cnn_library_HDRS})
 	target_link_libraries(cnn_shared ${LIBS})
-endif(WITH_CUDA_BACKEND)
+  endif(WITH_CUDA_BACKEND)
+endif(CNN_SHARED)
 #add_library(cnn ${cnn_library_SRCS} ${cnn_library_HDRS} ${LIBS})
 if(WITH_CUDA_BACKEND)
   set(CUDA_SEPARABLE_COMPILATION ON)
   list(APPEND CUDA_NVCC_FLAGS "-gencode;arch=compute_20,code=sm_20;-gencode;arch=compute_30,code=sm_30;-gencode;arch=compute_35,code=sm_35;-gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_52,code=sm_52;-gencode;arch=compute_52,code=compute_52;-std=c++11;-O2;-DVERBOSE;-Xcompiler;-fpic")
   SET(CUDA_PROPAGATE_HOST_FLAGS OFF)
   cuda_add_library(cnncuda STATIC gpu-ops.cu)
-  cuda_add_library(cnncuda_shared SHARED gpu-ops.cu)
+  if(CNN_SHARED)
+    cuda_add_library(cnncuda_shared SHARED gpu-ops.cu)
+  endif(CNN_SHARED)
 endif(WITH_CUDA_BACKEND)
 
 install(FILES ${cnn_library_HDRS} DESTINATION include/cnn)

cnn/cnn/aligned-mem-pool.h

@@ -8,6 +8,8 @@ namespace cnn {
 
 class AlignedMemoryPool {
  public:
+  typedef size_t PoolState;
+
   explicit AlignedMemoryPool(size_t cap, MemAllocator* a) : a(a) {
     sys_alloc(cap);
     zero_all();
@@ -36,6 +38,14 @@ class AlignedMemoryPool {
   bool is_shared() {
     return shared;
   }
+
+  PoolState get_state() const {
+    return used;
+  }
+
+  void restore_state(const PoolState& state) {
+    used = state;
+  }
  private:
   void sys_alloc(size_t cap) {
     capacity = a->round_up_align(cap);

cnn/cnn/exec.cc

@@ -10,6 +10,7 @@ ExecutionEngine::~ExecutionEngine() {}
 
 void SimpleExecutionEngine::invalidate() {
   num_nodes_evaluated = 0;
+  fxs->free();
 }
 
 const Tensor& SimpleExecutionEngine::forward() { 

cnn/cnn/init.cc

@@ -30,7 +30,7 @@ static void RemoveArgs(int& argc, char**& argv, int& argi, int n) {
   assert(argc >= 0);
 }
 
-void Initialize(int& argc, char**& argv, unsigned random_seed, bool shared_parameters) {
+unsigned Initialize(int& argc, char**& argv, unsigned random_seed, bool shared_parameters) {
   vector<Device*> gpudevices;
 #if HAVE_CUDA
   cerr << "[cnn] initializing CUDA\n";
@@ -88,6 +88,8 @@ void Initialize(int& argc, char**& argv, unsigned random_seed, bool shared_param
   kSCALAR_ONE = default_device->kSCALAR_ONE;
   kSCALAR_ZERO = default_device->kSCALAR_ZERO;
   cerr << "[cnn] memory allocation done.\n";
+
+  return random_seed;
 }
 
 void Cleanup() {

cnn/cnn/init.h

@@ -3,7 +3,7 @@
 
 namespace cnn {
 
-void Initialize(int& argc, char**& argv, unsigned random_seed = 0, bool shared_parameters = false);
+unsigned Initialize(int& argc, char**& argv, unsigned random_seed = 0, bool shared_parameters = false);
 void Cleanup();
 
 } // namespace cnn

cnn/cnn/model.cc

@@ -160,6 +160,7 @@ void LookupParameters::clear() {
 
 Model::~Model() {
   for (auto p : all_params) delete p;
+  default_device->mem->free(gradient_norm_scratch);
 }
 
 void Model::project_weights(float radius) {

cnn/cnn/model.h

@@ -61,6 +61,7 @@ struct LookupParameters : public ParametersBase {
   void squared_l2norm(float* sqnorm) const override;
   void g_squared_l2norm(float* sqnorm) const override;
   size_t size() const override;
+  size_t num_values() const { return values.size(); }
   void Initialize(unsigned index, const std::vector<float>& val);
 
   void copy(const LookupParameters & val);
@@ -103,6 +104,15 @@ struct LookupParameters : public ParametersBase {
 class Model {
  public:
   Model() : gradient_norm_scratch() {}
+  Model(const Model&) = delete;
+  Model(Model&& m) {
+    all_params = std::move(m.all_params);
+    lookup_params = std::move(m.lookup_params);
+    params = std::move(m.params);
+    // Free our scratch memory before claiming the other model's.
+    default_device->mem->free(gradient_norm_scratch);
+    gradient_norm_scratch = m.gradient_norm_scratch;
+  }
   ~Model();
   float gradient_l2_norm() const;
   void reset_gradient();

cnn/cnn/tensor.h

@@ -7,6 +7,7 @@
 #include "cnn/dim.h"
 #include "cnn/random.h"
 #include "cnn/aligned-mem-pool.h"
+#include "devices.h"
 
 #if HAVE_CUDA
 #include <cuda.h>
@@ -26,6 +27,7 @@ namespace cnn {
 #define EIGEN_BACKEND 1
 
 typedef float real;
+extern Device* default_device; // for allocating memory on a load
 
 struct Tensor {
   Tensor() = default;
@@ -160,8 +162,12 @@ struct Tensor {
     float* vc = static_cast<float*>(std::malloc(d.size() * sizeof(float)));
     ar & boost::serialization::make_array(vc, d.size());
     CUDA_CHECK(cudaMemcpyAsync(v, vc, d.size() * sizeof(float), cudaMemcpyHostToDevice));
+    free(vc);
 #else
-    v = static_cast<float*>(_mm_malloc(d.size() * sizeof(float), 32));
+    // UGLY HACK to avoid memory leak: node values and gradients don't get
+    // stored to disk; only parameters. So allocate memory for loading from the
+    // parameters pool.
+    v = static_cast<float*>(default_device->ps->allocate(d.size() * sizeof(float)));
     ar & boost::serialization::make_array(v, d.size());
 #endif
   }

parser/CMakeLists.txt

@@ -1,8 +1,19 @@
 PROJECT(lstm-parser:parser)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
-ADD_LIBRARY(lstm-parser-core lstm-parser.cc corpus.cc)
-target_link_libraries(lstm-parser-core cnn ${Boost_LIBRARIES})
-
+add_library(lstm-parser-core STATIC lstm-parser.cc corpus.cc 
+            neural-transition-tagger.cpp)
 ADD_EXECUTABLE(lstm-parse lstm-parser-driver.cc)
-target_link_libraries(lstm-parse lstm-parser-core ${Boost_LIBRARIES})
+
+if(WITH_CUDA_BACKEND)
+  add_dependencies(lstm-parser-core cnncuda)
+  target_link_libraries(lstm-parser-core cnncuda)
+  CUDA_ADD_CUBLAS_TO_TARGET(lstm-parser-core)
+
+  add_dependencies(lstm-parse cnncuda)
+  target_link_libraries(lstm-parse cnncuda)
+  CUDA_ADD_CUBLAS_TO_TARGET(lstm-parse)
+endif(WITH_CUDA_BACKEND)
+
+target_link_libraries(lstm-parser-core cnn ${Boost_LIBRARIES})
+target_link_libraries(lstm-parse lstm-parser-core ${Boost_LIBRARIES})