[cpp-package] Add C++ basic tutorial and build instruction (apache#5971)

* Add C++ basic tutorial and build instruction

* Remove binaries

* Fix lint

* Avoid sign-compare

cpp-package/example/Makefile

@@ -0,0 +1,16 @@
+CPPEX_SRC = $(wildcard *.cpp)
+CPPEX_EXE = $(patsubst %.cpp, %, $(CPPEX_SRC))
+
+CFLAGS += -I../../include -I../../nnvm/include -I../../dmlc-core/include
+CPPEX_CFLAGS += -I../include
+CPPEX_EXTRA_LDFLAGS := -L. -lmxnet
+
+.PHONY: all clean
+
+all: $(CPPEX_EXE)
+
+$(CPPEX_EXE):% : %.cpp libmxnet.so ../include/mxnet-cpp/*.h
+	$(CXX) -std=c++0x $(CFLAGS) $(CPPEX_CFLAGS) -o $@ $(filter %.cpp %.a, $^) $(CPPEX_EXTRA_LDFLAGS)
+
+clean:
+	rm -f $(CPPEX_EXE)

cpp-package/example/get_mnist.sh

@@ -0,0 +1,9 @@
+if [ ! -d "./mnist_data" ]; then
+  mkdir mnist_data
+  (cd mnist_data; wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz)
+  (cd mnist_data; wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz)
+  (cd mnist_data; wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz)
+  (cd mnist_data; wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz)
+  (cd mnist_data; gzip -d *.gz)
+fi
+echo "Data downloaded"

cpp-package/example/mlp_cpu.cpp

@@ -0,0 +1,118 @@
+/*!
+ * Copyright (c) 2017 by Contributors
+ * Xin Li yakumolx@gmail.com
+ */
+#include <chrono>
+#include "mxnet-cpp/MxNetCpp.h"
+
+using namespace std;
+using namespace mxnet::cpp;
+
+Symbol mlp(const vector<int> &layers) {
+  auto x = Symbol::Variable("X");
+  auto label = Symbol::Variable("label");
+
+  vector<Symbol> weights(layers.size());
+  vector<Symbol> biases(layers.size());
+  vector<Symbol> outputs(layers.size());
+
+  for (size_t i = 0; i < layers.size(); ++i) {
+    weights[i] = Symbol::Variable("w" + to_string(i));
+    biases[i] = Symbol::Variable("b" + to_string(i));
+    Symbol fc = FullyConnected(
+      i == 0? x : outputs[i-1],  // data
+      weights[i],
+      biases[i],
+      layers[i]);
+    outputs[i] = i == layers.size()-1 ? fc : Activation(fc, ActivationActType::relu);
+  }
+
+  return SoftmaxOutput(outputs.back(), label);
+}
+
+int main(int argc, char** argv) {
+  const int image_size = 28;
+  const vector<int> layers{128, 64, 10};
+  const int batch_size = 100;
+  const int max_epoch = 10;
+  const float learning_rate = 0.1;
+  const float weight_decay = 1e-2;
+
+  auto train_iter = MXDataIter("MNISTIter")
+      .SetParam("image", "./mnist_data/train-images-idx3-ubyte")
+      .SetParam("label", "./mnist_data/train-labels-idx1-ubyte")
+      .SetParam("batch_size", batch_size)
+      .SetParam("flat", 1)
+      .CreateDataIter();
+  auto val_iter = MXDataIter("MNISTIter")
+      .SetParam("image", "./mnist_data/t10k-images-idx3-ubyte")
+      .SetParam("label", "./mnist_data/t10k-labels-idx1-ubyte")
+      .SetParam("batch_size", batch_size)
+      .SetParam("flat", 1)
+      .CreateDataIter();
+
+  auto net = mlp(layers);
+
+  Context ctx = Context::cpu();  // Use CPU for training
+
+  std::map<string, NDArray> args;
+  args["X"] = NDArray(Shape(batch_size, image_size*image_size), ctx);
+  args["label"] = NDArray(Shape(batch_size), ctx);
+  // Let MXNet infer shapes other parameters such as weights
+  net.InferArgsMap(ctx, &args, args);
+
+  // Initialize all parameters with uniform distribution U(-0.01, 0.01)
+  auto initializer = Uniform(0.01);
+  for (auto& arg : args) {
+    // arg.first is parameter name, and arg.second is the value
+    initializer(arg.first, &arg.second);
+  }
+
+  // Create sgd optimizer
+  Optimizer* opt = OptimizerRegistry::Find("sgd");
+  opt->SetParam("rescale_grad", 1.0/batch_size);
+
+  // Start training
+  for (int iter = 0; iter < max_epoch; ++iter) {
+    int samples = 0;
+    train_iter.Reset();
+
+    auto tic = chrono::system_clock::now();
+    while (train_iter.Next()) {
+      samples += batch_size;
+      auto data_batch = train_iter.GetDataBatch();
+      // Set data and label
+      args["X"] = data_batch.data;
+      args["label"] = data_batch.label;
+
+      // Create executor by binding parmeters to the model
+      auto *exec = net.SimpleBind(ctx, args);
+      // Compute gradients
+      exec->Forward(true);
+      exec->Backward();
+      // Update parameters
+      exec->UpdateAll(opt, learning_rate, weight_decay);
+      // Remember to free the memory
+      delete exec;
+    }
+    auto toc = chrono::system_clock::now();
+
+    Accuracy acc;
+    val_iter.Reset();
+    while (val_iter.Next()) {
+      auto data_batch = val_iter.GetDataBatch();
+      args["X"] = data_batch.data;
+      args["label"] = data_batch.label;
+      auto *exec = net.SimpleBind(ctx, args);
+      // Forward pass is enough as no gradient is needed when evaluating
+      exec->Forward(false);
+      acc.Update(data_batch.label, exec->outputs[0]);
+      delete exec;
+    }
+    float duration = chrono::duration_cast<chrono::milliseconds>(toc - tic).count() / 1000.0;
+    LG << "Epoch: " << iter << " " << samples/duration << " samples/sec Accuracy: " << acc.Get();
+  }
+
+  MXNotifyShutdown();
+  return 0;
+}

cpp-package/example/mlp_gpu.cpp

@@ -0,0 +1,121 @@
+/*!
+ * Copyright (c) 2016 by Contributors
+ * Xin Li yakumolx@gmail.com
+ */
+#include <chrono>
+#include "mxnet-cpp/MxNetCpp.h"
+
+using namespace std;
+using namespace mxnet::cpp;
+
+Symbol mlp(const vector<int> &layers) {
+  auto x = Symbol::Variable("X");
+  auto label = Symbol::Variable("label");
+
+  vector<Symbol> weights(layers.size());
+  vector<Symbol> biases(layers.size());
+  vector<Symbol> outputs(layers.size());
+
+  for (size_t i = 0; i < layers.size(); ++i) {
+    weights[i] = Symbol::Variable("w" + to_string(i));
+    biases[i] = Symbol::Variable("b" + to_string(i));
+    Symbol fc = FullyConnected(
+      i == 0? x : outputs[i-1],  // data
+      weights[i],
+      biases[i],
+      layers[i]);
+    outputs[i] = i == layers.size()-1? fc : Activation(fc, ActivationActType::relu);
+  }
+
+  return SoftmaxOutput(outputs.back(), label);
+}
+
+int main(int argc, char** argv) {
+  const int image_size = 28;
+  const vector<int> layers{128, 64, 10};
+  const int batch_size = 100;
+  const int max_epoch = 10;
+  const float learning_rate = 0.1;
+  const float weight_decay = 1e-2;
+
+  auto train_iter = MXDataIter("MNISTIter")
+      .SetParam("image", "./mnist_data/train-images-idx3-ubyte")
+      .SetParam("label", "./mnist_data/train-labels-idx1-ubyte")
+      .SetParam("batch_size", batch_size)
+      .SetParam("flat", 1)
+      .CreateDataIter();
+  auto val_iter = MXDataIter("MNISTIter")
+      .SetParam("image", "./mnist_data/t10k-images-idx3-ubyte")
+      .SetParam("label", "./mnist_data/t10k-labels-idx1-ubyte")
+      .SetParam("batch_size", batch_size)
+      .SetParam("flat", 1)
+      .CreateDataIter();
+
+  auto net = mlp(layers);
+
+  Context ctx = Context::gpu();  // Use GPU for training
+
+  std::map<string, NDArray> args;
+  args["X"] = NDArray(Shape(batch_size, image_size*image_size), ctx);
+  args["label"] = NDArray(Shape(batch_size), ctx);
+  // Let MXNet infer shapes of other parameters such as weights
+  net.InferArgsMap(ctx, &args, args);
+
+  // Initialize all parameters with uniform distribution U(-0.01, 0.01)
+  auto initializer = Uniform(0.01);
+  for (auto& arg : args) {
+    // arg.first is parameter name, and arg.second is the value
+    initializer(arg.first, &arg.second);
+  }
+
+  // Create sgd optimizer
+  Optimizer* opt = OptimizerRegistry::Find("sgd");
+  opt->SetParam("rescale_grad", 1.0/batch_size);
+
+  // Start training
+  for (int iter = 0; iter < max_epoch; ++iter) {
+    int samples = 0;
+    train_iter.Reset();
+
+    auto tic = chrono::system_clock::now();
+    while (train_iter.Next()) {
+      samples += batch_size;
+      auto data_batch = train_iter.GetDataBatch();
+      // Data provided by DataIter are stored in memory, should be copied to GPU first.
+      data_batch.data.CopyTo(&args["X"]);
+      data_batch.label.CopyTo(&args["label"]);
+      // CopyTo is imperative, need to wait for it to complete.
+      NDArray::WaitAll();
+
+      // Create executor by binding parmeters to the model
+      auto *exec = net.SimpleBind(ctx, args);
+      // Compute gradients
+      exec->Forward(true);
+      exec->Backward();
+      // Update parameters
+      exec->UpdateAll(opt, learning_rate, weight_decay);
+      // Remember to free the memory
+      delete exec;
+    }
+    auto toc = chrono::system_clock::now();
+
+    Accuracy acc;
+    val_iter.Reset();
+    while (val_iter.Next()) {
+      auto data_batch = val_iter.GetDataBatch();
+      data_batch.data.CopyTo(&args["X"]);
+      data_batch.label.CopyTo(&args["label"]);
+      NDArray::WaitAll();
+      auto *exec = net.SimpleBind(ctx, args);
+      // Only forward pass is enough as no gradient is needed when evaluating
+      exec->Forward(false);
+      acc.Update(data_batch.label, exec->outputs[0]);
+      delete exec;
+    }
+    float duration = chrono::duration_cast<chrono::milliseconds>(toc - tic).count() / 1000.0;
+    LG << "Epoch: " << iter << " " << samples/duration << " samples/sec Accuracy: " << acc.Get();
+  }
+
+  MXNotifyShutdown();
+  return 0;
+}

cpp-package/include/mxnet-cpp/initializer.h

@@ -75,6 +75,68 @@ class Initializer {
   virtual void InitDefault(NDArray* arr) {}
 };
 
+class Zero : public Initializer {
+ public:
+  Zero() {}
+ protected:
+  void InitWeight(NDArray *arr) override {
+    InitZero(arr);
+  }
+};
+
+class One : public Initializer {
+ public:
+  One() {}
+ protected:
+  void InitWeight(NDArray *arr) override {
+    InitOne(arr);
+  }
+};
+
+class Constant : public Initializer {
+ public:
+  explicit Constant(float value)
+    : value(value) {}
+ protected:
+  float value;
+  void InitWeight(NDArray *arr) override {
+    (*arr) = value;
+  }
+};
+
+class Uniform : public Initializer {
+ public:
+  explicit Uniform(float scale)
+    : Uniform(-scale, scale) {}
+  Uniform(float begin, float end)
+    : begin(begin), end(end) {}
+ protected:
+  float begin, end;
+  void InitWeight(NDArray *arr) override {
+    NDArray::SampleUniform(begin, end, arr);
+  }
+};
+
+class Normal : public Initializer {
+ public:
+  Normal(float mu, float sigma)
+    : mu(mu), sigma(sigma) {}
+ protected:
+  float mu, sigma;
+  void InitWeight(NDArray *arr) override {
+    NDArray::SampleGaussian(mu, sigma, arr);
+  }
+};
+
+class Bilinear : public Initializer {
+ public:
+  Bilinear() {}
+ protected:
+  void InitWeight(NDArray *arr) override {
+    InitBilinear(arr);
+  }
+};
+
 class Xavier : public Initializer {
  public:
   enum RandType {
@@ -92,7 +154,7 @@ class Xavier : public Initializer {
       : rand_type(rand_type), factor_type(factor_type), magnitude(magnitude) {}
 
  protected:
-  virtual void InitWeight(NDArray* arr) {
+  void InitWeight(NDArray* arr) override {
     Shape shape(arr->GetShape());
     float hw_scale = 1.0f;
     if (shape.ndim() > 2) {

docs/get_started/build_from_source.md

@@ -4,6 +4,7 @@ This document explains how to build MXNet from sources. Building MXNet from sour
 
 1. Build the MXNet shared library, `libmxnet.so`, from [C++ source files](#build-the-shared-library)
 2. Install the language binding for MXNet. MXNet supports - [Python](#build-the-python-package),
+   [C++](#build-the-cpp-package),
    [Scala](#build-the-scala-package), [R](#build-the-r-package), and
    [Julia](#build-the-julia-package).
 
@@ -276,6 +277,8 @@ File
 contains all the compilation options. You can edit it and then `make`. There are
 some example build options
 
+If you want to build MXNet with C++ language binding, please make sure you read [Build the C++ package](#build-the-cpp-package) first.
+
 </div>
 
 <div class="linux">
@@ -355,6 +358,11 @@ The Python package can be installed by one of the following three ways:
    cd python; sudo python setup.py install
    ```
 
+## Build the C++ package
+The C++ package has the same prerequisites as the MXNet library, you should also have `python` 2 installed. (`python` 3 is not supported currently)
+
+To enable C++ package, just add `USE_CPP_PACKAGE=1` in the build options when building the MXNet shared library.
+
 ## Build the R package
 
 The R package requires `R` to be installed.