ENH python metrics added, fixes

Author: Jacob Schreiber

.gitignore

@@ -82,12 +82,12 @@ R-package/inst/*
 *.bin
 
 # ipython notebook
-example/notebooks/.ipynb_checkpoints/*
 *_pb2.py
+*.ipynb_checkpoints*
+input.txt*
 
 # Jetbrain
 .idea
 
 # ctags
-tags
-
+tags

Makefile

@@ -145,6 +145,8 @@ $(EXTRA_OPERATORS)/build/%_gpu.o: $(EXTRA_OPERATORS)/%.cu
 	$(NVCC) $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $(EXTRA_OPERATORS)/build/$*_gpu.o $< >$(EXTRA_OPERATORS)/build/$*_gpu.d
 	$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" $<
 
+# NOTE: to statically link libmxnet.a we need the option
+# --Wl,--whole-archive -lmxnet --Wl,--no-whole-archive
 lib/libmxnet.a: $(ALL_DEP)
 	@mkdir -p $(@D)
 	ar crv $@ $(filter %.o, $?)
@@ -153,7 +155,6 @@ lib/libmxnet.so: $(ALL_DEP)
 	@mkdir -p $(@D)
 	$(CXX) $(CFLAGS) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS)
 
-# ps-lite
 $(PS_PATH)/build/libps.a:
 	$(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps
 	ln -fs $(PS_PATH)/tracker .

cmake/Utils.cmake

@@ -1,3 +1,6 @@
+# For cmake_parse_arguments
+include(CMakeParseArguments)
+
 ################################################################################################
 # Command alias for debugging messages
 # Usage:
@@ -395,4 +398,5 @@ function(mxnet_source_group group)
     file(GLOB_RECURSE srcs2 ${CAFFE_SOURCE_GROUP_GLOB_RECURSE})
     source_group(${group} FILES ${srcs2})
   endif()
-endfunction()
+endfunction()
+

doc/build.md

@@ -35,7 +35,7 @@ Our goal is to build the shared library:
 The minimal building requirement is
 
 - A recent c++ compiler supporting C++ 11 such as `g++ >= 4.8` or `clang`
-- A BLAS library, such as `libblas`, `libblas`, `openblas` `intel mkl`
+- A BLAS library, such as `libblas`, `atlas`, `openblas` or `intel mkl`
 
 Optional libraries
 
@@ -239,6 +239,31 @@ Now you should have the R package as a tar.gz file and you can install it as a n
 R CMD INSTALL mxnet_0.5.tar.gz
 ```
 
+
+To install the package using GPU on Windows without building the package from scratch. Note that you need a couple of programs installed already:  
+- You'll need the [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit). This depends on Visual Studio, and a free compatible version would be [Visual Studio Community 2013](https://www.visualstudio.com/en-us/news/vs2013-community-vs.aspx). For instructions and compatibility checks, read http://docs.nvidia.com/cuda/cuda-getting-started-guide-for-microsoft-windows/ .
+
+- You will also need to register as a developer at nvidia and download CUDNN V3, https://developer.nvidia.com/cudnn . 
+
+
+1. Download the mxnet package as a ZIP from the Github repository https://github.com/dmlc/mxnet and unpack it. You will be editing the `/mxnet/R-package` folder.
+
+2. Download the most recent GPU-enabled package from the [Releases tab](https://github.com/dmlc/mxnet/releases). Unzip this file so you have a folder `/nocudnn`. Note that this file and the folder you'll save it in will be used for future reference and not directly for installing the package. Only some files will be copied from it into the `R-package` folder.
+
+(Note: you now have 2 folders we're working with, possibly in different locations, that we'll reference with `R-package/` and `nocudnn/`.)
+
+3. Download CUDNN V3 from https://developer.nvidia.com/cudnn. Unpack the .zip file and you'll see 3 folders, `/bin`, `/include`, `/lib`. Copy and replace these 3 folders into `nocudnn/3rdparty/cudnn/`, or unpack the .zip file there directly.
+
+4. Create the folder `R-package/inst/libs/x64`. We only support 64-bit operating system now, so you need the x64 folder;
+
+5. Put dll files in `R-package/inst/libs/x64`. 
+
+The first dll file you need is `nocudnn/lib/libmxnet.dll`. The other dll files you need are the ones in all 4 subfolders of `nocudnn/3rdparty/`, for the `cudnn` and `openblas` you'll need to look in the `/bin` folders. There should be 11 dll files now in `R-package/inst/libs/x64`.
+
+6. Copy the folder `nocudnn/include/` to `R-package/inst/`. So now you should have a folder `R-package/inst/include/` with 3 subfolders.
+
+7. Run `R CMD INSTALL --no-multiarch R-package`. Make sure that R is added to your PATH in Environment Variables. Running the command `Where R` in Command Prompt should return the location.
+
 Note on Library Build:
 
 We isolate the library build with Rcpp end to maximize the portability

example/cpp/Makefile

@@ -1,11 +1,16 @@
 CFLAGS=-I ../../include -Wall -O3 -msse3 -funroll-loops -Wno-unused-parameter -Wno-unknown-pragmas -fopenmp -I ../../mshadow -I ../../dmlc-core/include
 LDFLAGS=-L ../../lib -lmxnet -lopenblas -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -DMSHADOW_USE_CUDA=1
 
+CXX=g++
+
 mlp: ./mlp.cpp
-	g++ -std=c++0x $(CFLAGS) $(LDFLAGS) -o $@ $^
+	$(CXX) -std=c++0x $(CFLAGS) -o $@ $^ $(LDFLAGS)
 
 use_ndarray: ./use_ndarray.cpp
-	g++ -std=c++0x $(CFLAGS) $(LDFLAGS) -o $@ $^
+	$(CXX) -std=c++0x $(CFLAGS) -o $@ $^ $(LDFLAGS)
 
 lint:
 	python2 ../../dmlc-core/scripts/lint.py mxnet "cpp" ./
+
+clean:
+	rm -f mlp use_ndarray

example/kaggle-ndsb1/README.md

@@ -0,0 +1,34 @@
+Tutorial for Kaggle NDSB-1
+-----
+
+This is an MXNet example for Kaggle Nation Data Science Bowl 1.
+
+In this example we ignored submission part, only show local validation result.
+
+#### Step 1: Generate image list
+- Prepare original data, in layout like
+```
+--gen_img_list.py
+--data/
+    |
+    |--train/
+    |   |
+    |   |--acantharia_protist/...
+    |   |--.../
+    |--sampleSubmission.csv
+```
+- Run command ``` python gen_img_list.py train data/sampleSubmission.csv data/train/ train.lst``` to generate a full image list
+- Run command ```sed -n '1, 20000p' train.lst > tr.lst``` to generate local train list
+- Run command ```sed -n '20001p, 30337p' train.lst > va.lst``` to generate local validation list
+
+
+#### Step 2: Generate Image Record (new shape with short edge = 48)
+- Run command ```../../bin/im2rec tr.lst ./ tr.rec resize=48``` to generate training data record file
+- Run command ```../../bin/im2rec va.lst ./ va.rec resize=48``` to generate validation data record file
+
+#### Step 3: Train Model
+- Feel free to change hyper parameter in ```run_local.py```
+- Run ```python run_local.py``` to train the model
+- Sample code result: Train-accuracy=60.1%,  Validation-accuracy=62.1%
+
+

example/kaggle-ndsb1/gen_img_list.py

@@ -0,0 +1,43 @@
+import csv
+import os
+import sys
+import random
+
+if len(sys.argv) < 4:
+    print "Usage: gen_img_list.py train/test sample_submission.csv train_folder img.lst"
+    exit(1)
+
+random.seed(888)
+
+task = sys.argv[1]
+fc = csv.reader(file(sys.argv[2]))
+fi = sys.argv[3]
+fo = csv.writer(open(sys.argv[4], "w"), delimiter='\t', lineterminator='\n')
+
+# make class map
+head = fc.next()
+head = head[1:]
+
+# make image list
+img_lst = []
+cnt = 0
+if task == "train":
+    for i in xrange(len(head)):
+        path = fi + head[i]
+        lst = os.listdir(fi + head[i])
+        for img in lst:
+            img_lst.append((cnt, i, path + '/' + img))
+            cnt += 1
+else:
+    lst = os.listdir(fi)
+    for img in lst:
+        img_lst.append((cnt, 0, fi + img))
+        cnt += 1
+
+# shuffle
+random.shuffle(img_lst)
+
+#wirte
+for item in img_lst:
+    fo.writerow(item)
+

example/kaggle-ndsb1/run_local.py

@@ -0,0 +1,96 @@
+import mxnet as mx
+import numpy as np
+import logging
+
+# Example performance:
+# INFO:root:Epoch[34] Train-accuracy=0.601388
+# INFO:root:Epoch[34] Validation-accuracy=0.620949
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+# running device
+dev = mx.gpu()
+# batch size and input shape
+batch_size = 64
+data_shape = (3, 36, 36)
+# training data info for learning rate reduction
+num_examples = 20000
+epoch_size = num_examples / batch_size
+lr_factor_epoch = 15
+# model saving parameter
+model_prefix = "./models/sample_net"
+
+# train data iterator
+train = mx.io.ImageRecordIter(
+        path_imgrec = "tr.rec",
+        mean_r      = 128,
+        mean_g      = 128,
+        mean_b      = 128,
+        scale       = 0.0078125,
+        max_aspect_ratio = 0.35,
+        data_shape  = data_shape,
+        batch_size  = batch_size,
+        rand_crop   = True,
+        rand_mirror = True)
+
+# validate data iterator
+val = mx.io.ImageRecordIter(
+        path_imgrec = "va.rec",
+        mean_r      = 128,
+        mean_b      = 128,
+        mean_g      = 128,
+        scale       = 0.0078125,
+        rand_crop   = False,
+        rand_mirror = False,
+        data_shape  = data_shape,
+        batch_size  = batch_size)
+
+# network definition
+# stage 1
+net = mx.sym.Variable("data")
+net = mx.sym.Convolution(data=net, kernel=(5, 5), num_filter=32, pad=(2, 2))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Convolution(data=net, kernel=(5, 5), num_filter=64, pad=(2, 2))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Pooling(data=net, pool_type="max", kernel=(3, 3), stride=(2, 2))
+# stage 2
+net = mx.sym.Convolution(data=net, kernel=(3, 3), num_filter=64, pad=(1, 1))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Convolution(data=net, kernel=(3, 3), num_filter=64, pad=(1, 1))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Convolution(data=net, kernel=(3, 3), num_filter=128, pad=(1, 1))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Pooling(data=net, pool_type="max", kernel=(3, 3), stride=(2, 2))
+# stage 3
+net = mx.sym.Convolution(data=net, kernel=(3, 3), num_filter=256, pad=(1, 1))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Convolution(data=net, kernel=(3, 3), num_filter=256, pad=(1, 1))
+net = mx.sym.Activation(data=net, act_type="relu")
+net = mx.sym.Pooling(data=net, pool_type="avg", kernel=(9, 9), stride=(1, 1))
+# stage 4
+net = mx.sym.Flatten(data=net)
+net = mx.sym.Dropout(data=net, p=0.25)
+net = mx.sym.FullyConnected(data=net, num_hidden=121)
+net = mx.symbol.SoftmaxOutput(data=net, name='softmax')
+
+# Model parameter
+# This model will reduce learning rate by factor 0.1 for every 15 epoch
+model = mx.model.FeedForward(
+        ctx                = dev,
+        symbol             = net,
+        num_epoch          = 35,
+        learning_rate      = 0.01,
+        momentum           = 0.9,
+        wd                 = 0.0001,
+        clip_gradient      = 5,
+        lr_scheduler       = mx.lr_scheduler.FactorScheduler(step=epoch_size * lr_factor_epoch, factor = 0.1),
+        initializer        = mx.init.Xavier(factor_type="in", magnitude=2.34))
+
+# fit the model
+model.fit(
+        X                  = train,
+        eval_data          = val,
+        batch_end_callback = mx.callback.Speedometer(batch_size, 50),
+        epoch_end_callback = mx.callback.do_checkpoint(model_prefix))
+

example/kaggle-ndsb2/Preprocessing.py

@@ -10,6 +10,8 @@
 import numpy as np
 import dicom
 from skimage import io, transform
+from joblib import Parallel, delayed
+import dill
 
 def mkdir(fname):
    try:
@@ -53,29 +55,33 @@ def write_label_csv(fname, frames, label_map):
    fo.close()
 
 
+def get_data(lst,preproc):
+   data = []
+   result = []
+   for path in lst:
+       f = dicom.read_file(path)
+       img = preproc(f.pixel_array.astype(float) / np.max(f.pixel_array))
+       dst_path = path.rsplit(".", 1)[0] + ".64x64.jpg"
+       scipy.misc.imsave(dst_path, img)
+       result.append(dst_path)
+       data.append(img)
+   data = np.array(data, dtype=np.uint8)
+   data = data.reshape(data.size)
+   data = np.array(data,dtype=np.str_)
+   data = data.reshape(data.size)
+   return [data,result]
+
+
 def write_data_csv(fname, frames, preproc):
    """Write data to csv file"""
    fdata = open(fname, "w")
-   dwriter = csv.writer(fdata)
-   counter = 0
-   result = []
-   for lst in frames:
-       data = []
-       for path in lst:
-           f = dicom.read_file(path)
-           img = preproc(f.pixel_array.astype(float) / np.max(f.pixel_array))
-           dst_path = path.rsplit(".", 1)[0] + ".64x64.jpg"
-           scipy.misc.imsave(dst_path, img)
-           result.append(dst_path)
-           data.append(img)
-       data = np.array(data, dtype=np.uint8)
-       data = data.reshape(data.size)
-       dwriter.writerow(data)
-       counter += 1
-       if counter % 100 == 0:
-           print("%d slices processed" % counter)
-   print("All finished, %d slices in total" % counter)
+   dr = Parallel()(delayed(get_data)(lst,preproc) for lst in frames)
+   data,result = zip(*dr)
+   for entry in data:
+      fdata.write(','.join(entry)+'\r\n')
+   print("All finished, %d slices in total" % len(data))
    fdata.close()
+   result = np.ravel(result)
    return result