Skip to content

Commit ea25998

Browse files
piiswrongpiiswrong
committed
make ccsgd the default and add number of batch to predict
1 parent bf172a0 commit ea25998

18 files changed

+483
-332
lines changed

‎Makefile

+26-2
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,27 @@ OBJ = $(patsubst src/%.cc, build/%.o, $(SRC))
9898
CUSRC = $(wildcard src/*/*.cu)
9999
CUOBJ = $(patsubst src/%.cu, build/%_gpu.o, $(CUSRC))
100100

101+
ifneq ($(EXTRA_OPERATORS), NONE)
102+
EXTRA_SRC = $(wildcard $(EXTRA_OPERATORS)/*.cc $(EXTRA_OPERATORS)/*/*.cc)
103+
EXTRA_OBJ = $(patsubst $(EXTRA_OPERATORS)/%.cc, $(EXTRA_OPERATORS)/build/%.o, $(EXTRA_SRC))
104+
EXTRA_CUSRC = $(wildcard $(EXTRA_OPERATORS)/*.cu $(EXTRA_OPERATORS)/*/*.cu)
105+
EXTRA_CUOBJ = $(patsubst $(EXTRA_OPERATORS)/%.cu, $(EXTRA_OPERATORS)/build/%_gpu.o, $(EXTRA_CUSRC))
106+
else
107+
EXTRA_SRC =
108+
EXTRA_OBJ =
109+
EXTRA_CUSRC =
110+
EXTRA_CUOBJ =
111+
endif
112+
101113
LIB_DEP += $(DMLC_CORE)/libdmlc.a
102-
ALL_DEP = $(OBJ) $(LIB_DEP)
114+
ALL_DEP = $(OBJ) $(EXTRA_OBJ) $(LIB_DEP)
103115
ifeq ($(USE_CUDA), 1)
104-
ALL_DEP += $(CUOBJ)
116+
ALL_DEP += $(CUOBJ) $(EXTRA_CUOBJ)
105117
LDFLAGS += -lnvrtc -lcuda
106118
endif
107119

120+
121+
108122
build/%.o: src/%.cc
109123
@mkdir -p $(@D)
110124
$(CXX) -std=c++0x $(CFLAGS) -MM -MT build/$*.o $< >build/$*.d
@@ -115,6 +129,16 @@ build/%_gpu.o: src/%.cu
115129
$(NVCC) $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" -M -MT build/$*_gpu.o $< >build/$*_gpu.d
116130
$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" $<
117131

132+
$(EXTRA_OPERATORS)/build/%.o: $(EXTRA_OPERATORS)/%.cc
133+
@mkdir -p $(@D)
134+
$(CXX) -std=c++0x $(CFLAGS) -Isrc/operator -MM -MT $(EXTRA_OPERATORS)/build/$*.o $< >$(EXTRA_OPERATORS)/build/$*.d
135+
$(CXX) -std=c++0x -c $(CFLAGS) -Isrc/operator -c $< -o $@
136+
137+
$(EXTRA_OPERATORS)/build/%_gpu.o: $(EXTRA_OPERATORS)/%.cu
138+
@mkdir -p $(@D)
139+
$(NVCC) $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $(EXTRA_OPERATORS)/build/$*_gpu.o $< >$(EXTRA_OPERATORS)/build/$*_gpu.d
140+
$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" $<
141+
118142
lib/libmxnet.a: $(ALL_DEP)
119143
@mkdir -p $(@D)
120144
ar crv $@ $(filter %.o, $?)

‎make/config.mk

+7
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,10 @@ LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
9595
# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
9696
# sudo apt-get install -y libcurl4-openssl-dev
9797
USE_S3 = 0
98+
99+
#----------------------------
100+
# additional operators
101+
#----------------------------
102+
103+
# path to folders containing projects specific operators that you don't want to put in src/operators
104+
EXTRA_OPERATORS =

‎make/osx.mk

+7
Original file line numberDiff line numberDiff line change
@@ -82,3 +82,10 @@ LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
8282
# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
8383
# sudo apt-get install -y libcurl4-openssl-dev
8484
USE_S3 = 0
85+
86+
#----------------------------
87+
# additional operators
88+
#----------------------------
89+
90+
# path to folders containing projects specific operators that you don't want to put in src/operators
91+
EXTRA_OPERATORS =

‎python/mxnet/model.py

+10-3
Original file line numberDiff line numberDiff line change
@@ -493,7 +493,7 @@ class FeedForward(BASE_ESTIMATOR):
493493
The additional keyword arguments passed to optimizer.
494494
"""
495495
def __init__(self, symbol, ctx=None,
496-
num_epoch=None, epoch_size=None, optimizer='sgd',
496+
num_epoch=None, epoch_size=None, optimizer='ccsgd',
497497
initializer=Uniform(0.01),
498498
numpy_batch_size=128,
499499
arg_params=None, aux_params=None,
@@ -632,11 +632,13 @@ def _init_eval_iter(self, eval_data):
632632
'NDArray/numpy.ndarray/list pair (i.e. tuple/list of length 2)')
633633
return eval_data
634634

635-
def predict(self, X):
635+
def predict(self, X, num_batch=None):
636636
"""Run the prediction, always only use one device.
637637
Parameters
638638
----------
639639
X : mxnet.DataIter
640+
num_batch : int or None
641+
the number of batch to run. Go though all batches if None
640642
Returns
641643
-------
642644
y : numpy.ndarray or a list of numpy.ndarray if the network has multiple outputs.
@@ -652,7 +654,12 @@ def predict(self, X):
652654
data_arrays = [self._pred_exec.arg_dict[name] for name in data_names]
653655
output_list = [[] for _ in range(len(self._pred_exec.outputs))]
654656

657+
i = 0
655658
for batch in X:
659+
if num_batch is not None and i == num_batch:
660+
break
661+
i += 1
662+
656663
_load_data(batch, data_arrays)
657664
self._pred_exec.forward(is_train=False)
658665
padded = batch.pad
@@ -803,7 +810,7 @@ def load(prefix, epoch, ctx=None, **kwargs):
803810

804811
@staticmethod
805812
def create(symbol, X, y=None, ctx=None,
806-
num_epoch=None, epoch_size=None, optimizer='sgd', initializer=Uniform(0.01),
813+
num_epoch=None, epoch_size=None, optimizer='ccsgd', initializer=Uniform(0.01),
807814
eval_data=None, eval_metric='acc',
808815
epoch_end_callback=None, batch_end_callback=None,
809816
kvstore='local', logger=None, work_load_list=None, **kwargs):

‎python/mxnet/optimizer.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ def _init_cc_optimizer(name, param_keys, param_vals):
7272
handle to the optimizer
7373
"""
7474
creator = OptimizerCreator()
75-
check_call(_LIB.MXOptimizerFindCreator(ctypes.c_char_p(name),
75+
check_call(_LIB.MXOptimizerFindCreator(c_str(name),
7676
ctypes.byref(creator)))
7777
assert creator, "Cannot find c++ implementation of optimizer \
7878
registered with name "+name

‎python/mxnet/recordio.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ def unpack_img(s, iscolor=-1):
157157
img = cv2.imdecode(img, iscolor)
158158
return header, img
159159

160-
def pack_img(header, img, quality=80, format='.JPEG'):
160+
def pack_img(header, img, quality=80, img_fmt='.JPEG'):
161161
"""pack an image into MXImageRecord
162162
163163
Parameters
@@ -175,6 +175,6 @@ def pack_img(header, img, quality=80, format='.JPEG'):
175175
The packed string
176176
"""
177177
assert opencv_available
178-
ret, buf = cv2.imencode(format, img, [cv2.IMWRITE_JPEG_QUALITY, quality])
179-
assert ret
178+
ret, buf = cv2.imencode(img_fmt, img, [cv2.IMWRITE_JPEG_QUALITY, quality])
179+
assert ret, 'failed encoding image'
180180
return pack(header, buf.tostring())

‎src/operator/activation-inl.h

+2-7
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ namespace op {
2424
namespace activation {
2525
enum ActivationOpInputs {kData};
2626
enum ActivationOpOutputs {kOut};
27-
enum ActivationOpType {kReLU, kSigmoid, kTanh, kSoftReLU, kSoftmax};
27+
enum ActivationOpType {kReLU, kSigmoid, kTanh, kSoftReLU};
2828
} // activation
2929

3030
struct ActivationParam : public dmlc::Parameter<ActivationParam> {
@@ -36,7 +36,6 @@ struct ActivationParam : public dmlc::Parameter<ActivationParam> {
3636
.add_enum("sigmoid", activation::kSigmoid)
3737
.add_enum("tanh", activation::kTanh)
3838
.add_enum("softrelu", activation::kSoftReLU)
39-
.add_enum("softmax", activation::kSoftmax)
4039
.describe("Activation function to be applied.");
4140
}
4241
};
@@ -140,11 +139,7 @@ class ActivationProp : public OperatorProperty {
140139
const std::vector<int> &in_data,
141140
const std::vector<int> &out_data) const override {
142141
#if MXNET_USE_CUDNN == 1
143-
if (param_.act_type == activation::kSoftmax) {
144-
return {out_grad[activation::kOut], out_data[activation::kOut]};
145-
} else {
146-
return {out_grad[activation::kOut], out_data[activation::kOut], in_data[activation::kData]};
147-
}
142+
return {out_grad[activation::kOut], out_data[activation::kOut], in_data[activation::kData]};
148143
#else
149144
return {out_grad[activation::kOut], out_data[activation::kOut]};
150145
#endif // MXNET_USE_CUDNN

‎src/operator/cudnn_activation-inl.h

+23-63
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ class CuDNNActivationOp : public Operator {
2929
case activation::kTanh:
3030
mode_ = CUDNN_ACTIVATION_TANH;
3131
break;
32-
case activation::kSoftmax:
33-
break;
3432
default:
3533
LOG(FATAL) << "Not implmented";
3634
break;
@@ -53,13 +51,11 @@ class CuDNNActivationOp : public Operator {
5351
Stream<gpu> *s = ctx.get_stream<gpu>();
5452
Tensor<gpu, 4> data;
5553
Tensor<gpu, 4> out;
56-
cudnnSoftmaxMode_t softmax_mode;
5754
if (in_data[activation::kData].ndim() == 2) {
5855
Shape<4> dshape = Shape4(in_data[activation::kData].shape_[0],
5956
in_data[activation::kData].shape_[1], 1, 1);
6057
data = in_data[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
6158
out = out_data[activation::kOut].get_with_shape<gpu, 4, real_t>(dshape, s);
62-
softmax_mode = CUDNN_SOFTMAX_MODE_INSTANCE;
6359
} else {
6460
Shape<4> dshape;
6561
index_t size_left = in_data[activation::kData].Size();
@@ -74,7 +70,6 @@ class CuDNNActivationOp : public Operator {
7470
dshape[3] = size_left;
7571
data = in_data[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
7672
out = out_data[activation::kOut].get_with_shape<gpu, 4, real_t>(dshape, s);
77-
softmax_mode = CUDNN_SOFTMAX_MODE_CHANNEL;
7873
}
7974
float alpha = 1.0f;
8075
float beta = 0.0f;
@@ -90,26 +85,14 @@ class CuDNNActivationOp : public Operator {
9085
data.shape_[2],
9186
data.shape_[3]), CUDNN_STATUS_SUCCESS);
9287
}
93-
if (param_.act_type == activation::kSoftmax) {
94-
CHECK_EQ(cudnnSoftmaxForward(s->dnn_handle_,
95-
CUDNN_SOFTMAX_ACCURATE,
96-
softmax_mode,
97-
&alpha,
98-
shape_desc_,
99-
data.dptr_,
100-
&beta,
101-
shape_desc_,
102-
out.dptr_), CUDNN_STATUS_SUCCESS);
103-
} else {
104-
CHECK_EQ(cudnnActivationForward(s->dnn_handle_,
105-
mode_,
106-
&alpha,
107-
shape_desc_,
108-
data.dptr_,
109-
&beta,
110-
shape_desc_,
111-
out.dptr_), CUDNN_STATUS_SUCCESS);
112-
}
88+
CHECK_EQ(cudnnActivationForward(s->dnn_handle_,
89+
mode_,
90+
&alpha,
91+
shape_desc_,
92+
data.dptr_,
93+
&beta,
94+
shape_desc_,
95+
out.dptr_), CUDNN_STATUS_SUCCESS);
11396
}
11497

11598
virtual void Backward(const OpContext &ctx,
@@ -122,9 +105,7 @@ class CuDNNActivationOp : public Operator {
122105
using namespace mshadow;
123106
using namespace mshadow::expr;
124107
CHECK_EQ(out_grad.size(), 1);
125-
if (param_.act_type != activation::kSoftmax) {
126-
CHECK_EQ(in_data.size(), 1);
127-
}
108+
CHECK_EQ(in_data.size(), 1);
128109
CHECK_EQ(out_data.size(), 1);
129110
CHECK_EQ(req.size(), 1);
130111
CHECK_EQ(in_grad.size(), 1);
@@ -135,17 +116,13 @@ class CuDNNActivationOp : public Operator {
135116
Tensor<gpu, 4> data;
136117
Tensor<gpu, 4> output_data;
137118
Tensor<gpu, 4> input_grad;
138-
cudnnSoftmaxMode_t softmax_mode;
139119
if (in_grad[activation::kData].ndim() == 2) {
140120
Shape<4> dshape = Shape4(in_grad[activation::kData].shape_[0],
141121
in_grad[activation::kData].shape_[1], 1, 1);
142-
if (param_.act_type != activation::kSoftmax) {
143-
data = in_data[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
144-
}
122+
data = in_data[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
145123
grad = out_grad[activation::kOut].get_with_shape<gpu, 4, real_t>(dshape, s);
146124
output_data = out_data[activation::kOut].get_with_shape<gpu, 4, real_t>(dshape, s);
147125
input_grad = in_grad[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
148-
softmax_mode = CUDNN_SOFTMAX_MODE_INSTANCE;
149126
} else {
150127
Shape<4> dshape;
151128
index_t size_left = in_grad[activation::kData].Size();
@@ -158,41 +135,24 @@ class CuDNNActivationOp : public Operator {
158135
size_left /= dshape[i];
159136
}
160137
dshape[3] = size_left;
161-
if (param_.act_type != activation::kSoftmax) {
162-
data = in_data[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
163-
}
138+
data = in_data[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
164139
output_data = out_data[activation::kOut].get_with_shape<gpu, 4, real_t>(dshape, s);
165140
grad = out_grad[activation::kOut].get_with_shape<gpu, 4, real_t>(dshape, s);
166141
input_grad = in_grad[activation::kData].get_with_shape<gpu, 4, real_t>(dshape, s);
167-
softmax_mode = CUDNN_SOFTMAX_MODE_CHANNEL;
168142
}
169143
CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream<gpu>::OwnHandle);
170-
if (param_.act_type == activation::kSoftmax) {
171-
CHECK_EQ(cudnnSoftmaxBackward(s->dnn_handle_,
172-
CUDNN_SOFTMAX_ACCURATE,
173-
softmax_mode,
174-
&alpha,
175-
shape_desc_,
176-
output_data.dptr_,
177-
shape_desc_,
178-
grad.dptr_,
179-
&beta,
180-
shape_desc_,
181-
input_grad.dptr_), CUDNN_STATUS_SUCCESS);
182-
} else {
183-
CHECK_EQ(cudnnActivationBackward(s->dnn_handle_,
184-
mode_,
185-
&alpha,
186-
shape_desc_,
187-
output_data.dptr_,
188-
shape_desc_,
189-
grad.dptr_,
190-
shape_desc_,
191-
data.dptr_,
192-
&beta,
193-
shape_desc_,
194-
input_grad.dptr_), CUDNN_STATUS_SUCCESS);
195-
}
144+
CHECK_EQ(cudnnActivationBackward(s->dnn_handle_,
145+
mode_,
146+
&alpha,
147+
shape_desc_,
148+
output_data.dptr_,
149+
shape_desc_,
150+
grad.dptr_,
151+
shape_desc_,
152+
data.dptr_,
153+
&beta,
154+
shape_desc_,
155+
input_grad.dptr_), CUDNN_STATUS_SUCCESS);
196156
}
197157

198158
private:

0 commit comments

Comments
 (0)