diff --git a/python/mxnet/symbol_doc.py b/python/mxnet/symbol_doc.py index 353196fe..1e896c97 100644 --- a/python/mxnet/symbol_doc.py +++ b/python/mxnet/symbol_doc.py @@ -19,6 +19,7 @@ - all the operators (e.g. `FullyConnected`) - the name `test_utils` for `mxnet.test_utils` (e.g. `test_utils.reldiff`) - the name `mxnet` (e.g. `mxnet.nd.zeros`) +- the name `numpy` The following documents are recommended: @@ -40,6 +41,69 @@ def get_output_shape(sym, **input_shapes): return dict(zip(sym.list_outputs(), s_outputs)) +class ActivationDoc(SymbolDoc): + """ + Examples + -------- + A one-hidden-layer MLP with ReLU activation: + + >>> data = Variable('data') + >>> mlp = FullyConnected(data=data, num_hidden=128, name='proj') + >>> mlp = Activation(data=mlp, act_type='relu', name='activation') + >>> mlp = FullyConnected(data=mlp, num_hidden=10, name='mlp') + >>> mlp + + + Regression Test + --------------- + ReLU activation + + >>> test_suites = [ + ... ('relu', lambda x: numpy.maximum(x, 0)), + ... ('sigmoid', lambda x: 1 / (1 + numpy.exp(-x))), + ... ('tanh', lambda x: numpy.tanh(x)), + ... ('softrelu', lambda x: numpy.log(1 + numpy.exp(x))) + ... ] + >>> x = test_utils.random_arrays((2, 3, 4)) + >>> for act_type, numpy_impl in test_suites: + ... op = Activation(act_type=act_type, name='act') + ... y = test_utils.simple_forward(op, act_data=x) + ... y_np = numpy_impl(x) + ... print('%s: %s' % (act_type, test_utils.almost_equal(y, y_np))) + relu: True + sigmoid: True + tanh: True + softrelu: True + """ + + +class FlattenDoc(SymbolDoc): + """ + Examples + -------- + Flatten is usually applied before `FullyConnected`, to reshape the 4D tensor + produced by convolutional layers to 2D matrix: + + >>> data = Variable('data') # say this is 4D from some conv/pool + >>> flatten = Flatten(data=data, name='flat') # now this is 2D + >>> SymbolDoc.get_output_shape(flatten, data=(2, 3, 4, 5)) + {'flat_output': (2L, 60L)} + + Regression Test + --------------- + >>> test_dims = [(2, 3, 4, 5), (2, 3), (2,)] + >>> op = Flatten(name='flat') + >>> for dims in test_dims: + ... x = test_utils.random_arrays(dims) + ... y = test_utils.simple_forward(op, flat_data=x) + ... y_np = x.reshape((dims[0], numpy.prod(dims[1:]))) + ... print('%s: %s' % (dims, test_utils.almost_equal(y, y_np))) + (2, 3, 4, 5): True + (2, 3): True + (2,): True + """ + + class FullyConnectedDoc(SymbolDoc): """ Examples @@ -77,7 +141,6 @@ class FullyConnectedDoc(SymbolDoc): >>> test_utils.almost_equal(out, out_np) True """ - pass class ConcatDoc(SymbolDoc): @@ -97,7 +160,6 @@ class ConcatDoc(SymbolDoc): Note the shape should be the same except on the dimension that is being concatenated. """ - pass class BroadcastPlusDoc(SymbolDoc): diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 96cb3bf5..0be646af 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -34,8 +34,11 @@ def default_numerical_threshold(): def random_arrays(*shapes): """Generate some random numpy arrays.""" - return [np.random.randn(*s).astype(default_dtype()) - for s in shapes] + arrays = [np.random.randn(*s).astype(default_dtype()) + for s in shapes] + if len(arrays) == 1: + return arrays[0] + return arrays def np_reduce(dat, axis, keepdims, numpy_reduce_func): diff --git a/src/operator/activation.cc b/src/operator/activation.cc index 17b22565..e1b3180f 100644 --- a/src/operator/activation.cc +++ b/src/operator/activation.cc @@ -46,10 +46,18 @@ Operator *ActivationProp::CreateOperatorEx(Context ctx, std::vector *in_ DMLC_REGISTER_PARAMETER(ActivationParam); MXNET_REGISTER_OP_PROPERTY(Activation, ActivationProp) -.describe("Apply activation function to input. " - "Softmax Activation is only available with CUDNN on GPU " - "and will be computed at each location across channel if input is 4D.") -.add_argument("data", "Symbol", "Input data to activation function.") +.describe(R"(Elementwise activation function. + +The following activation types are supported (operations are applied elementwisely to each +scalar of the input tensor): + +- `relu`: Rectified Linear Unit, `y = max(x, 0)` +- `sigmoid`: `y = 1 / (1 + exp(-x))` +- `tanh`: Hyperbolic tangent, `y = (exp(x) - exp(-x)) / (exp(x) + exp(-x))` +- `softrelu`: Soft ReLU, or SoftPlus, `y = log(1 + exp(x))` + +See `LeakyReLU` for other activations with parameters. +)") .add_arguments(ActivationParam::__FIELDS__()); } // namespace op diff --git a/src/operator/reshape.cc b/src/operator/reshape.cc index ff5579b9..7719c019 100644 --- a/src/operator/reshape.cc +++ b/src/operator/reshape.cc @@ -36,7 +36,8 @@ MXNET_REGISTER_OP_PROPERTY(Reshape, ReshapeProp) .add_arguments(ReshapeParam::__FIELDS__()); MXNET_REGISTER_OP_PROPERTY(Flatten, FlattenProp) -.describe("Flatten input") +.describe(R"(Flatten input into 2D by collapsing all the higher dimensions. +A (d1, d2, ..., dK) tensor is flatten to (d1, d2* ... *dK) matrix.)") .add_argument("data", "Symbol", "Input data to flatten."); } // namespace op } // namespace mxnet diff --git a/tests/python/doctest/run.py b/tests/python/doctest/run.py index 93579b9f..b69b9eb9 100644 --- a/tests/python/doctest/run.py +++ b/tests/python/doctest/run.py @@ -1,10 +1,11 @@ import doctest import logging import mxnet +import numpy def import_into(globs, module, names=None, error_on_overwrite=True): """Import names from module into the globs dict. - + Parameters ---------- """ @@ -16,7 +17,7 @@ def import_into(globs, module, names=None, error_on_overwrite=True): mod_names = names for name in mod_names: - if name in globs: + if name in globs and globs[name] is not getattr(module, name): error_msg = 'Attempting to overwrite definition of %s' % name if error_on_overwrite: raise RuntimeError(error_msg) @@ -27,7 +28,7 @@ def import_into(globs, module, names=None, error_on_overwrite=True): def test_symbols(): - globs = {'mxnet': mxnet, 'test_utils': mxnet.test_utils} + globs = {'numpy': numpy, 'mxnet': mxnet, 'test_utils': mxnet.test_utils} # make sure all the operators are available import_into(globs, mxnet.symbol)