Streamlined fp16 examples (apache#7150)

Also added fp16 versions of inception-v3, inception-v4 and resnext

example/image-classification/common/data.py

@@ -19,8 +19,6 @@ def add_data_args(parser):
                       help='number of threads for data decoding')
     data.add_argument('--benchmark', type=int, default=0,
                       help='if 1, then feed the network with synthetic data')
-    data.add_argument('--dtype', type=str, default='float32',
-                      help='data type: float32 or float16')
     return data
 
 def add_data_aug_args(parser):
@@ -93,13 +91,9 @@ def reset(self):
 
 def get_rec_iter(args, kv=None):
     image_shape = tuple([int(l) for l in args.image_shape.split(',')])
-    dtype = np.float32;
-    if 'dtype' in args:
-        if args.dtype == 'float16':
-            dtype = np.float16
     if 'benchmark' in args and args.benchmark:
         data_shape = (args.batch_size,) + image_shape
-        train = SyntheticDataIter(args.num_classes, data_shape, 50, dtype)
+        train = SyntheticDataIter(args.num_classes, data_shape, 500, np.float32)
         return (train, None)
     if kv:
         (rank, nworker) = (kv.rank, kv.num_workers)

example/image-classification/common/fit.py

@@ -84,6 +84,8 @@ def add_fit_args(parser):
                        help='report the top-k accuracy. 0 means no report.')
     train.add_argument('--test-io', type=int, default=0,
                        help='1 means test reading speed without training')
+    train.add_argument('--dtype', type=str, default='float32',
+                       help='precision: float32 or float16')
     return train
 
 def fit(args, network, data_loader, **kwargs):

example/image-classification/symbols/alexnet.py

@@ -4,43 +4,48 @@
 Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. "Imagenet classification with deep convolutional neural networks." Advances in neural information processing systems. 2012.
 """
 import mxnet as mx
+import numpy as np
 
-def get_symbol(num_classes, **kwargs):
-    input_data = mx.symbol.Variable(name="data")
+def get_symbol(num_classes, dtype, **kwargs):
+    input_data = mx.sym.Variable(name="data")
+    if dtype == 'float16':
+        input_data = mx.sym.Cast(data=input_data, dtype=np.float16)
     # stage 1
-    conv1 = mx.symbol.Convolution(name='conv1',
+    conv1 = mx.sym.Convolution(name='conv1',
         data=input_data, kernel=(11, 11), stride=(4, 4), num_filter=96)
-    relu1 = mx.symbol.Activation(data=conv1, act_type="relu")
-    lrn1 = mx.symbol.LRN(data=relu1, alpha=0.0001, beta=0.75, knorm=2, nsize=5)
-    pool1 = mx.symbol.Pooling(
+    relu1 = mx.sym.Activation(data=conv1, act_type="relu")
+    lrn1 = mx.sym.LRN(data=relu1, alpha=0.0001, beta=0.75, knorm=2, nsize=5)
+    pool1 = mx.sym.Pooling(
         data=lrn1, pool_type="max", kernel=(3, 3), stride=(2,2))
     # stage 2
-    conv2 = mx.symbol.Convolution(name='conv2',
+    conv2 = mx.sym.Convolution(name='conv2',
         data=pool1, kernel=(5, 5), pad=(2, 2), num_filter=256)
-    relu2 = mx.symbol.Activation(data=conv2, act_type="relu")
-    lrn2 = mx.symbol.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5)
-    pool2 = mx.symbol.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max")
+    relu2 = mx.sym.Activation(data=conv2, act_type="relu")
+    lrn2 = mx.sym.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5)
+    pool2 = mx.sym.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max")
     # stage 3
-    conv3 = mx.symbol.Convolution(name='conv3',
+    conv3 = mx.sym.Convolution(name='conv3',
         data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=384)
-    relu3 = mx.symbol.Activation(data=conv3, act_type="relu")
-    conv4 = mx.symbol.Convolution(name='conv4',
+    relu3 = mx.sym.Activation(data=conv3, act_type="relu")
+    conv4 = mx.sym.Convolution(name='conv4',
         data=relu3, kernel=(3, 3), pad=(1, 1), num_filter=384)
-    relu4 = mx.symbol.Activation(data=conv4, act_type="relu")
-    conv5 = mx.symbol.Convolution(name='conv5',
+    relu4 = mx.sym.Activation(data=conv4, act_type="relu")
+    conv5 = mx.sym.Convolution(name='conv5',
         data=relu4, kernel=(3, 3), pad=(1, 1), num_filter=256)
-    relu5 = mx.symbol.Activation(data=conv5, act_type="relu")
-    pool3 = mx.symbol.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max")
+    relu5 = mx.sym.Activation(data=conv5, act_type="relu")
+    pool3 = mx.sym.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max")
     # stage 4
-    flatten = mx.symbol.Flatten(data=pool3)
-    fc1 = mx.symbol.FullyConnected(name='fc1', data=flatten, num_hidden=4096)
-    relu6 = mx.symbol.Activation(data=fc1, act_type="relu")
-    dropout1 = mx.symbol.Dropout(data=relu6, p=0.5)
+    flatten = mx.sym.Flatten(data=pool3)
+    fc1 = mx.sym.FullyConnected(name='fc1', data=flatten, num_hidden=4096)
+    relu6 = mx.sym.Activation(data=fc1, act_type="relu")
+    dropout1 = mx.sym.Dropout(data=relu6, p=0.5)
     # stage 5
-    fc2 = mx.symbol.FullyConnected(name='fc2', data=dropout1, num_hidden=4096)
-    relu7 = mx.symbol.Activation(data=fc2, act_type="relu")
-    dropout2 = mx.symbol.Dropout(data=relu7, p=0.5)
+    fc2 = mx.sym.FullyConnected(name='fc2', data=dropout1, num_hidden=4096)
+    relu7 = mx.sym.Activation(data=fc2, act_type="relu")
+    dropout2 = mx.sym.Dropout(data=relu7, p=0.5)
     # stage 6
-    fc3 = mx.symbol.FullyConnected(name='fc3', data=dropout2, num_hidden=num_classes)
-    softmax = mx.symbol.SoftmaxOutput(data=fc3, name='softmax')
+    fc3 = mx.sym.FullyConnected(name='fc3', data=dropout2, num_hidden=num_classes)
+    if dtype == 'float16':
+        fc3 = mx.sym.Cast(data=fc3, dtype=np.float32)
+    softmax = mx.sym.SoftmaxOutput(data=fc3, name='softmax')
     return softmax

example/image-classification/symbols/inception-v3.py

@@ -6,6 +6,7 @@
 Szegedy, Christian, et al. "Rethinking the Inception Architecture for Computer Vision." arXiv preprint arXiv:1512.00567 (2015).
 """
 import mxnet as mx
+import numpy as np
 
 def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''):
     conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix))
@@ -41,7 +42,7 @@ def Inception7B(data,
     tower_d3x3 = Conv(data, num_d3x3_red, name=('%s_tower' % name), suffix='_conv')
     tower_d3x3 = Conv(tower_d3x3, num_d3x3_1, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name=('%s_tower' % name), suffix='_conv_1')
     tower_d3x3 = Conv(tower_d3x3, num_d3x3_2, kernel=(3, 3), pad=(0, 0), stride=(2, 2), name=('%s_tower' % name), suffix='_conv_2')
-    pooling = mx.symbol.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pad=(0,0), pool_type="max", name=('max_pool_%s_pool' % name))
+    pooling = mx.sym.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pad=(0,0), pool_type="max", name=('max_pool_%s_pool' % name))
     concat = mx.sym.Concat(*[tower_3x3, tower_d3x3, pooling], name='ch_concat_%s_chconcat' % name)
     return concat
 
@@ -104,8 +105,13 @@ def Inception7E(data,
 
 # In[49]:
 
-def get_symbol(num_classes=1000, **kwargs):
-    data = mx.symbol.Variable(name="data")
+def get_symbol(num_classes=1000, dtype='float32', **kwargs):
+    data = mx.sym.Variable(name="data")
+    if dtype == 'float32':
+        data = mx.sym.identity(data=data, name='id')
+    else:
+        if dtype == 'float16':
+            data = mx.sym.Cast(data=data, dtype=np.float16)
     # stage 1
     conv = Conv(data, 32, kernel=(3, 3), stride=(2, 2), name="conv")
     conv_1 = Conv(conv, 32, kernel=(3, 3), name="conv_1")
@@ -163,6 +169,8 @@ def get_symbol(num_classes=1000, **kwargs):
     # pool
     pool = mx.sym.Pooling(data=in5b, kernel=(8, 8), stride=(1, 1), pool_type="avg", name="global_pool")
     flatten = mx.sym.Flatten(data=pool, name="flatten")
-    fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=num_classes, name='fc1')
-    softmax = mx.symbol.SoftmaxOutput(data=fc1, name='softmax')
+    fc1 = mx.sym.FullyConnected(data=flatten, num_hidden=num_classes, name='fc1')
+    if dtype == 'float16':
+        fc1 = mx.sym.Cast(data=fc1, dtype=np.float32)
+    softmax = mx.sym.SoftmaxOutput(data=fc1, name='softmax')
     return softmax

example/image-classification/symbols/inception-v4.py

@@ -12,13 +12,13 @@
     Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke
     arXiv.1602.07261
 '''
-import find_mxnet
 import mxnet as mx
+import numpy as np
 
 def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''):
-    conv = mx.symbol.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix))
-    bn = mx.symbol.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True)
-    act = mx.symbol.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix))
+    conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix))
+    bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True)
+    act = mx.sym.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix))
 
     return act
 
@@ -28,9 +28,9 @@ def Inception_stem(data, name= None):
     c = Conv(c, 32, kernel=(3, 3), name='%s_conv2_3*3' %name)
     c = Conv(c, 64, kernel=(3, 3), pad=(1, 1), name='%s_conv3_3*3' %name)
 
-    p1 = mx.symbol.Pooling(c, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name)
+    p1 = mx.sym.Pooling(c, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name)
     c2 = Conv(c, 96, kernel=(3, 3), stride=(2, 2), name='%s_conv4_3*3' %name)
-    concat = mx.symbol.Concat(*[p1, c2], name='%s_concat_1' %name)
+    concat = mx.sym.Concat(*[p1, c2], name='%s_concat_1' %name)
 
     c1 = Conv(concat, 64, kernel=(1, 1), pad=(0, 0), name='%s_conv5_1*1' %name)
     c1 = Conv(c1, 96, kernel=(3, 3), name='%s_conv6_3*3' %name)
@@ -40,18 +40,18 @@ def Inception_stem(data, name= None):
     c2 = Conv(c2, 64, kernel=(1, 7), pad=(0, 3), name='%s_conv9_1*7' %name)
     c2 = Conv(c2, 96, kernel=(3, 3), pad=(0, 0), name='%s_conv10_3*3' %name)
 
-    concat = mx.symbol.Concat(*[c1, c2], name='%s_concat_2' %name)
+    concat = mx.sym.Concat(*[c1, c2], name='%s_concat_2' %name)
 
     c1 = Conv(concat, 192, kernel=(3, 3), stride=(2, 2), name='%s_conv11_3*3' %name)
-    p1 = mx.symbol.Pooling(concat, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_2' %name)
+    p1 = mx.sym.Pooling(concat, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_2' %name)
 
-    concat = mx.symbol.Concat(*[c1, p1], name='%s_concat_3' %name)
+    concat = mx.sym.Concat(*[c1, p1], name='%s_concat_3' %name)
 
     return concat
 
 
 def InceptionA(input, name=None):
-    p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name)
+    p1 = mx.sym.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name)
     c1 = Conv(p1, 96, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name)
 
     c2 = Conv(input, 96, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name)
@@ -63,26 +63,26 @@ def InceptionA(input, name=None):
     c4 = Conv(c4, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv6_3*3' % name)
     c4 = Conv(c4, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv7_3*3' %name)
 
-    concat = mx.symbol.Concat(*[c1, c2, c3, c4], name='%s_concat_1' %name)
+    concat = mx.sym.Concat(*[c1, c2, c3, c4], name='%s_concat_1' %name)
 
     return concat
 
 
 def ReductionA(input, name=None):
-    p1 = mx.symbol.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name)
+    p1 = mx.sym.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name)
 
     c2 = Conv(input, 384, kernel=(3, 3), stride=(2, 2), name='%s_conv1_3*3' %name)
 
     c3 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name)
     c3 = Conv(c3, 224, kernel=(3, 3), pad=(1, 1), name='%s_conv3_3*3' %name)
     c3 = Conv(c3, 256, kernel=(3, 3), stride=(2, 2), pad=(0, 0), name='%s_conv4_3*3' %name)
 
-    concat = mx.symbol.Concat(*[p1, c2, c3], name='%s_concat_1' %name)
+    concat = mx.sym.Concat(*[p1, c2, c3], name='%s_concat_1' %name)
 
     return concat
 
 def InceptionB(input, name=None):
-    p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name)
+    p1 = mx.sym.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name)
     c1 = Conv(p1, 128, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name)
 
     c2 = Conv(input, 384, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name)
@@ -103,7 +103,7 @@ def InceptionB(input, name=None):
     return concat
 
 def ReductionB(input,name=None):
-    p1 = mx.symbol.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name)
+    p1 = mx.sym.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name)
 
     c2 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name)
     c2 = Conv(c2, 192, kernel=(3, 3), stride=(2, 2), name='%s_conv2_3*3' %name)
@@ -113,13 +113,13 @@ def ReductionB(input,name=None):
     c3 = Conv(c3, 320, kernel=(7, 1), pad=(3, 0), name='%s_conv5_7*1' %name)
     c3 = Conv(c3, 320, kernel=(3, 3), stride=(2, 2), name='%s_conv6_3*3' %name)
 
-    concat = mx.symbol.Concat(*[p1, c2, c3], name='%s_concat_1' %name)
+    concat = mx.sym.Concat(*[p1, c2, c3], name='%s_concat_1' %name)
 
     return concat
 
 
 def InceptionC(input, name=None):
-    p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name)
+    p1 = mx.sym.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name)
     c1 = Conv(p1, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name)
 
     c2 = Conv(input, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name)
@@ -134,13 +134,18 @@ def InceptionC(input, name=None):
     c4_1 = Conv(c4, 256, kernel=(3, 1), pad=(1, 0), name='%s_conv9_1*3' %name)
     c4_2 = Conv(c4, 256, kernel=(1, 3), pad=(0, 1), name='%s_conv10_3*1' %name)
 
-    concat = mx.symbol.Concat(*[c1, c2, c3_1, c3_2, c4_1, c4_2], name='%s_concat' %name)
+    concat = mx.sym.Concat(*[c1, c2, c3_1, c3_2, c4_1, c4_2], name='%s_concat' %name)
 
     return concat
 
 
-def get_symbol(num_classes=1000, **kwargs):
-    data = mx.symbol.Variable(name="data")
+def get_symbol(num_classes=1000, dtype='float32', **kwargs):
+    data = mx.sym.Variable(name="data")
+    if dtype == 'float32':
+        data = mx.sym.identity(data=data, name='id')
+    else:
+        if dtype == 'float16':
+            data = mx.sym.Cast(data=data, dtype=np.float16)
     x = Inception_stem(data, name='in_stem')
 
     #4 * InceptionA
@@ -179,13 +184,15 @@ def get_symbol(num_classes=1000, **kwargs):
         x = InceptionC(x, name='in%dC' %(i+1))
 
     #Average Pooling
-    x = mx.symbol.Pooling(x, kernel=(8, 8), pad=(1, 1), pool_type='avg', name='global_avgpool')
+    x = mx.sym.Pooling(x, kernel=(8, 8), pad=(1, 1), pool_type='avg', name='global_avgpool')
 
     #Dropout
-    x = mx.symbol.Dropout(x, p=0.2)
+    x = mx.sym.Dropout(x, p=0.2)
 
-    flatten = mx.symbol.Flatten(x, name='flatten')
-    fc1 = mx.symbol.FullyConnected(flatten, num_hidden=num_classes, name='fc1')
-    softmax = mx.symbol.SoftmaxOutput(fc1, name='softmax')
+    flatten = mx.sym.Flatten(x, name='flatten')
+    fc1 = mx.sym.FullyConnected(flatten, num_hidden=num_classes, name='fc1')
+    if dtype == 'float16':
+        fc1 = mx.sym.Cast(data=fc1, dtype=np.float32)
+    softmax = mx.sym.SoftmaxOutput(fc1, name='softmax')
 
     return softmax