[reconstruct] metric_v2; [bug fix] split batch,init logger [enhancement] generate V3_results.rst

Author: declmal

docs/mrt/V3_results.rst

@@ -5,7 +5,86 @@ MRT Quantization Results
 
 .. _mrt_quantization_results:
 
-**alexnet**:
-Iteration: 312 | evalfunc: top1=55.91% top5=78.75% | quantize: top1=51.54% top5=77.40% | Total Sample: 50080
+**alexnet**:Iteration: 312 | evalfunc: top1=55.90% top5=78.74% | quantize: top1=51.44% top5=77.23% | Total Sample: 50080
+
+
+
+**vgg19**:Iteration: 781 | evalfunc: top1=74.13% top5=91.77% | quantize: top1=73.29% top5=91.52% | Total Sample: 50048
+
+
+
+**densenet161**:Iteration: 312 | evalfunc: top1=77.62% top5=93.82% | quantize: top1=77.26% top5=93.67% | Total Sample: 50080
+
+
+
+**mobilenetv2_1.0**:Iteration: 312 | evalfunc: top1=72.05% top5=90.58% | quantize: top1=69.79% top5=89.14% | Total Sample: 50080
+
+
+
+**resnet18_v1**:Iteration: 312 | evalfunc: top1=70.96% top5=89.93% | quantize: top1=70.11% top5=89.61% | Total Sample: 50080
+
+
+
+**resnet18_v1b_0.89**:Iteration: 312 | evalfunc: top1=67.20% top5=87.45% | quantize: top1=63.79% top5=85.62% | Total Sample: 50080
+
+
+
+**resnet50_v1**:Iteration: 312 | evalfunc: top1=77.39% top5=93.59% | quantize: top1=76.45% top5=93.29% | Total Sample: 50080
+
+
+
+**resnet50_v2**:Iteration: 312 | evalfunc: top1=77.15% top5=93.44% | quantize: top1=74.15% top5=91.74% | Total Sample: 50080
+
+
+
+**squeezenet1.0**:Iteration: 312 | evalfunc: top1=57.19% top5=80.04% | quantize: top1=54.91% top5=78.64% | Total Sample: 50080
+
+
+
+**mobilenet1_0**:Iteration: 312 | evalfunc: top1=70.77% top5=89.97% | quantize: top1=66.11% top5=87.35% | Total Sample: 50080
+
+
+
+**shufflenet_v1**:Iteration: 312 | evalfunc: top1=63.48% top5=85.12% | quantize: top1=60.45% top5=82.95% | Total Sample: 50080
+
+
+
+**tf_inception_v3**:Iteration: 312 | evalfunc: top1=45.16% top5=67.93% | quantize: top1=49.62% top5=74.71% | Total Sample: 50080
+
+
+
+**cifar_resnet20_v1**:Iteration:  62 | evalfunc: top1=92.88% top5=99.78% | quantize: top1=92.82% top5=99.75% | Total Sample: 10080
+
+
+
+**mnist**:Iteration:  62 | evalfunc: top1=99.00% top5=100.00% | quantize: top1=98.96% top5=100.00% | Total Sample: 10080
+
+
+
+**qd10_resnetv1_20**:Iteration: 1732 | evalfunc: top1=85.72% top5=98.71% | quantize: top1=85.73% top5=98.70% | Total Sample: 277280
+
+
+
+**quickdraw**:Iteration: 1732 | evalfunc: top1=81.66% top5=98.22% | quantize: top1=81.57% top5=98.20% | Total Sample: 277280
+
+
+
+**ssd**:Iteration:  76 | evalfunc: 80.30% | quantize: 80.05% | Total Sample:  4928
+
+
+
+**ssd_512_mobilenet1.0_voc**:Iteration: 308 | evalfunc: 75.51% | quantize: 71.26% | Total Sample:  4944
+
+
+**yolo3_darknet53_voc**:Iteration: 102 | evalfunc: 81.51% | quantize: 81.51% | Total Sample:  4944
+
+
+
+**yolo3_mobilenet1.0_voc**:Iteration:  76 | evalfunc: 76.03% | quantize: 71.56% | Total Sample:  4928
+
+
+
+**trec**:Iteration:   3 | evalfunc: 98.19% | quantize: 97.99% | Total Sample:   640
+
 
 

python/mrt/V3/evaluate.py

@@ -33,6 +33,20 @@
 MRT_CFG.EVALUATE.DEVICE_IDS = None
 MRT_CFG.EVALUATE.ITER_NUM = 10
 
+def forward(net, data, ctx, baxis, olen):
+    #TODO(ryt.dev) documentation
+    """ Multiple xpu run support.
+    """
+    data = gluon.utils.split_and_load(
+        data, ctx_list=ctx, batch_axis=baxis, even_split=False)
+    outs = [net(d) for d in data]
+    if olen == 1:
+        outs = nd.concatenate(outs)
+    else:
+        outs = [nd.concatenate([outs[i][j] \
+            for i in range(len(outs))]) for j in range(olen)]
+    return outs
+
 def get_evaluation_info(cm_cfg, pass_cfg, logger=None):
     model_dir = cm_cfg.MODEL_DIR
     model_name = cm_cfg.MODEL_NAME
@@ -60,7 +74,6 @@ def get_evaluation_info(cm_cfg, pass_cfg, logger=None):
     # forward function for the orginal model
     model_prefix_fixed = model_prefix + ".fixed"
     omodel = Model.load(*load_fname(model_prefix_fixed))
-    #TODO(ryt.dev) [bug fix] load revised model
     graph = omodel.to_graph(ctx=ctx)
     dataset_name = conf_map["dataset_name"]
     input_shape = conf_map["input_shape"]
@@ -70,30 +83,26 @@ def get_evaluation_info(cm_cfg, pass_cfg, logger=None):
     baxis = get_batch_axis(input_shape)
     olen = len(omodel.symbol)
 
-    def forward(net, data, ctx):
-        """ Multiple xpu run support.
-        """
-        data = gluon.utils.split_and_load(
-            data, ctx_list=ctx, batch_axis=baxis, even_split=False)
-        outs = [net(d) for d in data]
-        if olen == 1:
-            outs = nd.concatenate(outs)
-        else:
-            outs = [nd.concatenate([outs[i][j] \
-                for i in range(len(outs))]) for j in range(olen)]
-        return outs
+    # def forward(net, data, ctx):
+        # """ Multiple xpu run support.
+        # """
+        # data = gluon.utils.split_and_load(
+            # data, ctx_list=ctx, batch_axis=baxis, even_split=False)
+        # outs = [net(d) for d in data]
+        # if olen == 1:
+            # outs = nd.concatenate(outs)
+        # else:
+            # outs = [nd.concatenate([outs[i][j] \
+                # for i in range(len(outs))]) for j in range(olen)]
+        # return outs
 
     def evalfunc(data, label):
-        outs = forward(graph, data, ctx=ctx)
+        # outs = forward(graph, data, ctx=ctx)
+        outs = forward(graph, data, ctx, baxis, olen)
         acc = dataset.validate(metric, outs, label)
         return acc
 
     # forward function for the quantized model
-    # TODO(ryt.dev) [bug fix] remove split batch check
-    num_xpus = len(ctx)
-    if batch % num_xpus:
-        raise RuntimeError("Batch must be divisible by the number of xpus")
-    split_batch = batch // num_xpus
     if conf_map.get("split_keys", "") != "":
         sym_all_file, prm_all_file, ext_all_file = load_fname(
             model_prefix, suffix="all.quantize", with_ext=True)
@@ -110,14 +119,13 @@ def evalfunc(data, label):
         oscales = mrt.get_output_scales()
         inputs_ext = mrt.get_inputs_ext()
         qmodel = mrt.current_model
-    rqmodel = reduce_graph(qmodel, {
-        'data': set_batch(input_shape, split_batch)})
-    qgraph = rqmodel.to_graph(ctx=ctx)
+    qgraph = qmodel.to_graph(ctx=ctx)
     qmetric = dataset.metrics()
 
     def quantize(data, label):
         data = sim.load_real_data(data, 'data', inputs_ext)
-        outs = forward(qgraph, data, ctx)
+        # outs = forward(qgraph, data, ctx)
+        outs = forward(qgraph, data, ctx, baxis, olen)
         outs = outs / oscales[0] if olen == 1 \
             else [(t / oscales[i]) for i, t in enumerate(outs)]
         acc = dataset.validate(qmetric, outs, label)
@@ -143,20 +151,6 @@ def evaluate(cm_cfg, pass_cfg, logger=None):
     else:
         logger.info("evaluatation stage skipped")
 
-def forward(net, data, ctx, baxis, olen):
-    #TODO(ryt.dev) documentation
-    """ Multiple xpu run support.
-    """
-    data = gluon.utils.split_and_load(
-        data, ctx_list=ctx, batch_axis=baxis, even_split=False)
-    outs = [net(d) for d in data]
-    if olen == 1:
-        outs = nd.concatenate(outs)
-    else:
-        outs = [nd.concatenate([outs[i][j] \
-            for i in range(len(outs))]) for j in range(olen)]
-    return outs
-
 def get_ctx_eval(ctx):
     #TODO(ryt.dev) documentation
     if isinstance(ctx, mx.Context):

python/mrt/V3/execute.py

@@ -12,6 +12,7 @@
 from mrt.V3.quantize import quantize
 from mrt.V3.evaluate import evaluate
 from mrt.V3.mrt_compile import mrt_compile
+from mrt.V3.utils import get_logger
 
 thismodule = sys.modules[__name__]
 
@@ -68,6 +69,7 @@ def run(cfg, logger=None):
         Console logger.
     """
     pass_name = cfg.COMMON.PASS_NAME
+    logger = get_logger(cfg.COMMON.VERBOSITY)
     if pass_name == "all":
         yaml_main(cfg, logger=logger)
     else:

python/mrt/dataset.py

@@ -611,7 +611,6 @@ def data_loader():
                     yield nd.transpose(nd.array(data)), nd.array(label)
                     data, label = [], []
             yield nd.transpose(nd.array(data)), nd.array(label)
-            raise RuntimeError("Data loader have been the end")
 
         self.data = data_loader()
 

python/mrt/transformer.py

@@ -109,6 +109,7 @@ def to_cvm(self, model_name, datadir="/data/stdout",
                               device_ids=device_ids)
 
     def fix_original_model(self, model_dir, model_name):
+        # unify graph names and check graph params
         _sym, _prm = tpass.unify_name_json(self.symbol, self.params)
         self.symbol, self.params = tpass.remove_params_prefix(_sym, _prm)
         model_prefix = path.join(model_dir, model_name+".fixed")
@@ -122,11 +123,6 @@ def init(model, input_shape=None):
     logger.info("Model initializing...")
 
     _sym, _prm = model.symbol, model.params
-
-    # unify graph names and check graph params
-    # TODO(ryt.dev) [bug fix, reconstruct] write fixed model in conf_map, move to fix_orginal_model
-    #  _sym, _prm = fix_original_model(sym, parmas)
-
     tpass.name_duplicate_check(_sym, _prm)
 
     if isinstance(input_shape, dict):

tests/mrt/model_zoo/alexnet.yaml

@@ -8,12 +8,12 @@ CALIBRATE:
   LAMBD: 16
   DATASET_NAME: imagenet
   DEVICE_TYPE: gpu
-  DEVICE_IDS: [2]
+  DEVICE_IDS: [0]
 QUANTIZE:
   INPUT_PRECISION: 8
   OUTPUT_PRECISION: 8
   DEVICE_TYPE: gpu
-  DEVICE_IDS: [2]
+  DEVICE_IDS: [0]
 EVALUATE:
   BATCH: 160
   DEVICE_TYPE: gpu

tests/mrt/model_zoo/ssd_512_mobilenet1.0_voc.yaml

@@ -42,7 +42,7 @@ QUANTIZE:
     ["ssd0_slice_axis43", "ssd0_normalizedboxcenterdecoder0_concat0"],
   ]
 EVALUATE:
-  BATCH: 64
+  BATCH: 16
   DEVICE_TYPE: gpu
   DEVICE_IDS: [0]
   ITER_NUM: 2000

tests/mrt/model_zoo/yolo3_mobilenet1.0_voc.yaml

@@ -16,11 +16,11 @@ CALIBRATE:
   NUM_CALIB: 1
   DATASET_NAME: voc
   DEVICE_TYPE: gpu
-  DEVICE_IDS: [2]
+  DEVICE_IDS: [0]
 QUANTIZE:
   OUTPUT_PRECISION: 30
   DEVICE_TYPE: gpu
-  DEVICE_IDS: [2]
+  DEVICE_IDS: [0]
   THRESHOLDS: [
     ["mrt_rewrite_mobilenet0_conv23_weight_0", 11],
     ["yolov30_yolooutputv30_tile0", 416],
@@ -43,5 +43,5 @@ QUANTIZE:
 EVALUATE:
   BATCH: 64
   DEVICE_TYPE: gpu
-  DEVICE_IDS: [2]
+  DEVICE_IDS: [0]
   ITER_NUM: 2000