Update README in PyTorch examples (intel#428)

examples/optimization/pytorch/huggingface/README.md

@@ -6,13 +6,13 @@ Welcome to Pytorch Huggingface examples. The examples is following from [Hugging
 
 | Task | PostTrainingDynamic | PostTrainingStatic | QuantizationAwareTraining
 |---|:---:|:---:|:---:|
-|**`language-modeling`**| ✅ | ✅ | N/A
-|**`multi-choice`**| ✅ | ✅ | N/A
-|**`question-answering`**| ✅ | ✅ | N/A
+|**`language-modeling`**| ✅ | ✅ | ✅
+|**`multi-choice`**| ✅ | ✅ | ✅
+|**`question-answering`**| ✅ | ✅ | ✅
 |**`text-classification`**| ✅ | ✅ | ✅
-|**`token-classification`**| ✅ | ✅ | N/A
-|**`summarization`**| ✅ | N/A | N/A
-|**`translation`**| ✅ | N/A | N/A
+|**`token-classification`**| ✅ | ✅ | ✅
+|**`summarization`**| ✅ | ✅ | ✅
+|**`translation`**| ✅ | ✅ | ✅
 
 ## Pruning approach
 

examples/optimization/pytorch/huggingface/language-modeling/quantization/inc/README.md

@@ -54,9 +54,9 @@ python run_mlm.py \
 
 |Type|Pretrained model|PostTrainingDynamic | PostTrainingStatic | QuantizationAwareTraining
 |---|------------------------------------|---|---|---
-|CLM|EleutherAI/gpt-neo-125M| ✅| ✅| N/A
-|MLM|bert-base-uncased| ✅| ✅| N/A
-|PLM|xlnet-base-cased| ✅| ✅| N/A
+|CLM|EleutherAI/gpt-neo-125M| ✅| ✅| ✅
+|MLM|bert-base-uncased| ✅| ✅| ✅
+|PLM|xlnet-base-cased| ✅| ✅| ✅
 
 ### Command
 

examples/optimization/pytorch/huggingface/multiple-choice/quantization/inc/README.md

@@ -20,7 +20,7 @@ python run_swag.py \
 
 |DATASET|Pretrained model|PostTrainingDynamic | PostTrainingStatic | QuantizationAwareTraining
 |---|------------------------------------|---|---|---
-|SWAG|ehdwns1516/bert-base-uncased_SWAG| ✅| ✅| ✅ N/A
+|SWAG|ehdwns1516/bert-base-uncased_SWAG| ✅| ✅| ✅
 
 
 

examples/optimization/pytorch/huggingface/question-answering/quantization/inc/README.md

@@ -34,4 +34,4 @@ python run_qa.py \
 
 |Dataset|Pretrained model|PostTrainingDynamic | PostTrainingStatic | QuantizationAwareTraining
 |---|------------------------------------|---|---|---
-|squad|distilbert-base-uncased-distilled-squad| ✅| ✅| N/A
+|squad|distilbert-base-uncased-distilled-squad| ✅| ✅| ✅

examples/optimization/pytorch/huggingface/token-classification/quantization/inc/README.md

@@ -21,7 +21,7 @@ Here is how to run the script:
 
 |Dataset|Pretrained model|PostTrainingDynamic | PostTrainingStatic | QuantizationAwareTraining
 |---|------------------------------------|---|---|---
-|NER|elastic/distilbert-base-uncased-finetuned-conll03-english| ✅| ✅| N/A
+|NER|elastic/distilbert-base-uncased-finetuned-conll03-english| ✅| ✅| ✅
 
 
 

intel_extension_for_transformers/__init__.py

@@ -21,7 +21,6 @@
     DistillationConfig,
     FlashDistillationConfig,
     TFDistillationConfig,
-    NncfConfig,
     Provider,
     PruningConfig,
     QuantizationConfig,

intel_extension_for_transformers/optimization/config.py

@@ -30,15 +30,12 @@
 from typing import List, Union
 from xmlrpc.client import boolean
 
-nncf = LazyImport("nncf")
-
 
 WEIGHTS_NAME = "pytorch_model.bin"
 
 
 class Provider(Enum):
     INC = "inc"
-    NNCF = "nncf"
 
 
 class DynamicLengthConfig(object):
@@ -662,56 +659,3 @@ def metrics(self, metrics: Union[List, Metric]):
             self.config.auto_distillation.search.higher_is_better.append(
                 metric.greater_is_better
                 )
-
-
-# pylint: disable=E0401
-class NncfConfig(object):   # pragma: no cover
-    def __init__(
-        self,
-        nncf_config,
-        distributed: bool = False,
-        to_onnx: bool = False,
-        metrics: Union[List, Metric] = None,
-    ):
-        super().__init__()
-        from nncf import NNCFConfig
-        assert isinstance(nncf_config, NNCFConfig)
-        self.nncf_config = nncf_config
-        if metrics is not None:
-            self._metrics = metrics
-        self._distributed = distributed
-        self._to_onnx = to_onnx
-
-
-    @property
-    def distributed(self):
-        return self._distributed
-
-    @distributed.setter
-    def distributed(self, distributed):
-        self._distributed = distributed
-
-    @property
-    def to_onnx(self):
-        return self._to_onnx
-
-    @to_onnx.setter
-    def to_onnx(self, to_onnx):
-        self._to_onnx = to_onnx
-
-    @property
-    def metrics(self):
-        return self._metrics
-
-    @metrics.setter
-    def metrics(self, metrics):
-        self._metrics = metrics
-
-    @property
-    def nncf_config(self):
-        return self._nncf_config
-
-    @nncf_config.setter
-    def nncf_config(self, nncf_config):
-        self._nncf_config = nncf_config
-

intel_extension_for_transformers/optimization/optimizer.py

@@ -175,33 +175,6 @@ def init_quantizer(
         self.quantizer = quantizer
         return quantizer
 
-    # pylint: disable=E0401
-    def _nncf_quantize(self):   # pragma: no cover
-        from intel_extension_for_transformers import NncfConfig
-        from nncf import create_compressed_model
-        compression_state = None
-        assert isinstance(self.quant_config, NncfConfig), \
-            "Please pass a NNCFConfig instance to trainer.quantize!"
-
-        self.metrics = self.quant_config.metrics
-        nncf_compression_state_file = self.quant_config.compression_state
-
-        if os.path.isfile(nncf_compression_state_file):
-            compression_state = torch.load(nncf_compression_state_file)
-        else:
-            compression_state = None
-
-        compression_algo_controller, model = create_compressed_model(
-            self.model, self.quant_config.nncf_config,
-            compression_state=compression_state
-        )
-
-        self.compression_ctrl = \
-            compression_algo_controller.distributed() \
-            if self.quant_config.distributed else compression_algo_controller
-
-        self.model = self._train_func(model)
-
     def _inc_quantize(
         self,
         quant_config,
@@ -246,9 +219,7 @@ def quantize(
         if self.quantizer is None:
             self._provider = Provider[provider.upper()].value
 
-        if self._provider == Provider.NNCF.value:
-            return self._nncf_quantize()
-        elif self._provider == Provider.INC.value:
+        if self._provider == Provider.INC.value:
             return self._inc_quantize(quant_config=quant_config, provider=provider)
         else:
             assert False, "Unsupport provider:{}".format(self._provider)

intel_extension_for_transformers/optimization/trainer.py

@@ -235,31 +235,6 @@ def init_quantizer(
         self.quantizer = quantizer
         return quantizer
 
-    # pylint: disable=E0401
-    def _nncf_quantize(self):  # pragma: no cover
-        from intel_extension_for_transformers import NncfConfig
-        from nncf import create_compressed_model
-        compression_state = None
-        assert isinstance(self.quant_config, NncfConfig), \
-            "Please pass a NNCFConfig instance to trainer.quantize!"
-
-        self.metrics = self.quant_config.metrics
-        nncf_compression_state_file = self.quant_config.compression_state
-
-        if os.path.isfile(nncf_compression_state_file):
-            compression_state = torch.load(nncf_compression_state_file)
-        else:
-            compression_state = None
-
-        compression_algo_controller, model = create_compressed_model(
-            self.model, self.quant_config.nncf_config, compression_state=compression_state)
-
-        self.compression_ctrl = \
-            compression_algo_controller.distributed() \
-            if self.quant_config.distributed else compression_algo_controller
-
-        self.model = self._train_func(model)
-
     def _inc_quantize(
         self,
         quant_config,
@@ -312,9 +287,7 @@ def quantize(
         if self.quantizer is None:
             self._provider = Provider[provider.upper()].value
 
-        if self._provider == Provider.NNCF.value:  # pragma: no cover
-            return self._nncf_quantize()
-        elif self._provider == Provider.INC.value:
+        if self._provider == Provider.INC.value:
             return self._inc_quantize(quant_config=quant_config, provider=provider)
         else:
             assert False, "Unsupport provider:{}".format(self._provider)
@@ -980,15 +953,6 @@ def _maybe_log_save_evaluate(self, tr_loss, model, trial, epoch,
                 tr_loss_scalar / (self.state.global_step - self._globalstep_last_logged), 4)
             logs["learning_rate"] = self._get_learning_rate()
 
-            # pylint: disable=E0401
-            if self.compression_ctrl is not None:
-                from nncf.common.utils.tensorboard import prepare_for_tensorboard
-                logs["compression_loss"] = self.compression_ctrl.loss().item()
-                compression_stats = self.compression_ctrl.statistics()
-                for key, value in prepare_for_tensorboard(compression_stats).items():
-                    logs["compression/statistics/{0}".format(key)] = value
-                print(compression_stats.to_str())
-
             self._total_loss_scalar += tr_loss_scalar
             self._globalstep_last_logged = self.state.global_step
             self.store_flos()
@@ -1370,10 +1334,7 @@ def _remove_unused_columns(self,
             return dataset
         if self._signature_columns is None:
             # Inspect model forward signature to keep only the arguments it accepts.
-            if self._provider == "nncf":
-                signature = inspect.signature(self.model.get_nncf_wrapped_model().forward)
-            else:
-                signature = inspect.signature(self.model.forward)
+            signature = inspect.signature(self.model.forward)
             self._signature_columns = list(signature.parameters.keys())
             # Labels may be named label or label_ids, the default data collator handles that.
             self._signature_columns += ["label", "label_ids", "teacher_logits"]
@@ -1712,11 +1673,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None):
         # They can then be reloaded using `from_pretrained()`
         if not isinstance(self.model, PreTrainedModel):  # pragma: no cover
             unwrapped_model = unwrap_model(self.model)
-            if self._provider == "nncf":
-                is_pretrained = isinstance(unwrapped_model.get_nncf_wrapped_model(),
-                                           PreTrainedModel)
-            else:
-                is_pretrained = isinstance(unwrapped_model, PreTrainedModel)
+            is_pretrained = isinstance(unwrapped_model, PreTrainedModel)
 
             if is_pretrained:
                 if state_dict is None: