refine README and python files

distilbert_finetune.py

@@ -0,0 +1,12 @@
+import torch
+from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
+
+tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
+model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased")
+
+inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
+with torch.no_grad():
+    logits = model(**inputs).logits
+
+predicted_class_id = logits.argmax().item()
+model.config.id2label[predicted_class_id]

training/benchmarks/gpt2/pytorch/config/_base.py

@@ -1,8 +1,13 @@
+# Required parameters
+
 vendor: str = None
+data_dir: str = None
 name: str = "GPT2"
+cudnn_benchmark: bool = False
+cudnn_deterministic: bool = True
+
 use_env: bool = True
 log_freq: int = 1
-dist_backend: str = None
 device: str = None
 
 # =========================================================
@@ -24,7 +29,6 @@
 # data
 # =========================================================
 
-data_dir: str = None
 train_data_prefix: str = "lambada_train_text_document"
 test_data_prefix: str = "lambada_test.json"
 init_checkpoint: str = "model_optim_rng.pt"
@@ -113,6 +117,7 @@
 # distributed parallel
 # =========================================================
 
+dist_backend: str = None
 DDP_impl: str = "native"
 gradient_accumulation_fusion: bool = False
 use_contiguous_buffers_in_local_ddp: bool = False

training/benchmarks/gpt2/pytorch/dataloaders/data_samplers.py → training/benchmarks/gpt2/pytorch/dataloaders/dataloader.py

@@ -64,7 +64,6 @@ def build_data_loader(dataset, train_batch_size, num_workers, drop_last,
 
     return data_loader
 
-
 class MegatronPretrainingSampler:
 
     def __init__(self, total_samples, consumed_samples, train_batch_size,
@@ -112,4 +111,3 @@ def __iter__(self):
         if len(batch) > 0 and not self.drop_last:
             start_idx, end_idx = self.get_start_end_idx()
             yield batch[start_idx:end_idx]
-

training/benchmarks/gpt2/pytorch/dataloaders/gpt_dataset.py

@@ -8,7 +8,7 @@
 import torch
 
 from dataloaders.indexed_dataset import make_dataset as make_indexed_dataset
-from dataloaders.data_samplers import build_pretraining_data_loader, build_data_loader
+from dataloaders.dataloader import build_pretraining_data_loader, build_data_loader
 from dataloaders import get_tokenizer
 
 import config

training/benchmarks/gpt2/pytorch/model/losses/cross_entropy.py

@@ -2,9 +2,13 @@
 
 
 def cross_entropy(outputs, target):
-    #para:   outputs, [b, s, vocab_size]
-    #        target, [b, s]
-    #return: loss, [b, s]
+    """
+    Compute the cross entropy loss of output and target. 
+
+    para:   outputs, [b, s, vocab_size]
+            target, [b, s]
+    return: loss, [b, s]
+    """
 
     logits = outputs.clone()
     # logits = outputs

training/benchmarks/gpt2/pytorch/run_pretraining.py

@@ -1,8 +1,8 @@
-"""GPT2 Pretraining"""
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""GPT2 Pretraining"""
 
 import argparse
 import os
@@ -29,7 +29,6 @@
 
 def main():
     import config
-    from config import mutable_params
     global logger
 
     if config.use_env and 'LOCAL_RANK' in os.environ:

training/benchmarks/gpt2/pytorch/train/trainer.py

@@ -1,3 +1,7 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+
 import math
 import time
 import os

training/benchmarks/gpt2/pytorch/train/trainer_adapter.py

@@ -1,3 +1,7 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+
 import os
 from typing import Tuple
 

training/benchmarks/gpt2/pytorch/train/training_state.py

@@ -1,3 +1,7 @@
+# Copyright © 2022 BAAI. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+
 from dataclasses import dataclass
 
 import torch

training/nvidia/gpt2-pytorch/README.md

@@ -21,7 +21,7 @@
 | 指标名称       | 指标值                  | 特殊说明                                    |
 | -------------- | ----------------------- | ------------------------------------------- |
 | 任务类别       | 自然语言编码            |                                             |
-| 模型           | bert-large-uncased      |                                             |
+| 模型           | megatron-gpt2-345m      |                                             |
 | 数据集         | Wikipedia               |                                             |
 | 数据精度       | precision,见“性能指标”  | 可选fp32/amp/fp16                           |
 | 超参修改       | fix_hp,见“性能指标”     | 跑满硬件设备评测吞吐量所需特殊超参          |
@@ -38,4 +38,4 @@
 
 | 配置                | precision | fix_hp           | e2e_time | p_whole | p_train | p_core | lambada_acc | mem       |
 | ------------------- | --------- | ---------------- | -------- | ------- | ------- | ------ | ------- | --------- |
-| A100单机8卡（1x8）  |  fp32      | bs=32,lr=0.00015 |          | 2.30    | 88.36    | 89.57   |         | 33.7/40.0 |
+| A100单机8卡（1x8）  |  fp32      | bs=32,lr=0.00015 |  853.75   | 2.30    | 88.36    | 89.57 |  0.7001 | 33.7/40.0 |

training/nvidia/gpt2-pytorch/config/config_A100x1x8.py

@@ -1,3 +1 @@
 from config_common import *
-
-dist_backend = "nccl"

training/nvidia/gpt2-pytorch/config/config_common.py

@@ -1,5 +1,6 @@
-vendor = 'kunlunxin'
+vendor = 'nvidia'
 
 # disable fp16
 fp16 = False
 
+dist_backend = "nccl"