tab없애기...ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader # 데이터로더\n",
    "from gluonnlp.data import SentencepieceTokenizer \n",
    "from kogpt2.utils import get_tokenizer\n",
    "from kogpt2.utils import download, tokenizer\n",
    "from kogpt2.model.torch_gpt2 import GPT2Config, GPT2LMHeadModel\n",
    "from kogpt2.data import Read_Dataset\n",
    "import gluonnlp\n",
    "from kogpt2.model.sample import sample_sequence\n",
    "from tqdm import tqdm\n",
    "import subprocess\n",
    "import os\n",
    "from tensorboardX import SummaryWriter\n",
    "import re\n",
    "import argparse\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--epoch', type=int, default=200,\n",
    "\t\t\t\t\thelp=\"epoch 를 통해서 학습 범위를 조절합니다.\")\n",
    "parser.add_argument('--save_path', type=str, default='./checkpoint/',\n",
    "\t\t\t\t\thelp=\"학습 결과를 저장하는 경로입니다.\")\n",
    "parser.add_argument('--load_path', type=str, default='./checkpoint/KoGPT2_checkpoint_100000.tar', #\n",
    "\t\t\t\t\thelp=\"학습된 결과를 불러오는 경로입니다.\")\n",
    "parser.add_argument('--samples', type=str, default=\"samples/\",\n",
    "\t\t\t\t\thelp=\"생성 결과를 저장할 경로입니다.\")\n",
    "parser.add_argument('--data_file_path', type=str, default='dataset/lyrics_dataset.txt',\n",
    "\t\t\t\t\thelp=\"학습할 데이터를 불러오는 경로입니다.\")\n",
    "parser.add_argument('--batch_size', type=int, default=8,\n",
    "\t\t\t\t\thelp=\"batch_size 를 지정합니다.\")\n",
    "args = parser.parse_args()\n",
    "\n",
    "'''\n",
    "pytorch_kogpt2 = {\n",
    "\t'url':\n",
    "\t'checkpoint/pytorch_kogpt2_676e9bcfa7.params',\n",
    "\t'fname': 'pytorch_kogpt2_676e9bcfa7.params',\n",
    "\t'chksum': '676e9bcfa7'\n",
    "}\n",
    "'''\n",
    "pytorch_kogpt2 = {\n",
    "\t'url':\n",
    "\t'https://kobert.blob.core.windows.net/models/kogpt2/pytorch/pytorch_kogpt2_676e9bcfa7.params',\n",
    "\t'fname': 'pytorch_kogpt2_676e9bcfa7.params',\n",
    "\t'chksum': '676e9bcfa7'\n",
    "}\n",
    "\n",
    "\n",
    "kogpt2_config = {\n",
    "\t\"initializer_range\": 0.02,\n",
    "\t\"layer_norm_epsilon\": 1e-05,\n",
    "\t\"n_ctx\": 1024,\n",
    "\t\"n_embd\": 768,\n",
    "\t\"n_head\": 12,\n",
    "\t\"n_layer\": 12,\n",
    "\t\"n_positions\": 1024,\n",
    "\t\"vocab_size\": 50000\n",
    "}\n",
    "\n",
    "def auto_enter(text):\n",
    "\ttext = (text.replace(\"   \", \"\\n\"))\n",
    "\ttext = text.split(\"\\n\")\n",
    "\n",
    "\ttext = [t.lstrip() for t in text if t != '']\n",
    "\treturn \"\\n\\n\".join(text)\n",
    "\n",
    "def get_gpu_memory_map():\n",
    "\t\"\"\"Get the current gpu usage.\n",
    "\n",
    "\tReturns\n",
    "\t-------\n",
    "\tusage: dict\n",
    "\t\tKeys are device ids as integers.\n",
    "\t\tValues are memory usage as integers in MB.\n",
    "\t\"\"\"\n",
    "\tresult = subprocess.check_output(\n",
    "\t\t[\n",
    "\t\t\t'nvidia-smi', '--query-gpu=memory.used',\n",
    "\t\t\t'--format=csv,nounits,noheader'\n",
    "\t\t], encoding='utf-8')\n",
    "\t# Convert lines into a dictionary\n",
    "\tgpu_memory = [int(x) for x in result.strip().split('\\n')]\n",
    "\tgpu_memory_map = dict(zip(range(len(gpu_memory)), gpu_memory))\n",
    "\treturn gpu_memory_map\n",
    "\n",
    "def main(epoch, save_path, load_path, samples, data_file_path, batch_size):\n",
    "\tctx = 'cuda'\n",
    "\tcachedir = '~/kogpt2/'\n",
    "\n",
    "\tsummary = SummaryWriter()\n",
    "\n",
    "\t# download model\n",
    "\tmodel_info = pytorch_kogpt2\n",
    "\tmodel_path = download(model_info['url'],\n",
    "\t\t\t\t\t\t   model_info['fname'],\n",
    "\t\t\t\t\t\t   model_info['chksum'],\n",
    "\t\t\t\t\t\t   cachedir=cachedir)\n",
    "\t# download vocab\n",
    "\tvocab_info = tokenizer\n",
    "\tvocab_path = download(vocab_info['url'],\n",
    "\t\t\t\t\t\t   vocab_info['fname'],\n",
    "\t\t\t\t\t\t   vocab_info['chksum'],\n",
    "\t\t\t\t\t\t   cachedir=cachedir)\n",
    "\n",
    "\t# KoGPT-2 언어 모델 학습을 위한 GPT2LMHeadModel 선언\n",
    "\tkogpt2model = GPT2LMHeadModel(config=GPT2Config.from_dict(kogpt2_config))\n",
    "\n",
    "\t# model_path 로부터 다운로드 받은 내용을 load_state_dict 으로 업로드\n",
    "\tkogpt2model.load_state_dict(torch.load(model_path))\n",
    "\n",
    "\tdevice = torch.device(ctx)\n",
    "\tkogpt2model.to(device)\n",
    "\n",
    "\t# 불러오기 부분\n",
    "\ttry:\n",
    "\t\tcheckpoint = torch.load(load_path, map_location=device)\n",
    "\n",
    "\t\t# KoGPT-2 언어 모델 학습을 위한 GPT2LMHeadModel 선언\n",
    "\t\tkogpt2model = GPT2LMHeadModel(config=GPT2Config.from_dict(kogpt2_config))\n",
    "\t\tkogpt2model.load_state_dict(checkpoint['model_state_dict'])\n",
    "\n",
    "\t\tkogpt2model.eval()\n",
    "\texcept:\n",
    "\t\tcount = 0\n",
    "\telse:\n",
    "\t\tcount = int(re.findall(\"\\d+\", load_path)[1])\n",
    "\n",
    "\tprint(count)\n",
    "\t# 추가로 학습하기 위해 .train() 사용\n",
    "\tkogpt2model.train()\n",
    "\tvocab_b_obj = gluonnlp.vocab.BERTVocab.from_sentencepiece(vocab_path,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t mask_token=None,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t sep_token=None,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t cls_token=None,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t unknown_token='<unk>',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t padding_token='<pad>',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t bos_token='<s>',\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t eos_token='</s>')\n",
    "\n",
    "\n",
    "\ttok_path = get_tokenizer()\n",
    "\tmodel, vocab = kogpt2model, vocab_b_obj\n",
    "\ttok = SentencepieceTokenizer(tok_path)\n",
    "\n",
    "\tdataset = Read_Dataset(data_file_path, vocab, tok)\n",
    "\tprint(\"Read_Dataset ok\")\n",
    "\tdata_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, pin_memory=True)\n",
    "\n",
    "\n",
    "\n",
    "\tlearning_rate = 3e-5\n",
    "\tcriterion = torch.nn.CrossEntropyLoss()\n",
    "\toptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "\tprint('KoGPT-2 Transfer Learning Start')\n",
    "\tavg_loss = (0.0, 0.0)\n",
    "\n",
    "\tfor epoch in range(epoch):\n",
    "\t\tfor data in data_loader:\n",
    "\t\t\toptimizer.zero_grad()\n",
    "\t\t\t#print(data) \n",
    "\t\t\tdata = torch.stack(data) # list of Tensor로 구성되어 있기 때문에 list를 stack을 통해 변환해준다.\n",
    "\t\t\tdata = data.transpose(1,0)\n",
    "\t\t\tdata = data.to(ctx)\n",
    "\t\t\tmodel = model.to(ctx)\n",
    "\n",
    "\t\t\toutputs = model(data, labels=data)\n",
    "\t\t\tloss, logits = outputs[:2]\n",
    "\t\t\tloss = loss.to(ctx)\n",
    "\t\t\tloss.backward()\n",
    "\t\t\tavg_loss = (avg_loss[0] * 0.99 + loss, avg_loss[1] * 0.99 + 1.0)\n",
    "\t\t\toptimizer.step()\n",
    "\t\t\tif count % 10 == 0:\n",
    "\t\t\t\tprint('epoch no.{0} train no.{1}  loss = {2:.5f} avg_loss = {3:.5f}' . format(epoch, count, loss, avg_loss[0] / avg_loss[1]))\n",
    "\t\t\t\tsummary.add_scalar('loss/avg_loss', avg_loss[0] / avg_loss[1], count)\n",
    "\t\t\t\tsummary.add_scalar('loss/loss', loss, count)\n",
    "\n",
    "\t\t\t# generator 진행\n",
    "\t\t\tif (count > 0 and count % 1000 == 0) or (len(data) < batch_size):\n",
    "\t\t\t\tsent = sample_sequence(model.to(\"cpu\"), tok,vocab, sent=\"추억의\", text_size=100, temperature=0.7, top_p=0.8, top_k=40)\n",
    "\t\t\t\tsent = sent.replace(\"<unused0>\", \"\\n\") # 비효율적이지만 엔터를 위해서 등장\n",
    "\t\t\t\tsent = auto_enter(sent)\n",
    "\t\t\t\tprint(sent)\n",
    "\n",
    "\t\t\t\tsummary.add_text('Text', sent, count)\n",
    "\n",
    "\t\t\t\tif count > 20000:\n",
    "\t\t\t\t\tnow = []\n",
    "\t\t\t\t\tfor n in os.listdir(samples): #ipynb.checkpoint 땜에ㅠ\n",
    "\t\t\t\t\t\ttry:\n",
    "\t\t\t\t\t\t\t#print(int(n))\n",
    "\t\t\t\t\t\t\tnow.append(int(n))\n",
    "\t\t\t\t\t\texcept:\n",
    "\t\t\t\t\t\t\tcontinue\n",
    "\t\t\t\t\tif(len(now)==0):\n",
    "\t\t\t\t\t\tnow=0\n",
    "\t\t\t\t\telse:\n",
    "\t\t\t\t\t\tnow = max(now)\n",
    "\t\t\t\t\tf = open(samples + str(now + 1), 'w', encoding=\"utf-8\")\n",
    "\t\t\t\t\tf.write(sent)\n",
    "\t\t\t\t\tf.close()\n",
    "\t\t\t#########################################\n",
    "\t\t\tcount += 1\n",
    "\n",
    "\t\t\tif (count > 0 and count % 20000 == 0 and count>400000) or (len(data) < batch_size):\n",
    "\t\t\t\t# 모델 저장\n",
    "\t\t\t\ttry:\n",
    "\t\t\t\t\ttorch.save({\n",
    "\t\t\t\t\t\t'epoch': epoch,\n",
    "\t\t\t\t\t\t'train_no': count,\n",
    "\t\t\t\t\t\t'model_state_dict': model.state_dict(),\n",
    "\t\t\t\t\t\t'optimizer_state_dict': optimizer.state_dict(),\n",
    "\t\t\t\t\t\t'loss': loss\n",
    "\t\t\t\t\t}, save_path + 'KoGPT2_checkpoint_' + str(count) + '.tar')\n",
    "\t\t\t\texcept:\n",
    "\t\t\t\t\tpass\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\tmain(args.epoch, args.save_path, args.load_path, args.samples, args.data_file_path, args.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader # 데이터로더\n",
    "from gluonnlp.data import SentencepieceTokenizer \n",
    "from kogpt2.utils import get_tokenizer\n",
    "from kogpt2.utils import download, tokenizer\n",
    "from kogpt2.model.torch_gpt2 import GPT2Config, GPT2LMHeadModel\n",
    "from kogpt2.data import Read_Dataset\n",
    "import gluonnlp\n",
    "from kogpt2.model.sample import sample_sequence\n",
    "from tqdm import tqdm\n",
    "import subprocess\n",
    "import os\n",
    "from tensorboardX import SummaryWriter\n",
    "import re\n",
    "import argparse\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--epoch', type=int, default=200,\n",
    "                    help=\"epoch 를 통해서 학습 범위를 조절합니다.\")\n",
    "parser.add_argument('--save_path', type=str, default='./checkpoint/',\n",
    "                    help=\"학습 결과를 저장하는 경로입니다.\")\n",
    "parser.add_argument('--load_path', type=str, default='./checkpoint/KoGPT2_checkpoint_100000.tar', #\n",
    "                    help=\"학습된 결과를 불러오는 경로입니다.\")\n",
    "parser.add_argument('--samples', type=str, default=\"samples/\",\n",
    "                    help=\"생성 결과를 저장할 경로입니다.\")\n",
    "parser.add_argument('--data_file_path', type=str, default='dataset/lyrics_dataset.txt',\n",
    "                    help=\"학습할 데이터를 불러오는 경로입니다.\")\n",
    "parser.add_argument('--batch_size', type=int, default=8,\n",
    "                    help=\"batch_size 를 지정합니다.\")\n",
    "args = parser.parse_args()\n",
    "\n",
    "'''\n",
    "pytorch_kogpt2 = {\n",
    "    'url':\n",
    "    'checkpoint/pytorch_kogpt2_676e9bcfa7.params',\n",
    "    'fname': 'pytorch_kogpt2_676e9bcfa7.params',\n",
    "    'chksum': '676e9bcfa7'\n",
    "}\n",
    "'''\n",
    "pytorch_kogpt2 = {\n",
    "    'url':\n",
    "    'https://kobert.blob.core.windows.net/models/kogpt2/pytorch/pytorch_kogpt2_676e9bcfa7.params',\n",
    "    'fname': 'pytorch_kogpt2_676e9bcfa7.params',\n",
    "    'chksum': '676e9bcfa7'\n",
    "}\n",
    "\n",
    "\n",
    "kogpt2_config = {\n",
    "    \"initializer_range\": 0.02,\n",
    "    \"layer_norm_epsilon\": 1e-05,\n",
    "    \"n_ctx\": 1024,\n",
    "    \"n_embd\": 768,\n",
    "    \"n_head\": 12,\n",
    "    \"n_layer\": 12,\n",
    "    \"n_positions\": 1024,\n",
    "    \"vocab_size\": 50000\n",
    "}\n",
    "\n",
    "def auto_enter(text):\n",
    "    text = (text.replace(\"   \", \"\\n\"))\n",
    "    text = text.split(\"\\n\")\n",
    "\n",
    "    text = [t.lstrip() for t in text if t != '']\n",
    "    return \"\\n\\n\".join(text)\n",
    "\n",
    "def get_gpu_memory_map():\n",
    "    \"\"\"Get the current gpu usage.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    usage: dict\n",
    "        Keys are device ids as integers.\n",
    "        Values are memory usage as integers in MB.\n",
    "    \"\"\"\n",
    "    result = subprocess.check_output(\n",
    "        [\n",
    "            'nvidia-smi', '--query-gpu=memory.used',\n",
    "            '--format=csv,nounits,noheader'\n",
    "        ], encoding='utf-8')\n",
    "    # Convert lines into a dictionary\n",
    "    gpu_memory = [int(x) for x in result.strip().split('\\n')]\n",
    "    gpu_memory_map = dict(zip(range(len(gpu_memory)), gpu_memory))\n",
    "    return gpu_memory_map\n",
    "\n",
    "def main(epoch, save_path, load_path, samples, data_file_path, batch_size):\n",
    "    ctx = 'cuda'\n",
    "    cachedir = '~/kogpt2/'\n",
    "\n",
    "    summary = SummaryWriter()\n",
    "\n",
    "    # download model\n",
    "    model_info = pytorch_kogpt2\n",
    "    model_path = download(model_info['url'],\n",
    "                           model_info['fname'],\n",
    "                           model_info['chksum'],\n",
    "                           cachedir=cachedir)\n",
    "    # download vocab\n",
    "    vocab_info = tokenizer\n",
    "    vocab_path = download(vocab_info['url'],\n",
    "                           vocab_info['fname'],\n",
    "                           vocab_info['chksum'],\n",
    "                           cachedir=cachedir)\n",
    "\n",
    "    # KoGPT-2 언어 모델 학습을 위한 GPT2LMHeadModel 선언\n",
    "    kogpt2model = GPT2LMHeadModel(config=GPT2Config.from_dict(kogpt2_config))\n",
    "\n",
    "    # model_path 로부터 다운로드 받은 내용을 load_state_dict 으로 업로드\n",
    "    kogpt2model.load_state_dict(torch.load(model_path))\n",
    "\n",
    "    device = torch.device(ctx)\n",
    "    kogpt2model.to(device)\n",
    "\n",
    "    # 불러오기 부분\n",
    "    try:\n",
    "        checkpoint = torch.load(load_path, map_location=device)\n",
    "\n",
    "        # KoGPT-2 언어 모델 학습을 위한 GPT2LMHeadModel 선언\n",
    "        kogpt2model = GPT2LMHeadModel(config=GPT2Config.from_dict(kogpt2_config))\n",
    "        kogpt2model.load_state_dict(checkpoint['model_state_dict'])\n",
    "\n",
    "        kogpt2model.eval()\n",
    "    except:\n",
    "        count = 0\n",
    "    else:\n",
    "        count = int(re.findall(\"\\d+\", load_path)[1])\n",
    "\n",
    "    print(count)\n",
    "    # 추가로 학습하기 위해 .train() 사용\n",
    "    kogpt2model.train()\n",
    "    vocab_b_obj = gluonnlp.vocab.BERTVocab.from_sentencepiece(vocab_path,\n",
    "                                                         mask_token=None,\n",
    "                                                         sep_token=None,\n",
    "                                                         cls_token=None,\n",
    "                                                         unknown_token='<unk>',\n",
    "                                                         padding_token='<pad>',\n",
    "                                                         bos_token='<s>',\n",
    "                                                         eos_token='</s>')\n",
    "\n",
    "\n",
    "    tok_path = get_tokenizer()\n",
    "    model, vocab = kogpt2model, vocab_b_obj\n",
    "    tok = SentencepieceTokenizer(tok_path)\n",
    "\n",
    "    dataset = Read_Dataset(data_file_path, vocab, tok)\n",
    "    print(\"Read_Dataset ok\")\n",
    "    data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, pin_memory=True)\n",
    "\n",
    "\n",
    "\n",
    "    learning_rate = 3e-5\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "    print('KoGPT-2 Transfer Learning Start')\n",
    "    avg_loss = (0.0, 0.0)\n",
    "\n",
    "    for epoch in range(epoch):\n",
    "        for data in data_loader:\n",
    "            optimizer.zero_grad()\n",
    "            #print(data) \n",
    "            data = torch.stack(data) # list of Tensor로 구성되어 있기 때문에 list를 stack을 통해 변환해준다.\n",
    "            data = data.transpose(1,0)\n",
    "            data = data.to(ctx)\n",
    "            model = model.to(ctx)\n",
    "\n",
    "            outputs = model(data, labels=data)\n",
    "            loss, logits = outputs[:2]\n",
    "            loss = loss.to(ctx)\n",
    "            loss.backward()\n",
    "            avg_loss = (avg_loss[0] * 0.99 + loss, avg_loss[1] * 0.99 + 1.0)\n",
    "            optimizer.step()\n",
    "            if count % 10 == 0:\n",
    "                print('epoch no.{0} train no.{1}  loss = {2:.5f} avg_loss = {3:.5f}' . format(epoch, count, loss, avg_loss[0] / avg_loss[1]))\n",
    "                summary.add_scalar('loss/avg_loss', avg_loss[0] / avg_loss[1], count)\n",
    "                summary.add_scalar('loss/loss', loss, count)\n",
    "\n",
    "            # generator 진행\n",
    "            if (count > 0 and count % 1000 == 0) or (len(data) < batch_size):\n",
    "                sent = sample_sequence(model.to(\"cpu\"), tok,vocab, sent=\"추억의\", text_size=100, temperature=0.7, top_p=0.8, top_k=40)\n",
    "                sent = sent.replace(\"<unused0>\", \"\\n\") # 비효율적이지만 엔터를 위해서 등장\n",
    "                sent = auto_enter(sent)\n",
    "                print(sent)\n",
    "\n",
    "                summary.add_text('Text', sent, count)\n",
    "\n",
    "                if count > 20000:\n",
    "                    now = []\n",
    "                    for n in os.listdir(samples): #ipynb.checkpoint 땜에ㅠ\n",
    "                        try:\n",
    "                            #print(int(n))\n",
    "                            now.append(int(n))\n",
    "                        except:\n",
    "                            continue\n",
    "                    if(len(now)==0):\n",
    "                        now=0\n",
    "                    else:\n",
    "                        now = max(now)\n",
    "                    f = open(samples + str(now + 1), 'w', encoding=\"utf-8\")\n",
    "                    f.write(sent)\n",
    "                    f.close()\n",
    "            #########################################\n",
    "            count += 1\n",
    "\n",
    "            if (count > 0 and count % 20000 == 0 and count>400000) or (len(data) < batch_size):\n",
    "                # 모델 저장\n",
    "                try:\n",
    "                    torch.save({\n",
    "                        'epoch': epoch,\n",
    "                        'train_no': count,\n",
    "                        'model_state_dict': model.state_dict(),\n",
    "                        'optimizer_state_dict': optimizer.state_dict(),\n",
    "                        'loss': loss\n",
    "                    }, save_path + 'KoGPT2_checkpoint_' + str(count) + '.tar')\n",
    "                except:\n",
    "                    pass\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main(args.epoch, args.save_path, args.load_path, args.samples, args.data_file_path, args.batch_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}