seq2seq 오류 체킹

Author: iml1111

src/11_seq2seq/data_preparation/detokenizer.py

@@ -1,4 +1,7 @@
+#-*- coding:utf-8 -*-
 import sys
+sys.stdin.reconfigure(encoding='utf-8')
+
 
 if __name__ == "__main__":
     for line in sys.stdin:

src/11_seq2seq/modules/data_loader.py

@@ -7,9 +7,9 @@ class DataLoader:
 
     def __init__(
         self,
-        train_fn,
-        valid_fn,
-        exts,
+        train_fn=None,
+        valid_fn=None,
+        exts=None,
         batch_size=64,
         device='cpu',
         max_vocab=9999999,
@@ -39,40 +39,41 @@ def __init__(
             eos_token=None,
         )
 
-        train = TranslationDataset(
-            path=train_fn,
-            exts=exts,
-            fields=[('src', self.src), ('tgt', self.tgt)],
-            max_length=max_length
-        )
-        valid = TranslationDataset(
-            path=valid_fn,
-            exts=exts,
-            fields=[('src', self.src), ('tgt', self.tgt)],
-            max_length=max_length,
-        )
-
-        self.train_iter = data.BucketIterator(
-            train,
-            batch_size=batch_size,
-            device='cuda:%d' % device if device >= 0 else 'cpu',
-            shuffle=shuffle,
-            # 비슷한 길이끼리 미니 배치를 만들도록 정렬
-            sort_key=lambda x: len(x.tgt) + (max_length * len(x.src)),
-            sort_within_batch=True,
-        )
-        self.valid_iter = data.BucketIterator(
-            valid,
-            batch_size=batch_size,
-            device='cuda:%d' % device if device >= 0 else 'cpu',
-            shuffle=False,
-            # 비슷한 길이끼리 미니 배치를 만들도록 정렬
-            sort_key=lambda x: len(x.tgt) + (max_length * len(x.src)),
-            sort_within_batch=True,
-        )
-
-        self.src.build_vocab(train, max_size=max_vocab)
-        self.tgt.build_vocab(train, max_size=max_vocab)
+        if train_fn is not None and valid_fn is not None and exts is not None:
+            train = TranslationDataset(
+                path=train_fn,
+                exts=exts,
+                fields=[('src', self.src), ('tgt', self.tgt)],
+                max_length=max_length
+            )
+            valid = TranslationDataset(
+                path=valid_fn,
+                exts=exts,
+                fields=[('src', self.src), ('tgt', self.tgt)],
+                max_length=max_length,
+            )
+
+            self.train_iter = data.BucketIterator(
+                train,
+                batch_size=batch_size,
+                device='cuda:%d' % device if device >= 0 else 'cpu',
+                shuffle=shuffle,
+                # 비슷한 길이끼리 미니 배치를 만들도록 정렬
+                sort_key=lambda x: len(x.tgt) + (max_length * len(x.src)),
+                sort_within_batch=True,
+            )
+            self.valid_iter = data.BucketIterator(
+                valid,
+                batch_size=batch_size,
+                device='cuda:%d' % device if device >= 0 else 'cpu',
+                shuffle=False,
+                # 비슷한 길이끼리 미니 배치를 만들도록 정렬
+                sort_key=lambda x: len(x.tgt) + (max_length * len(x.src)),
+                sort_within_batch=True,
+            )
+
+            self.src.build_vocab(train, max_size=max_vocab)
+            self.tgt.build_vocab(train, max_size=max_vocab)
 
     def load_vocab(self, src_vocab, tgt_vocab):
         self.src.vocab = src_vocab

src/11_seq2seq/modules/seq2seq.py

@@ -17,8 +17,8 @@ def __init__(self, word_vec_size, hidden_size, n_layers=4, dropout_p=.2):
             hidden_size=int(hidden_size / 2),
             num_layers=n_layers,
             dropout=dropout_p,
+            bidirectional=True,
             batch_first=True,
-            bidirectional=True
         )
 
     def forward(self, emb):
@@ -264,6 +264,9 @@ def forward(self, src, tgt):
             mask = self.generate_mask(x, x_length)
         else:
             x = src
+
+        if isinstance(tgt, tuple):
+            tgt = tgt[0]
 
         #---------Encoder Step---------#
         # emb_src = (batch_size, length_n, word_vec_size)

src/11_seq2seq/modules/trainer.py

@@ -113,10 +113,7 @@ def train(engine, mini_batch):
 
         # 현재 batch 내에 모든 토큰 수
         word_count = int(mini_batch.tgt[1].sum())
-
-        # 점점 커짐
         p_norm = float(get_parameter_norm(engine.model.parameters()))
-        # 점점 작아짐
         g_norm = float(get_grad_norm(engine.model.parameters()))
 
         # Gradient Accumulation 여부, 맞아 떨어진다면 step까지 수행, 아니면 스킵

src/11_seq2seq/train.py

@@ -179,31 +179,6 @@ def get_optimizer(model, config):
     return optimizer
 
 
-def get_scheduler(optimizer, config):
-    '''
-    # learing_rate 스케쥴러
-    학습도중, learning_rate를 조정하기 위한 역할
-    ex) epoch 9까지 lr:1로 하다가, 10부터 0.5, 0.25, 0.125 식으로 낮춰라 등
-    아래 코드에서는 lr_decay_start번째부터 config.lr_step를 곱하며 낮춰감
-    # 그러나 이부분은 사용하지 않음 X
-    '''
-    if config.lr_step > 0:
-        lr_scheduler = optim.lr_scheduler.MultiStepLR(
-            optimizer,
-            milestones=[i for i in range(
-                max(0, config.lr_decay_start - 1),
-                (config.init_epoch - 1) + config.n_epochs,
-                config.lr_step
-            )],
-            gamma=config.lr_gamma,
-            last_epoch=config.init_epoch - 1 if config.init_epoch > 1 else -1,
-        )
-    else:
-        lr_scheduler = None
-
-    return lr_scheduler
-
-
 def main(config, model_weight=None, opt_weight=None):
     def print_config(config):
         pp = pprint.PrettyPrinter(indent=4)
@@ -232,10 +207,10 @@ def print_config(config):
 
     optimizer = get_optimizer(model, config)
 
-    if opt_weight and (config.use_adam or config.use_radam):
+    if opt_weight:
         optimizer.load_state_dict(opt_weight)
 
-    lr_scheduler = get_scheduler(optimizer, config)
+    lr_scheduler = None
 
     if config.verbose >= 2:
         print(model)

src/11_seq2seq/translate.py

@@ -5,7 +5,7 @@
 import torch
 from modules.data_loader import DataLoader
 import modules.data_loader as data_loader
-from modules.models.seq2seq import Seq2Seq
+from modules.seq2seq import Seq2Seq
 
 
 def define_argparser():