sub.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms, models\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import argparse\n",
    "import logging\n",
    "from datetime import datetime\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import random\n",
    "\n",
    "from enums import (\n",
    "    ModelEnum,\n",
    "    OptimizerEnum,\n",
    "    LRSchedulerEnum,\n",
    "    MixEnum\n",
    ")\n",
    "from preprocess.preprocess_data import DataPreProcessor\n",
    "from preprocess.preprocess_model import ModelPreProcessor\n",
    "from train import Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_seed = 4943872"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_arg_parse():\n",
    "    parser = argparse.ArgumentParser()\n",
    "    \n",
    "    # parser.add_argument('-t', '--test', type=int, help='테스트모드', default=0)\n",
    "    # parser.add_argument('-pm', '--model_path', type=str, help='모델 폴더 이름', default='')\n",
    "    # parser.add_argument('--amp', type=int, help='amp 옵션', default=0)\n",
    "    \n",
    "    # parser.add_argument('-rs', '--random_seed', type=int, help='학습 랜덤 시드. -1은 랜덤 시드를 고정하지 않음.', default=4943872)\n",
    "    # parser.add_argument('-lf', '--log_file', type=int, help='로그 파일 출력 여부. 0=false, 1=true', default=1)\n",
    "    # parser.add_argument('-po', '--port', type=int, default=2033)\n",
    "    \n",
    "    # parser.add_argument('-m', '--model', type=ModelEnum, help='학습 모델', choices=list(ModelEnum), default=ModelEnum.custom)\n",
    "    # parser.add_argument('-p', '--parallel', type=int, help='멀티 gpu 사용 여부. 0=false, 1=true', default=0)\n",
    "    # parser.add_argument('-op', '--optimizer', type=OptimizerEnum, help='옵티마이저', choices=list(OptimizerEnum), default=OptimizerEnum.sgd)\n",
    "    # parser.add_argument('-ls', '--lr_scheduler', type=LRSchedulerEnum, help='lr 스케쥴러', choices=list(LRSchedulerEnum), default=LRSchedulerEnum.custom_annealing)\n",
    "    \n",
    "    # parser.add_argument('-ds', '--split_ratio', type=float, help='train/validation 분할 비율', default=0.2)\n",
    "    # parser.add_argument('-am', '--augmentation_mode', type=int, help='data augmentation mode', default=0)\n",
    "    # parser.add_argument('-asp', '--augment_split', type=float, help='augmentation 분할 비율', default=0.5)\n",
    "    # parser.add_argument('-w', '--num_worker', type=int, help='train/validation 분할 비율', default=0)\n",
    "    # parser.add_argument('-b', '--batch_size', type=int, help='학습 배치사이즈', default=128)\n",
    "    # parser.add_argument('-mc', '--mix_step', type=int, help='mix 적용시 몇 step마다 적용할지. 0은 모든 step에 적용.', default=0)\n",
    "    # parser.add_argument('-mt', '--mix_method', type=MixEnum, help='mix 방법', choices=list(MixEnum), default=MixEnum.mixup)\n",
    "    \n",
    "    # parser.add_argument('-pd', '--p_depth', type=int, help='pyramnidnet depth', default=164)\n",
    "    # parser.add_argument('-pa', '--p_alpha', type=int, help='pyramnidnet alpha', default=84)\n",
    "    # parser.add_argument('-ps', '--p_shake', type=int, help='pyramnidnet shake', default=0)\n",
    "    \n",
    "    # parser.add_argument('-e', '--epoch', type=int, help='epoch', default=100)\n",
    "    # parser.add_argument('-mlr', '--max_learning_rate', type=float, help='optimizer/scheduler max learning rate 설정 (custom cos scheduler는 반대)', default=0.1)\n",
    "    # parser.add_argument('-milr', '--min_learning_rate', type=float, help='optimizer/scheduler min learning rate 설정 (custom cos scheduler는 반대)', default=1e-4)\n",
    "    # parser.add_argument('-wd', '--weight_decay', type=float, help='optimizer weight decay 설정', default=5e-4)\n",
    "    # parser.add_argument('-gc', '--gradient_clip', type=float, help='gradient clip 설정. -1은 비활성화', default=0.1)\n",
    "    # parser.add_argument('-lsm', '--label_smoothing', type=float, help='label smoothing 설정', default=0)\n",
    "    # parser.add_argument('-es', '--early_stopping', type=int, help='ealry stoppin epoch 지정. -1은 비활성화', default=-1)\n",
    "    # parser.add_argument('-ad', '--adaptive', type=int, help=\"adaptive SAM 사용 여부\", default=1)\n",
    "    # parser.add_argument('--rho', type=int, help=\"SAM rho 파라미터\", default=2.0)\n",
    "    # parser.add_argument('-cm', '--cos_max', type=int, help=\"cos annealing 주기\", default=50)\n",
    "    # parser.add_argument('-sm', '--step_milestone', nargs='+', type=int, help='step lr scheduler milestone', default=[50])\n",
    "    \n",
    "    args = parser.parse_args('')\n",
    "    # args.p_shake = args.p_shake == 1\n",
    "    \n",
    "    # args = {\n",
    "    #     'test': 0, \n",
    "    #     'model_path': '', \n",
    "    #     'amp': 0, \n",
    "    #     'random_seed': random_seed, \n",
    "    #     'log_file': 0, \n",
    "    #     'port': 2033, \n",
    "    #     'model': 'pyramidnet_custom', \n",
    "    #     'parallel': 0, \n",
    "    #     'optimizer': 'sam', \n",
    "    #     'lr_scheduler': 'step_lr',\n",
    "    #     'split_ratio': 0.2,\n",
    "    #     'augmentation_mode': 0,\n",
    "    #     'augment_split': 0.5,\n",
    "    #     'num_worker': 6,\n",
    "    #     'batch_size': 64,\n",
    "    #     'mix_step': 0,\n",
    "    #     'mix_method': 'cutmix',\n",
    "    #     'p_depth': 163,\n",
    "    #     'p_alpha': 84,\n",
    "    #     'p_shake': False,\n",
    "    #     'epoch': 150,\n",
    "    #     'max_learning_rate': 0.1,\n",
    "    #     'min_learning_rate': 0.001,\n",
    "    #     'weight_decay': 0.0001,\n",
    "    #     'gradient_clip': -1.0,\n",
    "    #     'label_smoothing': 0.1,\n",
    "    #     'early_stopping': -1,\n",
    "    #     'adaptive': 1,\n",
    "    #     'rho': 2,\n",
    "    #     'cos_max': 50,\n",
    "    #     'step_milestone': [60, 120]\n",
    "    # }\n",
    "    args.test = 0\n",
    "    args.model_path = ''\n",
    "    args.amp = 0\n",
    "    args.random_seed = random_seed\n",
    "    args.log_file = 0\n",
    "    args.port = 2033\n",
    "    args.model = ModelEnum.pyramidnet_custom\n",
    "    args.parallel = 0\n",
    "    args.optimizer = OptimizerEnum.sam\n",
    "    args.lr_scheduler = LRSchedulerEnum.step_lr\n",
    "    args.split_ratio = 0.2\n",
    "    args.augmentation_mode = 0\n",
    "    args.augment_split = 0.5\n",
    "    args.num_worker = 6\n",
    "    args.batch_size = 64\n",
    "    args.mix_step = 0\n",
    "    args.mix_method = MixEnum.cutmix\n",
    "    args.p_depth = 163\n",
    "    args.p_alpha = 84\n",
    "    args.p_shake = False\n",
    "    args.epoch = 150\n",
    "    args.max_learning_rate = 0.1\n",
    "    args.min_learning_rate = 0.001\n",
    "    args.weight_decay = 0.0001\n",
    "    args.gradient_clip = -1.0\n",
    "    args.label_smoothing = 0.1\n",
    "    args.early_stopping = -1\n",
    "    args.adaptive = 1\n",
    "    args.rho = 2\n",
    "    args.cos_max = 50\n",
    "    args.step_milestone = [60, 120]\n",
    "    \n",
    "    return args\n",
    "\n",
    "\n",
    "def init_logger(args):\n",
    "    logger = logging.getLogger(\"main\")\n",
    "    logger.setLevel(logging.DEBUG)\n",
    "\n",
    "    formatter = logging.Formatter(\"%(asctime)s %(levelname)s:%(message)s\")\n",
    "    handler = logging.StreamHandler()\n",
    "    handler.setFormatter(formatter)\n",
    "    logger.addHandler(handler)\n",
    "    \n",
    "    if args.log_file == 1:\n",
    "        log_save_path = \"./log\"\n",
    "        if not os.path.exists(log_save_path):\n",
    "            os.makedirs(log_save_path)\n",
    "\n",
    "        datetime_now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "        formatter_file = logging.Formatter(\"%(asctime)s %(levelname)s:%(message)s\")\n",
    "        handler_file = logging.FileHandler(os.path.join(log_save_path, f'{datetime_now}.log'))\n",
    "        handler_file.setLevel(logging.DEBUG)\n",
    "        handler_file.setFormatter(formatter_file)\n",
    "        logger.addHandler(handler_file)\n",
    "\n",
    "    def catch_exception(exc_type, exc_value, exc_traceback):\n",
    "        if issubclass(exc_type, KeyboardInterrupt):\n",
    "            sys.__excepthook__(exc_type, exc_value, exc_traceback)\n",
    "            return\n",
    "\n",
    "        logger = logging.getLogger(\"main\")\n",
    "        \n",
    "        logger.error(\n",
    "            \"Unexpected exception.\",\n",
    "            exc_info=(exc_type, exc_value, exc_traceback)\n",
    "        )\n",
    "\n",
    "    sys.excepthook = catch_exception\n",
    "\n",
    "\n",
    "def fix_random(args):\n",
    "    logger = logging.getLogger(\"main\")\n",
    "    \n",
    "    if args.random_seed == -1:\n",
    "        logger.debug('random seed not fix')\n",
    "        return\n",
    "    \n",
    "    seed = args.random_seed\n",
    "    deterministic = True\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    if 1:\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "    logger.debug('random seed fix')\n",
    "\n",
    "\n",
    "def setup_for_distributed(is_master):\n",
    "    import builtins as __builtin__\n",
    "    builtin_print = __builtin__.print\n",
    "\n",
    "    def print(*args, **kwargs):\n",
    "        force = kwargs.pop('force', False)\n",
    "        if is_master or force:\n",
    "            builtin_print(*args, **kwargs)\n",
    "\n",
    "    __builtin__.print = print\n",
    "\n",
    "\n",
    "def init_distributed_training(rank, opts):\n",
    "    opts.rank = rank\n",
    "    opts.gpu = opts.rank % torch.cuda.device_count()\n",
    "    local_gpu_id = int(opts.gpu_ids[opts.rank])\n",
    "    torch.cuda.set_device(local_gpu_id)\n",
    "    \n",
    "    if opts.rank is not None:\n",
    "        print(\"Use GPU: {} for training\".format(local_gpu_id))\n",
    "\n",
    "    torch.distributed.init_process_group(backend='nccl',\n",
    "                            init_method='tcp://127.0.0.1:' + str(opts.port),\n",
    "                            world_size=opts.ngpus_per_node,\n",
    "                            rank=opts.rank)\n",
    "\n",
    "    torch.distributed.barrier()\n",
    "    setup_for_distributed(opts.rank == 0)\n",
    "    print('opts :',opts)\n",
    "\n",
    "\n",
    "def main(rank, args):\n",
    "    if args.parallel == 1:\n",
    "        init_distributed_training(rank, args)\n",
    "    \n",
    "    \n",
    "    logger = logging.getLogger('main')\n",
    "    logger.debug(f'args: {vars(args)}')\n",
    "    \n",
    "    logger.debug(f'init data preprocessing')\n",
    "    \n",
    "    data_prep = DataPreProcessor(args.parallel, args.augmentation_mode)\n",
    "    \n",
    "    data_prep.transform_data()\n",
    "    data_prep.split_data(args.split_ratio, args.augment_split)\n",
    "    data_prep.get_data_loader(args.batch_size, args.num_worker)\n",
    "    \n",
    "    logger.debug(f'init model')\n",
    "    if args.model is ModelEnum.custom:\n",
    "        # Custom 모델 추가\n",
    "        model_pre = ModelPreProcessor(args, None)\n",
    "    else:\n",
    "        model_pre = ModelPreProcessor(args)\n",
    "    \n",
    "    logger.debug(f'init trainer')\n",
    "    trainer = Trainer(args, model_pre, data_prep)\n",
    "    \n",
    "    if args.test == 1:\n",
    "        logger.debug(f'test start')\n",
    "        trainer.model_save_path = os.path.join('.', 'model_dir', args.model_path)\n",
    "        trainer.get_result()\n",
    "    else:\n",
    "        logger.debug(f'train start')\n",
    "        trainer.train()\n",
    "        trainer.get_result()\n",
    "        trainer.save_history()\n",
    "\n",
    "    logger.debug(f'finish process')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-18 12:44:14,456 DEBUG:random seed fix\n"
     ]
    }
   ],
   "source": [
    "args = get_arg_parse()\n",
    "init_logger(args)\n",
    "fix_random(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-18 12:44:22,964 DEBUG:args: {'test': 0, 'model_path': '', 'amp': 0, 'random_seed': 4943872, 'log_file': 0, 'port': 2033, 'model': <ModelEnum.pyramidnet_custom: 'pyramidnet_custom'>, 'parallel': 0, 'optimizer': <OptimizerEnum.sam: 'sam'>, 'lr_scheduler': <LRSchedulerEnum.step_lr: 'step_lr'>, 'split_ratio': 0.2, 'augmentation_mode': 0, 'augment_split': 0.5, 'num_worker': 6, 'batch_size': 64, 'mix_step': 0, 'mix_method': <MixEnum.cutmix: 'cutmix'>, 'p_depth': 163, 'p_alpha': 84, 'p_shake': False, 'epoch': 150, 'max_learning_rate': 0.1, 'min_learning_rate': 0.001, 'weight_decay': 0.0001, 'gradient_clip': -1.0, 'label_smoothing': 0.1, 'early_stopping': -1, 'adaptive': 1, 'rho': 2, 'cos_max': 50, 'step_milestone': [60, 120]}\n",
      "2024-10-18 12:44:22,964 DEBUG:init data preprocessing\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-18 12:44:26,925 DEBUG:data mean: [0.5070751592371323, 0.48654887331495095, 0.4409178433670343]\tdata std: [0.26733428587941854, 0.25643846292120615, 0.2761504713263903]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-10-18 12:44:28,808 DEBUG:data preprocessor init\n",
      "2024-10-18 12:44:28,808 DEBUG:set transform\tmode: 0\n",
      "2024-10-18 12:44:28,941 DEBUG:data split - train size: 80000\tvalidation size: 20000\n",
      "2024-10-18 12:44:28,942 DEBUG:init model\n",
      "2024-10-18 12:44:28,942 DEBUG:model init: pyramidnet_custom 163 84\n",
      "2024-10-18 12:44:28,985 DEBUG:init trainer\n",
      "/home/k1a22015/develop/jupyter/vision-py/model/pyramidnet.py:105: UserWarning: The torch.cuda.*DtypeTensor constructors are no longer recommended. It's best to use methods such as torch.tensor(data, dtype=*, device='cuda') to create tensors. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:78.)\n",
      "  padding = torch.autograd.Variable(torch.cuda.FloatTensor(batch_size, residual_channel - shortcut_channel, featuremap_size[0], featuremap_size[1]).fill_(0))\n",
      "2024-10-18 12:44:29,913 DEBUG:==========================================================================================\n",
      "Layer (type:depth-idx)                   Output Shape              Param #\n",
      "==========================================================================================\n",
      "PyramidNet                               [1, 100]                  --\n",
      "├─Conv2d: 1-1                            [1, 16, 32, 32]           432\n",
      "├─BatchNorm2d: 1-2                       [1, 16, 32, 32]           32\n",
      "├─Sequential: 1-3                        [1, 176, 32, 32]          --\n",
      "│    └─Bottleneck: 2-1                   [1, 72, 32, 32]           --\n",
      "│    │    └─BatchNorm2d: 3-1             [1, 16, 32, 32]           32\n",
      "│    │    └─Conv2d: 3-2                  [1, 18, 32, 32]           288\n",
      "│    │    └─BatchNorm2d: 3-3             [1, 18, 32, 32]           36\n",
      "│    │    └─ReLU: 3-4                    [1, 18, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-5                  [1, 18, 32, 32]           2,916\n",
      "│    │    └─BatchNorm2d: 3-6             [1, 18, 32, 32]           36\n",
      "│    │    └─ReLU: 3-7                    [1, 18, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-8                  [1, 72, 32, 32]           1,296\n",
      "│    │    └─BatchNorm2d: 3-9             [1, 72, 32, 32]           144\n",
      "│    └─Bottleneck: 2-2                   [1, 76, 32, 32]           --\n",
      "│    │    └─BatchNorm2d: 3-10            [1, 72, 32, 32]           144\n",
      "│    │    └─Conv2d: 3-11                 [1, 19, 32, 32]           1,368\n",
      "│    │    └─BatchNorm2d: 3-12            [1, 19, 32, 32]           38\n",
      "│    │    └─ReLU: 3-13                   [1, 19, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-14                 [1, 19, 32, 32]           3,249\n",
      "│    │    └─BatchNorm2d: 3-15            [1, 19, 32, 32]           38\n",
      "│    │    └─ReLU: 3-16                   [1, 19, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-17                 [1, 76, 32, 32]           1,444\n",
      "│    │    └─BatchNorm2d: 3-18            [1, 76, 32, 32]           152\n",
      "│    └─Bottleneck: 2-3                   [1, 84, 32, 32]           --\n",
      "│    │    └─BatchNorm2d: 3-19            [1, 76, 32, 32]           152\n",
      "│    │    └─Conv2d: 3-20                 [1, 21, 32, 32]           1,596\n",
      "│    │    └─BatchNorm2d: 3-21            [1, 21, 32, 32]           42\n",
      "│    │    └─ReLU: 3-22                   [1, 21, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-23                 [1, 21, 32, 32]           3,969\n",
      "│    │    └─BatchNorm2d: 3-24            [1, 21, 32, 32]           42\n",
      "│    │    └─ReLU: 3-25                   [1, 21, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-26                 [1, 84, 32, 32]           1,764\n",
      "│    │    └─BatchNorm2d: 3-27            [1, 84, 32, 32]           168\n",
      "│    └─Bottleneck: 2-4                   [1, 92, 32, 32]           --\n",
      "│    │    └─BatchNorm2d: 3-28            [1, 84, 32, 32]           168\n",
      "│    │    └─Conv2d: 3-29                 [1, 23, 32, 32]           1,932\n",
      "│    │    └─BatchNorm2d: 3-30            [1, 23, 32, 32]           46\n",
      "│    │    └─ReLU: 3-31                   [1, 23, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-32                 [1, 23, 32, 32]           4,761\n",
      "│    │    └─BatchNorm2d: 3-33            [1, 23, 32, 32]           46\n",
      "│    │    └─ReLU: 3-34                   [1, 23, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-35                 [1, 92, 32, 32]           2,116\n",
      "│    │    └─BatchNorm2d: 3-36            [1, 92, 32, 32]           184\n",
      "│    └─Bottleneck: 2-5                   [1, 96, 32, 32]           --\n",
      "│    │    └─BatchNorm2d: 3-37            [1, 92, 32, 32]           184\n",
      "│    │    └─Conv2d: 3-38                 [1, 24, 32, 32]           2,208\n",
      "│    │    └─BatchNorm2d: 3-39            [1, 24, 32, 32]           48\n",
      "│    │    └─ReLU: 3-40                   [1, 24, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-41                 [1, 24, 32, 32]           5,184\n",
      "│    │    └─BatchNorm2d: 3-42            [1, 24, 32, 32]           48\n",
      "│    │    └─ReLU: 3-43                   [1, 24, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-44                 [1, 96, 32, 32]           2,304\n",
      "│    │    └─BatchNorm2d: 3-45            [1, 96, 32, 32]           192\n",
      "│    └─Bottleneck: 2-6                   [1, 104, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-46            [1, 96, 32, 32]           192\n",
      "│    │    └─Conv2d: 3-47                 [1, 26, 32, 32]           2,496\n",
      "│    │    └─BatchNorm2d: 3-48            [1, 26, 32, 32]           52\n",
      "│    │    └─ReLU: 3-49                   [1, 26, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-50                 [1, 26, 32, 32]           6,084\n",
      "│    │    └─BatchNorm2d: 3-51            [1, 26, 32, 32]           52\n",
      "│    │    └─ReLU: 3-52                   [1, 26, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-53                 [1, 104, 32, 32]          2,704\n",
      "│    │    └─BatchNorm2d: 3-54            [1, 104, 32, 32]          208\n",
      "│    └─Bottleneck: 2-7                   [1, 112, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-55            [1, 104, 32, 32]          208\n",
      "│    │    └─Conv2d: 3-56                 [1, 28, 32, 32]           2,912\n",
      "│    │    └─BatchNorm2d: 3-57            [1, 28, 32, 32]           56\n",
      "│    │    └─ReLU: 3-58                   [1, 28, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-59                 [1, 28, 32, 32]           7,056\n",
      "│    │    └─BatchNorm2d: 3-60            [1, 28, 32, 32]           56\n",
      "│    │    └─ReLU: 3-61                   [1, 28, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-62                 [1, 112, 32, 32]          3,136\n",
      "│    │    └─BatchNorm2d: 3-63            [1, 112, 32, 32]          224\n",
      "│    └─Bottleneck: 2-8                   [1, 116, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-64            [1, 112, 32, 32]          224\n",
      "│    │    └─Conv2d: 3-65                 [1, 29, 32, 32]           3,248\n",
      "│    │    └─BatchNorm2d: 3-66            [1, 29, 32, 32]           58\n",
      "│    │    └─ReLU: 3-67                   [1, 29, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-68                 [1, 29, 32, 32]           7,569\n",
      "│    │    └─BatchNorm2d: 3-69            [1, 29, 32, 32]           58\n",
      "│    │    └─ReLU: 3-70                   [1, 29, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-71                 [1, 116, 32, 32]          3,364\n",
      "│    │    └─BatchNorm2d: 3-72            [1, 116, 32, 32]          232\n",
      "│    └─Bottleneck: 2-9                   [1, 124, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-73            [1, 116, 32, 32]          232\n",
      "│    │    └─Conv2d: 3-74                 [1, 31, 32, 32]           3,596\n",
      "│    │    └─BatchNorm2d: 3-75            [1, 31, 32, 32]           62\n",
      "│    │    └─ReLU: 3-76                   [1, 31, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-77                 [1, 31, 32, 32]           8,649\n",
      "│    │    └─BatchNorm2d: 3-78            [1, 31, 32, 32]           62\n",
      "│    │    └─ReLU: 3-79                   [1, 31, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-80                 [1, 124, 32, 32]          3,844\n",
      "│    │    └─BatchNorm2d: 3-81            [1, 124, 32, 32]          248\n",
      "│    └─Bottleneck: 2-10                  [1, 128, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-82            [1, 124, 32, 32]          248\n",
      "│    │    └─Conv2d: 3-83                 [1, 32, 32, 32]           3,968\n",
      "│    │    └─BatchNorm2d: 3-84            [1, 32, 32, 32]           64\n",
      "│    │    └─ReLU: 3-85                   [1, 32, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-86                 [1, 32, 32, 32]           9,216\n",
      "│    │    └─BatchNorm2d: 3-87            [1, 32, 32, 32]           64\n",
      "│    │    └─ReLU: 3-88                   [1, 32, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-89                 [1, 128, 32, 32]          4,096\n",
      "│    │    └─BatchNorm2d: 3-90            [1, 128, 32, 32]          256\n",
      "│    └─Bottleneck: 2-11                  [1, 136, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-91            [1, 128, 32, 32]          256\n",
      "│    │    └─Conv2d: 3-92                 [1, 34, 32, 32]           4,352\n",
      "│    │    └─BatchNorm2d: 3-93            [1, 34, 32, 32]           68\n",
      "│    │    └─ReLU: 3-94                   [1, 34, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-95                 [1, 34, 32, 32]           10,404\n",
      "│    │    └─BatchNorm2d: 3-96            [1, 34, 32, 32]           68\n",
      "│    │    └─ReLU: 3-97                   [1, 34, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-98                 [1, 136, 32, 32]          4,624\n",
      "│    │    └─BatchNorm2d: 3-99            [1, 136, 32, 32]          272\n",
      "│    └─Bottleneck: 2-12                  [1, 144, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-100           [1, 136, 32, 32]          272\n",
      "│    │    └─Conv2d: 3-101                [1, 36, 32, 32]           4,896\n",
      "│    │    └─BatchNorm2d: 3-102           [1, 36, 32, 32]           72\n",
      "│    │    └─ReLU: 3-103                  [1, 36, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-104                [1, 36, 32, 32]           11,664\n",
      "│    │    └─BatchNorm2d: 3-105           [1, 36, 32, 32]           72\n",
      "│    │    └─ReLU: 3-106                  [1, 36, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-107                [1, 144, 32, 32]          5,184\n",
      "│    │    └─BatchNorm2d: 3-108           [1, 144, 32, 32]          288\n",
      "│    └─Bottleneck: 2-13                  [1, 148, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-109           [1, 144, 32, 32]          288\n",
      "│    │    └─Conv2d: 3-110                [1, 37, 32, 32]           5,328\n",
      "│    │    └─BatchNorm2d: 3-111           [1, 37, 32, 32]           74\n",
      "│    │    └─ReLU: 3-112                  [1, 37, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-113                [1, 37, 32, 32]           12,321\n",
      "│    │    └─BatchNorm2d: 3-114           [1, 37, 32, 32]           74\n",
      "│    │    └─ReLU: 3-115                  [1, 37, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-116                [1, 148, 32, 32]          5,476\n",
      "│    │    └─BatchNorm2d: 3-117           [1, 148, 32, 32]          296\n",
      "│    └─Bottleneck: 2-14                  [1, 156, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-118           [1, 148, 32, 32]          296\n",
      "│    │    └─Conv2d: 3-119                [1, 39, 32, 32]           5,772\n",
      "│    │    └─BatchNorm2d: 3-120           [1, 39, 32, 32]           78\n",
      "│    │    └─ReLU: 3-121                  [1, 39, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-122                [1, 39, 32, 32]           13,689\n",
      "│    │    └─BatchNorm2d: 3-123           [1, 39, 32, 32]           78\n",
      "│    │    └─ReLU: 3-124                  [1, 39, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-125                [1, 156, 32, 32]          6,084\n",
      "│    │    └─BatchNorm2d: 3-126           [1, 156, 32, 32]          312\n",
      "│    └─Bottleneck: 2-15                  [1, 164, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-127           [1, 156, 32, 32]          312\n",
      "│    │    └─Conv2d: 3-128                [1, 41, 32, 32]           6,396\n",
      "│    │    └─BatchNorm2d: 3-129           [1, 41, 32, 32]           82\n",
      "│    │    └─ReLU: 3-130                  [1, 41, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-131                [1, 41, 32, 32]           15,129\n",
      "│    │    └─BatchNorm2d: 3-132           [1, 41, 32, 32]           82\n",
      "│    │    └─ReLU: 3-133                  [1, 41, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-134                [1, 164, 32, 32]          6,724\n",
      "│    │    └─BatchNorm2d: 3-135           [1, 164, 32, 32]          328\n",
      "│    └─Bottleneck: 2-16                  [1, 168, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-136           [1, 164, 32, 32]          328\n",
      "│    │    └─Conv2d: 3-137                [1, 42, 32, 32]           6,888\n",
      "│    │    └─BatchNorm2d: 3-138           [1, 42, 32, 32]           84\n",
      "│    │    └─ReLU: 3-139                  [1, 42, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-140                [1, 42, 32, 32]           15,876\n",
      "│    │    └─BatchNorm2d: 3-141           [1, 42, 32, 32]           84\n",
      "│    │    └─ReLU: 3-142                  [1, 42, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-143                [1, 168, 32, 32]          7,056\n",
      "│    │    └─BatchNorm2d: 3-144           [1, 168, 32, 32]          336\n",
      "│    └─Bottleneck: 2-17                  [1, 176, 32, 32]          --\n",
      "│    │    └─BatchNorm2d: 3-145           [1, 168, 32, 32]          336\n",
      "│    │    └─Conv2d: 3-146                [1, 44, 32, 32]           7,392\n",
      "│    │    └─BatchNorm2d: 3-147           [1, 44, 32, 32]           88\n",
      "│    │    └─ReLU: 3-148                  [1, 44, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-149                [1, 44, 32, 32]           17,424\n",
      "│    │    └─BatchNorm2d: 3-150           [1, 44, 32, 32]           88\n",
      "│    │    └─ReLU: 3-151                  [1, 44, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-152                [1, 176, 32, 32]          7,744\n",
      "│    │    └─BatchNorm2d: 3-153           [1, 176, 32, 32]          352\n",
      "├─Sequential: 1-4                        [1, 288, 16, 16]          --\n",
      "│    └─Bottleneck: 2-18                  [1, 184, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-154           [1, 176, 32, 32]          352\n",
      "│    │    └─Conv2d: 3-155                [1, 46, 32, 32]           8,096\n",
      "│    │    └─BatchNorm2d: 3-156           [1, 46, 32, 32]           92\n",
      "│    │    └─ReLU: 3-157                  [1, 46, 32, 32]           --\n",
      "│    │    └─Conv2d: 3-158                [1, 46, 16, 16]           19,044\n",
      "│    │    └─BatchNorm2d: 3-159           [1, 46, 16, 16]           92\n",
      "│    │    └─ReLU: 3-160                  [1, 46, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-161                [1, 184, 16, 16]          8,464\n",
      "│    │    └─BatchNorm2d: 3-162           [1, 184, 16, 16]          368\n",
      "│    │    └─AvgPool2d: 3-163             [1, 176, 16, 16]          --\n",
      "│    └─Bottleneck: 2-19                  [1, 188, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-164           [1, 184, 16, 16]          368\n",
      "│    │    └─Conv2d: 3-165                [1, 47, 16, 16]           8,648\n",
      "│    │    └─BatchNorm2d: 3-166           [1, 47, 16, 16]           94\n",
      "│    │    └─ReLU: 3-167                  [1, 47, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-168                [1, 47, 16, 16]           19,881\n",
      "│    │    └─BatchNorm2d: 3-169           [1, 47, 16, 16]           94\n",
      "│    │    └─ReLU: 3-170                  [1, 47, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-171                [1, 188, 16, 16]          8,836\n",
      "│    │    └─BatchNorm2d: 3-172           [1, 188, 16, 16]          376\n",
      "│    └─Bottleneck: 2-20                  [1, 196, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-173           [1, 188, 16, 16]          376\n",
      "│    │    └─Conv2d: 3-174                [1, 49, 16, 16]           9,212\n",
      "│    │    └─BatchNorm2d: 3-175           [1, 49, 16, 16]           98\n",
      "│    │    └─ReLU: 3-176                  [1, 49, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-177                [1, 49, 16, 16]           21,609\n",
      "│    │    └─BatchNorm2d: 3-178           [1, 49, 16, 16]           98\n",
      "│    │    └─ReLU: 3-179                  [1, 49, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-180                [1, 196, 16, 16]          9,604\n",
      "│    │    └─BatchNorm2d: 3-181           [1, 196, 16, 16]          392\n",
      "│    └─Bottleneck: 2-21                  [1, 204, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-182           [1, 196, 16, 16]          392\n",
      "│    │    └─Conv2d: 3-183                [1, 51, 16, 16]           9,996\n",
      "│    │    └─BatchNorm2d: 3-184           [1, 51, 16, 16]           102\n",
      "│    │    └─ReLU: 3-185                  [1, 51, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-186                [1, 51, 16, 16]           23,409\n",
      "│    │    └─BatchNorm2d: 3-187           [1, 51, 16, 16]           102\n",
      "│    │    └─ReLU: 3-188                  [1, 51, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-189                [1, 204, 16, 16]          10,404\n",
      "│    │    └─BatchNorm2d: 3-190           [1, 204, 16, 16]          408\n",
      "│    └─Bottleneck: 2-22                  [1, 208, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-191           [1, 204, 16, 16]          408\n",
      "│    │    └─Conv2d: 3-192                [1, 52, 16, 16]           10,608\n",
      "│    │    └─BatchNorm2d: 3-193           [1, 52, 16, 16]           104\n",
      "│    │    └─ReLU: 3-194                  [1, 52, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-195                [1, 52, 16, 16]           24,336\n",
      "│    │    └─BatchNorm2d: 3-196           [1, 52, 16, 16]           104\n",
      "│    │    └─ReLU: 3-197                  [1, 52, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-198                [1, 208, 16, 16]          10,816\n",
      "│    │    └─BatchNorm2d: 3-199           [1, 208, 16, 16]          416\n",
      "│    └─Bottleneck: 2-23                  [1, 216, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-200           [1, 208, 16, 16]          416\n",
      "│    │    └─Conv2d: 3-201                [1, 54, 16, 16]           11,232\n",
      "│    │    └─BatchNorm2d: 3-202           [1, 54, 16, 16]           108\n",
      "│    │    └─ReLU: 3-203                  [1, 54, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-204                [1, 54, 16, 16]           26,244\n",
      "│    │    └─BatchNorm2d: 3-205           [1, 54, 16, 16]           108\n",
      "│    │    └─ReLU: 3-206                  [1, 54, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-207                [1, 216, 16, 16]          11,664\n",
      "│    │    └─BatchNorm2d: 3-208           [1, 216, 16, 16]          432\n",
      "│    └─Bottleneck: 2-24                  [1, 224, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-209           [1, 216, 16, 16]          432\n",
      "│    │    └─Conv2d: 3-210                [1, 56, 16, 16]           12,096\n",
      "│    │    └─BatchNorm2d: 3-211           [1, 56, 16, 16]           112\n",
      "│    │    └─ReLU: 3-212                  [1, 56, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-213                [1, 56, 16, 16]           28,224\n",
      "│    │    └─BatchNorm2d: 3-214           [1, 56, 16, 16]           112\n",
      "│    │    └─ReLU: 3-215                  [1, 56, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-216                [1, 224, 16, 16]          12,544\n",
      "│    │    └─BatchNorm2d: 3-217           [1, 224, 16, 16]          448\n",
      "│    └─Bottleneck: 2-25                  [1, 228, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-218           [1, 224, 16, 16]          448\n",
      "│    │    └─Conv2d: 3-219                [1, 57, 16, 16]           12,768\n",
      "│    │    └─BatchNorm2d: 3-220           [1, 57, 16, 16]           114\n",
      "│    │    └─ReLU: 3-221                  [1, 57, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-222                [1, 57, 16, 16]           29,241\n",
      "│    │    └─BatchNorm2d: 3-223           [1, 57, 16, 16]           114\n",
      "│    │    └─ReLU: 3-224                  [1, 57, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-225                [1, 228, 16, 16]          12,996\n",
      "│    │    └─BatchNorm2d: 3-226           [1, 228, 16, 16]          456\n",
      "│    └─Bottleneck: 2-26                  [1, 236, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-227           [1, 228, 16, 16]          456\n",
      "│    │    └─Conv2d: 3-228                [1, 59, 16, 16]           13,452\n",
      "│    │    └─BatchNorm2d: 3-229           [1, 59, 16, 16]           118\n",
      "│    │    └─ReLU: 3-230                  [1, 59, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-231                [1, 59, 16, 16]           31,329\n",
      "│    │    └─BatchNorm2d: 3-232           [1, 59, 16, 16]           118\n",
      "│    │    └─ReLU: 3-233                  [1, 59, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-234                [1, 236, 16, 16]          13,924\n",
      "│    │    └─BatchNorm2d: 3-235           [1, 236, 16, 16]          472\n",
      "│    └─Bottleneck: 2-27                  [1, 240, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-236           [1, 236, 16, 16]          472\n",
      "│    │    └─Conv2d: 3-237                [1, 60, 16, 16]           14,160\n",
      "│    │    └─BatchNorm2d: 3-238           [1, 60, 16, 16]           120\n",
      "│    │    └─ReLU: 3-239                  [1, 60, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-240                [1, 60, 16, 16]           32,400\n",
      "│    │    └─BatchNorm2d: 3-241           [1, 60, 16, 16]           120\n",
      "│    │    └─ReLU: 3-242                  [1, 60, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-243                [1, 240, 16, 16]          14,400\n",
      "│    │    └─BatchNorm2d: 3-244           [1, 240, 16, 16]          480\n",
      "│    └─Bottleneck: 2-28                  [1, 248, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-245           [1, 240, 16, 16]          480\n",
      "│    │    └─Conv2d: 3-246                [1, 62, 16, 16]           14,880\n",
      "│    │    └─BatchNorm2d: 3-247           [1, 62, 16, 16]           124\n",
      "│    │    └─ReLU: 3-248                  [1, 62, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-249                [1, 62, 16, 16]           34,596\n",
      "│    │    └─BatchNorm2d: 3-250           [1, 62, 16, 16]           124\n",
      "│    │    └─ReLU: 3-251                  [1, 62, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-252                [1, 248, 16, 16]          15,376\n",
      "│    │    └─BatchNorm2d: 3-253           [1, 248, 16, 16]          496\n",
      "│    └─Bottleneck: 2-29                  [1, 256, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-254           [1, 248, 16, 16]          496\n",
      "│    │    └─Conv2d: 3-255                [1, 64, 16, 16]           15,872\n",
      "│    │    └─BatchNorm2d: 3-256           [1, 64, 16, 16]           128\n",
      "│    │    └─ReLU: 3-257                  [1, 64, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-258                [1, 64, 16, 16]           36,864\n",
      "│    │    └─BatchNorm2d: 3-259           [1, 64, 16, 16]           128\n",
      "│    │    └─ReLU: 3-260                  [1, 64, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-261                [1, 256, 16, 16]          16,384\n",
      "│    │    └─BatchNorm2d: 3-262           [1, 256, 16, 16]          512\n",
      "│    └─Bottleneck: 2-30                  [1, 260, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-263           [1, 256, 16, 16]          512\n",
      "│    │    └─Conv2d: 3-264                [1, 65, 16, 16]           16,640\n",
      "│    │    └─BatchNorm2d: 3-265           [1, 65, 16, 16]           130\n",
      "│    │    └─ReLU: 3-266                  [1, 65, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-267                [1, 65, 16, 16]           38,025\n",
      "│    │    └─BatchNorm2d: 3-268           [1, 65, 16, 16]           130\n",
      "│    │    └─ReLU: 3-269                  [1, 65, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-270                [1, 260, 16, 16]          16,900\n",
      "│    │    └─BatchNorm2d: 3-271           [1, 260, 16, 16]          520\n",
      "│    └─Bottleneck: 2-31                  [1, 268, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-272           [1, 260, 16, 16]          520\n",
      "│    │    └─Conv2d: 3-273                [1, 67, 16, 16]           17,420\n",
      "│    │    └─BatchNorm2d: 3-274           [1, 67, 16, 16]           134\n",
      "│    │    └─ReLU: 3-275                  [1, 67, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-276                [1, 67, 16, 16]           40,401\n",
      "│    │    └─BatchNorm2d: 3-277           [1, 67, 16, 16]           134\n",
      "│    │    └─ReLU: 3-278                  [1, 67, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-279                [1, 268, 16, 16]          17,956\n",
      "│    │    └─BatchNorm2d: 3-280           [1, 268, 16, 16]          536\n",
      "│    └─Bottleneck: 2-32                  [1, 276, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-281           [1, 268, 16, 16]          536\n",
      "│    │    └─Conv2d: 3-282                [1, 69, 16, 16]           18,492\n",
      "│    │    └─BatchNorm2d: 3-283           [1, 69, 16, 16]           138\n",
      "│    │    └─ReLU: 3-284                  [1, 69, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-285                [1, 69, 16, 16]           42,849\n",
      "│    │    └─BatchNorm2d: 3-286           [1, 69, 16, 16]           138\n",
      "│    │    └─ReLU: 3-287                  [1, 69, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-288                [1, 276, 16, 16]          19,044\n",
      "│    │    └─BatchNorm2d: 3-289           [1, 276, 16, 16]          552\n",
      "│    └─Bottleneck: 2-33                  [1, 280, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-290           [1, 276, 16, 16]          552\n",
      "│    │    └─Conv2d: 3-291                [1, 70, 16, 16]           19,320\n",
      "│    │    └─BatchNorm2d: 3-292           [1, 70, 16, 16]           140\n",
      "│    │    └─ReLU: 3-293                  [1, 70, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-294                [1, 70, 16, 16]           44,100\n",
      "│    │    └─BatchNorm2d: 3-295           [1, 70, 16, 16]           140\n",
      "│    │    └─ReLU: 3-296                  [1, 70, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-297                [1, 280, 16, 16]          19,600\n",
      "│    │    └─BatchNorm2d: 3-298           [1, 280, 16, 16]          560\n",
      "│    └─Bottleneck: 2-34                  [1, 288, 16, 16]          --\n",
      "│    │    └─BatchNorm2d: 3-299           [1, 280, 16, 16]          560\n",
      "│    │    └─Conv2d: 3-300                [1, 72, 16, 16]           20,160\n",
      "│    │    └─BatchNorm2d: 3-301           [1, 72, 16, 16]           144\n",
      "│    │    └─ReLU: 3-302                  [1, 72, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-303                [1, 72, 16, 16]           46,656\n",
      "│    │    └─BatchNorm2d: 3-304           [1, 72, 16, 16]           144\n",
      "│    │    └─ReLU: 3-305                  [1, 72, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-306                [1, 288, 16, 16]          20,736\n",
      "│    │    └─BatchNorm2d: 3-307           [1, 288, 16, 16]          576\n",
      "├─Sequential: 1-5                        [1, 400, 8, 8]            --\n",
      "│    └─Bottleneck: 2-35                  [1, 296, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-308           [1, 288, 16, 16]          576\n",
      "│    │    └─Conv2d: 3-309                [1, 74, 16, 16]           21,312\n",
      "│    │    └─BatchNorm2d: 3-310           [1, 74, 16, 16]           148\n",
      "│    │    └─ReLU: 3-311                  [1, 74, 16, 16]           --\n",
      "│    │    └─Conv2d: 3-312                [1, 74, 8, 8]             49,284\n",
      "│    │    └─BatchNorm2d: 3-313           [1, 74, 8, 8]             148\n",
      "│    │    └─ReLU: 3-314                  [1, 74, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-315                [1, 296, 8, 8]            21,904\n",
      "│    │    └─BatchNorm2d: 3-316           [1, 296, 8, 8]            592\n",
      "│    │    └─AvgPool2d: 3-317             [1, 288, 8, 8]            --\n",
      "│    └─Bottleneck: 2-36                  [1, 300, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-318           [1, 296, 8, 8]            592\n",
      "│    │    └─Conv2d: 3-319                [1, 75, 8, 8]             22,200\n",
      "│    │    └─BatchNorm2d: 3-320           [1, 75, 8, 8]             150\n",
      "│    │    └─ReLU: 3-321                  [1, 75, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-322                [1, 75, 8, 8]             50,625\n",
      "│    │    └─BatchNorm2d: 3-323           [1, 75, 8, 8]             150\n",
      "│    │    └─ReLU: 3-324                  [1, 75, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-325                [1, 300, 8, 8]            22,500\n",
      "│    │    └─BatchNorm2d: 3-326           [1, 300, 8, 8]            600\n",
      "│    └─Bottleneck: 2-37                  [1, 308, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-327           [1, 300, 8, 8]            600\n",
      "│    │    └─Conv2d: 3-328                [1, 77, 8, 8]             23,100\n",
      "│    │    └─BatchNorm2d: 3-329           [1, 77, 8, 8]             154\n",
      "│    │    └─ReLU: 3-330                  [1, 77, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-331                [1, 77, 8, 8]             53,361\n",
      "│    │    └─BatchNorm2d: 3-332           [1, 77, 8, 8]             154\n",
      "│    │    └─ReLU: 3-333                  [1, 77, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-334                [1, 308, 8, 8]            23,716\n",
      "│    │    └─BatchNorm2d: 3-335           [1, 308, 8, 8]            616\n",
      "│    └─Bottleneck: 2-38                  [1, 316, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-336           [1, 308, 8, 8]            616\n",
      "│    │    └─Conv2d: 3-337                [1, 79, 8, 8]             24,332\n",
      "│    │    └─BatchNorm2d: 3-338           [1, 79, 8, 8]             158\n",
      "│    │    └─ReLU: 3-339                  [1, 79, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-340                [1, 79, 8, 8]             56,169\n",
      "│    │    └─BatchNorm2d: 3-341           [1, 79, 8, 8]             158\n",
      "│    │    └─ReLU: 3-342                  [1, 79, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-343                [1, 316, 8, 8]            24,964\n",
      "│    │    └─BatchNorm2d: 3-344           [1, 316, 8, 8]            632\n",
      "│    └─Bottleneck: 2-39                  [1, 320, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-345           [1, 316, 8, 8]            632\n",
      "│    │    └─Conv2d: 3-346                [1, 80, 8, 8]             25,280\n",
      "│    │    └─BatchNorm2d: 3-347           [1, 80, 8, 8]             160\n",
      "│    │    └─ReLU: 3-348                  [1, 80, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-349                [1, 80, 8, 8]             57,600\n",
      "│    │    └─BatchNorm2d: 3-350           [1, 80, 8, 8]             160\n",
      "│    │    └─ReLU: 3-351                  [1, 80, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-352                [1, 320, 8, 8]            25,600\n",
      "│    │    └─BatchNorm2d: 3-353           [1, 320, 8, 8]            640\n",
      "│    └─Bottleneck: 2-40                  [1, 328, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-354           [1, 320, 8, 8]            640\n",
      "│    │    └─Conv2d: 3-355                [1, 82, 8, 8]             26,240\n",
      "│    │    └─BatchNorm2d: 3-356           [1, 82, 8, 8]             164\n",
      "│    │    └─ReLU: 3-357                  [1, 82, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-358                [1, 82, 8, 8]             60,516\n",
      "│    │    └─BatchNorm2d: 3-359           [1, 82, 8, 8]             164\n",
      "│    │    └─ReLU: 3-360                  [1, 82, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-361                [1, 328, 8, 8]            26,896\n",
      "│    │    └─BatchNorm2d: 3-362           [1, 328, 8, 8]            656\n",
      "│    └─Bottleneck: 2-41                  [1, 336, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-363           [1, 328, 8, 8]            656\n",
      "│    │    └─Conv2d: 3-364                [1, 84, 8, 8]             27,552\n",
      "│    │    └─BatchNorm2d: 3-365           [1, 84, 8, 8]             168\n",
      "│    │    └─ReLU: 3-366                  [1, 84, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-367                [1, 84, 8, 8]             63,504\n",
      "│    │    └─BatchNorm2d: 3-368           [1, 84, 8, 8]             168\n",
      "│    │    └─ReLU: 3-369                  [1, 84, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-370                [1, 336, 8, 8]            28,224\n",
      "│    │    └─BatchNorm2d: 3-371           [1, 336, 8, 8]            672\n",
      "│    └─Bottleneck: 2-42                  [1, 340, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-372           [1, 336, 8, 8]            672\n",
      "│    │    └─Conv2d: 3-373                [1, 85, 8, 8]             28,560\n",
      "│    │    └─BatchNorm2d: 3-374           [1, 85, 8, 8]             170\n",
      "│    │    └─ReLU: 3-375                  [1, 85, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-376                [1, 85, 8, 8]             65,025\n",
      "│    │    └─BatchNorm2d: 3-377           [1, 85, 8, 8]             170\n",
      "│    │    └─ReLU: 3-378                  [1, 85, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-379                [1, 340, 8, 8]            28,900\n",
      "│    │    └─BatchNorm2d: 3-380           [1, 340, 8, 8]            680\n",
      "│    └─Bottleneck: 2-43                  [1, 348, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-381           [1, 340, 8, 8]            680\n",
      "│    │    └─Conv2d: 3-382                [1, 87, 8, 8]             29,580\n",
      "│    │    └─BatchNorm2d: 3-383           [1, 87, 8, 8]             174\n",
      "│    │    └─ReLU: 3-384                  [1, 87, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-385                [1, 87, 8, 8]             68,121\n",
      "│    │    └─BatchNorm2d: 3-386           [1, 87, 8, 8]             174\n",
      "│    │    └─ReLU: 3-387                  [1, 87, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-388                [1, 348, 8, 8]            30,276\n",
      "│    │    └─BatchNorm2d: 3-389           [1, 348, 8, 8]            696\n",
      "│    └─Bottleneck: 2-44                  [1, 352, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-390           [1, 348, 8, 8]            696\n",
      "│    │    └─Conv2d: 3-391                [1, 88, 8, 8]             30,624\n",
      "│    │    └─BatchNorm2d: 3-392           [1, 88, 8, 8]             176\n",
      "│    │    └─ReLU: 3-393                  [1, 88, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-394                [1, 88, 8, 8]             69,696\n",
      "│    │    └─BatchNorm2d: 3-395           [1, 88, 8, 8]             176\n",
      "│    │    └─ReLU: 3-396                  [1, 88, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-397                [1, 352, 8, 8]            30,976\n",
      "│    │    └─BatchNorm2d: 3-398           [1, 352, 8, 8]            704\n",
      "│    └─Bottleneck: 2-45                  [1, 360, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-399           [1, 352, 8, 8]            704\n",
      "│    │    └─Conv2d: 3-400                [1, 90, 8, 8]             31,680\n",
      "│    │    └─BatchNorm2d: 3-401           [1, 90, 8, 8]             180\n",
      "│    │    └─ReLU: 3-402                  [1, 90, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-403                [1, 90, 8, 8]             72,900\n",
      "│    │    └─BatchNorm2d: 3-404           [1, 90, 8, 8]             180\n",
      "│    │    └─ReLU: 3-405                  [1, 90, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-406                [1, 360, 8, 8]            32,400\n",
      "│    │    └─BatchNorm2d: 3-407           [1, 360, 8, 8]            720\n",
      "│    └─Bottleneck: 2-46                  [1, 368, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-408           [1, 360, 8, 8]            720\n",
      "│    │    └─Conv2d: 3-409                [1, 92, 8, 8]             33,120\n",
      "│    │    └─BatchNorm2d: 3-410           [1, 92, 8, 8]             184\n",
      "│    │    └─ReLU: 3-411                  [1, 92, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-412                [1, 92, 8, 8]             76,176\n",
      "│    │    └─BatchNorm2d: 3-413           [1, 92, 8, 8]             184\n",
      "│    │    └─ReLU: 3-414                  [1, 92, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-415                [1, 368, 8, 8]            33,856\n",
      "│    │    └─BatchNorm2d: 3-416           [1, 368, 8, 8]            736\n",
      "│    └─Bottleneck: 2-47                  [1, 372, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-417           [1, 368, 8, 8]            736\n",
      "│    │    └─Conv2d: 3-418                [1, 93, 8, 8]             34,224\n",
      "│    │    └─BatchNorm2d: 3-419           [1, 93, 8, 8]             186\n",
      "│    │    └─ReLU: 3-420                  [1, 93, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-421                [1, 93, 8, 8]             77,841\n",
      "│    │    └─BatchNorm2d: 3-422           [1, 93, 8, 8]             186\n",
      "│    │    └─ReLU: 3-423                  [1, 93, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-424                [1, 372, 8, 8]            34,596\n",
      "│    │    └─BatchNorm2d: 3-425           [1, 372, 8, 8]            744\n",
      "│    └─Bottleneck: 2-48                  [1, 380, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-426           [1, 372, 8, 8]            744\n",
      "│    │    └─Conv2d: 3-427                [1, 95, 8, 8]             35,340\n",
      "│    │    └─BatchNorm2d: 3-428           [1, 95, 8, 8]             190\n",
      "│    │    └─ReLU: 3-429                  [1, 95, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-430                [1, 95, 8, 8]             81,225\n",
      "│    │    └─BatchNorm2d: 3-431           [1, 95, 8, 8]             190\n",
      "│    │    └─ReLU: 3-432                  [1, 95, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-433                [1, 380, 8, 8]            36,100\n",
      "│    │    └─BatchNorm2d: 3-434           [1, 380, 8, 8]            760\n",
      "│    └─Bottleneck: 2-49                  [1, 388, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-435           [1, 380, 8, 8]            760\n",
      "│    │    └─Conv2d: 3-436                [1, 97, 8, 8]             36,860\n",
      "│    │    └─BatchNorm2d: 3-437           [1, 97, 8, 8]             194\n",
      "│    │    └─ReLU: 3-438                  [1, 97, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-439                [1, 97, 8, 8]             84,681\n",
      "│    │    └─BatchNorm2d: 3-440           [1, 97, 8, 8]             194\n",
      "│    │    └─ReLU: 3-441                  [1, 97, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-442                [1, 388, 8, 8]            37,636\n",
      "│    │    └─BatchNorm2d: 3-443           [1, 388, 8, 8]            776\n",
      "│    └─Bottleneck: 2-50                  [1, 392, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-444           [1, 388, 8, 8]            776\n",
      "│    │    └─Conv2d: 3-445                [1, 98, 8, 8]             38,024\n",
      "│    │    └─BatchNorm2d: 3-446           [1, 98, 8, 8]             196\n",
      "│    │    └─ReLU: 3-447                  [1, 98, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-448                [1, 98, 8, 8]             86,436\n",
      "│    │    └─BatchNorm2d: 3-449           [1, 98, 8, 8]             196\n",
      "│    │    └─ReLU: 3-450                  [1, 98, 8, 8]             --\n",
      "│    │    └─Conv2d: 3-451                [1, 392, 8, 8]            38,416\n",
      "│    │    └─BatchNorm2d: 3-452           [1, 392, 8, 8]            784\n",
      "│    └─Bottleneck: 2-51                  [1, 400, 8, 8]            --\n",
      "│    │    └─BatchNorm2d: 3-453           [1, 392, 8, 8]            784\n",
      "│    │    └─Conv2d: 3-454                [1, 100, 8, 8]            39,200\n",
      "│    │    └─BatchNorm2d: 3-455           [1, 100, 8, 8]            200\n",
      "│    │    └─ReLU: 3-456                  [1, 100, 8, 8]            --\n",
      "│    │    └─Conv2d: 3-457                [1, 100, 8, 8]            90,000\n",
      "│    │    └─BatchNorm2d: 3-458           [1, 100, 8, 8]            200\n",
      "│    │    └─ReLU: 3-459                  [1, 100, 8, 8]            --\n",
      "│    │    └─Conv2d: 3-460                [1, 400, 8, 8]            40,000\n",
      "│    │    └─BatchNorm2d: 3-461           [1, 400, 8, 8]            800\n",
      "├─BatchNorm2d: 1-6                       [1, 400, 8, 8]            800\n",
      "├─ReLU: 1-7                              [1, 400, 8, 8]            --\n",
      "├─AvgPool2d: 1-8                         [1, 400, 1, 1]            --\n",
      "├─Linear: 1-9                            [1, 100]                  40,100\n",
      "==========================================================================================\n",
      "Total params: 3,588,608\n",
      "Trainable params: 3,588,608\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (Units.MEGABYTES): 705.58\n",
      "==========================================================================================\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 114.73\n",
      "Params size (MB): 14.35\n",
      "Estimated Total Size (MB): 129.09\n",
      "==========================================================================================\n",
      "2024-10-18 12:44:29,916 DEBUG:optimizer init: sam\tlr: 0.1\tweight_decay: 0.0001\n",
      "2024-10-18 12:44:29,917 DEBUG:lr scheduler step cycle\n",
      "2024-10-18 12:44:29,917 DEBUG:save path: ./model_dir/20241018_124429\n",
      "2024-10-18 12:44:29,918 DEBUG:train start\n",
      "train: 17it [00:07,  2.14it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m main(\u001b[38;5;241m0\u001b[39m, args)\n",
      "Cell \u001b[0;32mIn[3], line 230\u001b[0m, in \u001b[0;36mmain\u001b[0;34m(rank, args)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    229\u001b[0m     logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain start\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m--> 230\u001b[0m     trainer\u001b[38;5;241m.\u001b[39mtrain()\n\u001b[1;32m    231\u001b[0m     trainer\u001b[38;5;241m.\u001b[39mget_result()\n\u001b[1;32m    232\u001b[0m     trainer\u001b[38;5;241m.\u001b[39msave_history()\n",
      "File \u001b[0;32m~/develop/jupyter/vision-py/train.py:314\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    310\u001b[0m validationloader \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_pre\u001b[38;5;241m.\u001b[39mvalidationloader\n\u001b[1;32m    312\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mepoch):\n\u001b[0;32m--> 314\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimizer, train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__feed(trainloader, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcriterion, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptimizer, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m    315\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtrain_losses\u001b[38;5;241m.\u001b[39mappend(train_loss)\n\u001b[1;32m    317\u001b[0m     train_top1, _, _, _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_eval(trainloader, get_top5\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, get_superclass\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "File \u001b[0;32m~/develop/jupyter/vision-py/train.py:270\u001b[0m, in \u001b[0;36mTrainer.__feed\u001b[0;34m(self, train_loader, model, criterion, optimizer, device)\u001b[0m\n\u001b[1;32m    267\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    268\u001b[0m         optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[0;32m--> 270\u001b[0m running_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mitem() \u001b[38;5;241m*\u001b[39m X\u001b[38;5;241m.\u001b[39msize(\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m    272\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m update_time:\n\u001b[1;32m    273\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlrs\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m__get_lr())\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "main(0, args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}