normalizer_tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chidhambararajan/docVuVenv/lib/python3.7/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: /home/chidhambararajan/docVuVenv/lib/python3.7/site-packages/torchvision/image.so: undefined symbol: _ZNK3c1010TensorImpl36is_contiguous_nondefault_policy_implENS_12MemoryFormatE\n",
      "  warn(f\"Failed to load image Python extension: {e}\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1, 149, 64)\n",
      "torch.Size([1, 2354]) (1, 1, 149, 64) (1, 149, 64)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 69\n",
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mtheseriousprogrammer\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.13.10"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>./wandb/run-20230220_072012-qhawbf0w</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/theseriousprogrammer/atmob_cls/runs/qhawbf0w' target=\"_blank\">silver-wood-37</a></strong> to <a href='https://wandb.ai/theseriousprogrammer/atmob_cls' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/theseriousprogrammer/atmob_cls' target=\"_blank\">https://wandb.ai/theseriousprogrammer/atmob_cls</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/theseriousprogrammer/atmob_cls/runs/qhawbf0w' target=\"_blank\">https://wandb.ai/theseriousprogrammer/atmob_cls/runs/qhawbf0w</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using 16bit None Automatic Mixed Precision (AMP)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n"
     ]
    }
   ],
   "source": [
    "from trainer import pl_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mapper torch.Size([1152, 1750])\n",
      "feature_network.layers.0.weight torch.Size([1])\n",
      "feature_network.layers.0.bias torch.Size([1])\n",
      "feature_network.layers.1.0.weight torch.Size([16, 1, 3, 3])\n",
      "feature_network.layers.1.1.weight torch.Size([16])\n",
      "feature_network.layers.1.1.bias torch.Size([16])\n",
      "feature_network.layers.2.conv.0.weight torch.Size([16, 16, 3, 3])\n",
      "feature_network.layers.2.conv.1.weight torch.Size([16])\n",
      "feature_network.layers.2.conv.1.bias torch.Size([16])\n",
      "feature_network.layers.2.conv.4.weight torch.Size([16, 16, 1, 1])\n",
      "feature_network.layers.2.conv.5.weight torch.Size([16])\n",
      "feature_network.layers.2.conv.5.bias torch.Size([16])\n",
      "feature_network.layers.3.conv.0.weight torch.Size([36, 16, 1, 1])\n",
      "feature_network.layers.3.conv.1.weight torch.Size([36])\n",
      "feature_network.layers.3.conv.1.bias torch.Size([36])\n",
      "feature_network.layers.3.conv.3.weight torch.Size([36, 36, 3, 3])\n",
      "feature_network.layers.3.conv.4.weight torch.Size([36])\n",
      "feature_network.layers.3.conv.4.bias torch.Size([36])\n",
      "feature_network.layers.3.conv.7.weight torch.Size([24, 36, 1, 1])\n",
      "feature_network.layers.3.conv.8.weight torch.Size([24])\n",
      "feature_network.layers.3.conv.8.bias torch.Size([24])\n",
      "feature_network.layers.4.conv.0.weight torch.Size([44, 24, 1, 1])\n",
      "feature_network.layers.4.conv.1.weight torch.Size([44])\n",
      "feature_network.layers.4.conv.1.bias torch.Size([44])\n",
      "feature_network.layers.4.conv.3.weight torch.Size([44, 44, 3, 3])\n",
      "feature_network.layers.4.conv.4.weight torch.Size([44])\n",
      "feature_network.layers.4.conv.4.bias torch.Size([44])\n",
      "feature_network.layers.4.conv.7.weight torch.Size([24, 44, 1, 1])\n",
      "feature_network.layers.4.conv.8.weight torch.Size([24])\n",
      "feature_network.layers.4.conv.8.bias torch.Size([24])\n",
      "feature_network.layers.5.conv.0.weight torch.Size([48, 24, 1, 1])\n",
      "feature_network.layers.5.conv.1.weight torch.Size([48])\n",
      "feature_network.layers.5.conv.1.bias torch.Size([48])\n",
      "feature_network.layers.5.conv.3.weight torch.Size([48, 48, 5, 5])\n",
      "feature_network.layers.5.conv.4.weight torch.Size([48])\n",
      "feature_network.layers.5.conv.4.bias torch.Size([48])\n",
      "feature_network.layers.5.conv.5.scale torch.Size([1])\n",
      "feature_network.layers.5.conv.5.query_layer.weight torch.Size([48, 48, 1, 1])\n",
      "feature_network.layers.5.conv.5.query_layer.bias torch.Size([48])\n",
      "feature_network.layers.5.conv.5.key_layer.weight torch.Size([48, 48, 1, 1])\n",
      "feature_network.layers.5.conv.5.key_layer.bias torch.Size([48])\n",
      "feature_network.layers.5.conv.5.value.weight torch.Size([48, 48, 3, 3])\n",
      "feature_network.layers.5.conv.5.value.bias torch.Size([48])\n",
      "feature_network.layers.5.conv.7.weight torch.Size([40, 48, 1, 1])\n",
      "feature_network.layers.5.conv.8.weight torch.Size([40])\n",
      "feature_network.layers.5.conv.8.bias torch.Size([40])\n",
      "feature_network.layers.6.conv.0.weight torch.Size([120, 40, 1, 1])\n",
      "feature_network.layers.6.conv.1.weight torch.Size([120])\n",
      "feature_network.layers.6.conv.1.bias torch.Size([120])\n",
      "feature_network.layers.6.conv.3.weight torch.Size([120, 120, 5, 5])\n",
      "feature_network.layers.6.conv.4.weight torch.Size([120])\n",
      "feature_network.layers.6.conv.4.bias torch.Size([120])\n",
      "feature_network.layers.6.conv.7.weight torch.Size([40, 120, 1, 1])\n",
      "feature_network.layers.6.conv.8.weight torch.Size([40])\n",
      "feature_network.layers.6.conv.8.bias torch.Size([40])\n",
      "feature_network.layers.7.conv.0.weight torch.Size([120, 40, 1, 1])\n",
      "feature_network.layers.7.conv.1.weight torch.Size([120])\n",
      "feature_network.layers.7.conv.1.bias torch.Size([120])\n",
      "feature_network.layers.7.conv.3.weight torch.Size([120, 120, 5, 5])\n",
      "feature_network.layers.7.conv.4.weight torch.Size([120])\n",
      "feature_network.layers.7.conv.4.bias torch.Size([120])\n",
      "feature_network.layers.7.conv.7.weight torch.Size([40, 120, 1, 1])\n",
      "feature_network.layers.7.conv.8.weight torch.Size([40])\n",
      "feature_network.layers.7.conv.8.bias torch.Size([40])\n",
      "feature_network.layers.8.conv.0.weight torch.Size([60, 40, 1, 1])\n",
      "feature_network.layers.8.conv.1.weight torch.Size([60])\n",
      "feature_network.layers.8.conv.1.bias torch.Size([60])\n",
      "feature_network.layers.8.conv.3.weight torch.Size([60, 60, 5, 5])\n",
      "feature_network.layers.8.conv.4.weight torch.Size([60])\n",
      "feature_network.layers.8.conv.4.bias torch.Size([60])\n",
      "feature_network.layers.8.conv.7.weight torch.Size([48, 60, 1, 1])\n",
      "feature_network.layers.8.conv.8.weight torch.Size([48])\n",
      "feature_network.layers.8.conv.8.bias torch.Size([48])\n",
      "feature_network.layers.9.conv.0.weight torch.Size([72, 48, 1, 1])\n",
      "feature_network.layers.9.conv.1.weight torch.Size([72])\n",
      "feature_network.layers.9.conv.1.bias torch.Size([72])\n",
      "feature_network.layers.9.conv.3.weight torch.Size([72, 72, 5, 5])\n",
      "feature_network.layers.9.conv.4.weight torch.Size([72])\n",
      "feature_network.layers.9.conv.4.bias torch.Size([72])\n",
      "feature_network.layers.9.conv.5.scale torch.Size([1])\n",
      "feature_network.layers.9.conv.5.query_layer.weight torch.Size([72, 72, 1, 1])\n",
      "feature_network.layers.9.conv.5.query_layer.bias torch.Size([72])\n",
      "feature_network.layers.9.conv.5.key_layer.weight torch.Size([72, 72, 1, 1])\n",
      "feature_network.layers.9.conv.5.key_layer.bias torch.Size([72])\n",
      "feature_network.layers.9.conv.5.value.weight torch.Size([72, 72, 3, 3])\n",
      "feature_network.layers.9.conv.5.value.bias torch.Size([72])\n",
      "feature_network.layers.9.conv.7.weight torch.Size([48, 72, 1, 1])\n",
      "feature_network.layers.9.conv.8.weight torch.Size([48])\n",
      "feature_network.layers.9.conv.8.bias torch.Size([48])\n",
      "feature_network.layers.10.conv.0.weight torch.Size([144, 48, 1, 1])\n",
      "feature_network.layers.10.conv.1.weight torch.Size([144])\n",
      "feature_network.layers.10.conv.1.bias torch.Size([144])\n",
      "feature_network.layers.10.conv.3.weight torch.Size([144, 144, 5, 5])\n",
      "feature_network.layers.10.conv.4.weight torch.Size([144])\n",
      "feature_network.layers.10.conv.4.bias torch.Size([144])\n",
      "feature_network.layers.10.conv.7.weight torch.Size([96, 144, 1, 1])\n",
      "feature_network.layers.10.conv.8.weight torch.Size([96])\n",
      "feature_network.layers.10.conv.8.bias torch.Size([96])\n",
      "feature_network.layers.11.conv.0.weight torch.Size([288, 96, 1, 1])\n",
      "feature_network.layers.11.conv.1.weight torch.Size([288])\n",
      "feature_network.layers.11.conv.1.bias torch.Size([288])\n",
      "feature_network.layers.11.conv.3.weight torch.Size([288, 288, 5, 5])\n",
      "feature_network.layers.11.conv.4.weight torch.Size([288])\n",
      "feature_network.layers.11.conv.4.bias torch.Size([288])\n",
      "feature_network.layers.11.conv.7.weight torch.Size([96, 288, 1, 1])\n",
      "feature_network.layers.11.conv.8.weight torch.Size([96])\n",
      "feature_network.layers.11.conv.8.bias torch.Size([96])\n",
      "feature_network.layers.12.conv.0.weight torch.Size([288, 96, 1, 1])\n",
      "feature_network.layers.12.conv.1.weight torch.Size([288])\n",
      "feature_network.layers.12.conv.1.bias torch.Size([288])\n",
      "feature_network.layers.12.conv.3.weight torch.Size([288, 288, 5, 5])\n",
      "feature_network.layers.12.conv.4.weight torch.Size([288])\n",
      "feature_network.layers.12.conv.4.bias torch.Size([288])\n",
      "feature_network.layers.12.conv.7.weight torch.Size([96, 288, 1, 1])\n",
      "feature_network.layers.12.conv.8.weight torch.Size([96])\n",
      "feature_network.layers.12.conv.8.bias torch.Size([96])\n",
      "feature_network.layers.13.scale torch.Size([1])\n",
      "feature_network.layers.13.query_layer.weight torch.Size([96, 96, 1, 1])\n",
      "feature_network.layers.13.query_layer.bias torch.Size([96])\n",
      "feature_network.layers.13.key_layer.weight torch.Size([96, 96, 1, 1])\n",
      "feature_network.layers.13.key_layer.bias torch.Size([96])\n",
      "feature_network.layers.13.value.weight torch.Size([96, 96, 3, 3])\n",
      "feature_network.layers.13.value.bias torch.Size([96])\n",
      "feature_network.layers.14.0.weight torch.Size([576, 96, 1, 1])\n",
      "feature_network.layers.14.1.weight torch.Size([576])\n",
      "feature_network.layers.14.1.bias torch.Size([576])\n",
      "feature_network.last_layers.1.weight torch.Size([1152])\n",
      "feature_network.last_layers.1.bias torch.Size([1152])\n"
     ]
    }
   ],
   "source": [
    "for name, param in pl_model.pytorch_model.named_parameters() :\n",
    "    print(name, param.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BatchNorm2d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<MeanBackward0>) tensor(nan, grad_fn=<VarBackward0>)\n",
      "Conv2d(1, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(1, 16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) tensor(-0.0021, grad_fn=<MeanBackward0>) tensor(0.0070, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.2500, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) tensor(-0.0007, grad_fn=<MeanBackward0>) tensor(0.0004, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.2500, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(16, 16, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(16, 16, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-0.0051, grad_fn=<MeanBackward0>) tensor(0.0039, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.2500, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(16, 36, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(16, 36, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-0.0011, grad_fn=<MeanBackward0>) tensor(0.0017, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1667, grad_fn=<MeanBackward0>) tensor(2.2839e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(36, 36, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(36, 36, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) tensor(9.2898e-05, grad_fn=<MeanBackward0>) tensor(8.5733e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(36, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1667, grad_fn=<MeanBackward0>) tensor(2.2839e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(36, 24, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(36, 24, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-0.0007, grad_fn=<MeanBackward0>) tensor(0.0012, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.2041, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(24, 44, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(24, 44, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(0.0012, grad_fn=<MeanBackward0>) tensor(0.0009, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(44, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(44, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1508, grad_fn=<MeanBackward0>) tensor(2.2721e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(44, 44, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(44, 44, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False) tensor(0.0002, grad_fn=<MeanBackward0>) tensor(5.7373e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(44, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(44, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1508, grad_fn=<MeanBackward0>) tensor(2.2721e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(44, 24, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(44, 24, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(0.0006, grad_fn=<MeanBackward0>) tensor(0.0009, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.2041, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(24, 48, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-0.0004, grad_fn=<MeanBackward0>) tensor(0.0009, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1443, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), bias=False) tensor(4.5204e-06, grad_fn=<MeanBackward0>) tensor(1.7361e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1443, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1)) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1)) tensor(-0.0004, grad_fn=<MeanBackward0>) tensor(0.0004, grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1)) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(1, 1), stride=(1, 1)) tensor(-0.0006, grad_fn=<MeanBackward0>) tensor(0.0004, grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) tensor(3.6784e-05, grad_fn=<MeanBackward0>) tensor(4.8226e-05, grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 40, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 40, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-0.0005, grad_fn=<MeanBackward0>) tensor(0.0005, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1581, grad_fn=<MeanBackward0>) tensor(2.2774e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(40, 120, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(40, 120, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(0.0004, grad_fn=<MeanBackward0>) tensor(0.0002, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0913, grad_fn=<MeanBackward0>) tensor(5.5978e-17, grad_fn=<VarBackward0>)\n",
      "Conv2d(120, 120, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(120, 120, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(5.3468e-07, grad_fn=<MeanBackward0>) tensor(2.7778e-06, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0913, grad_fn=<MeanBackward0>) tensor(5.5978e-17, grad_fn=<VarBackward0>)\n",
      "Conv2d(120, 40, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(120, 40, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-0.0003, grad_fn=<MeanBackward0>) tensor(0.0002, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1581, grad_fn=<MeanBackward0>) tensor(2.2774e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(40, 120, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(40, 120, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(6.2548e-05, grad_fn=<MeanBackward0>) tensor(0.0002, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0913, grad_fn=<MeanBackward0>) tensor(5.5978e-17, grad_fn=<VarBackward0>)\n",
      "Conv2d(120, 120, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(120, 120, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(3.8836e-06, grad_fn=<MeanBackward0>) tensor(2.7778e-06, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0913, grad_fn=<MeanBackward0>) tensor(5.5978e-17, grad_fn=<VarBackward0>)\n",
      "Conv2d(120, 40, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(120, 40, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(0.0002, grad_fn=<MeanBackward0>) tensor(0.0002, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1581, grad_fn=<MeanBackward0>) tensor(2.2774e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(40, 60, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(40, 60, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(0.0002, grad_fn=<MeanBackward0>) tensor(0.0004, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1291, grad_fn=<MeanBackward0>) tensor(2.2581e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(60, 60, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(60, 60, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(2.1907e-06, grad_fn=<MeanBackward0>) tensor(1.1111e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1291, grad_fn=<MeanBackward0>) tensor(2.2581e-16, grad_fn=<VarBackward0>)\n",
      "Conv2d(60, 48, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(60, 48, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-3.3942e-05, grad_fn=<MeanBackward0>) tensor(0.0003, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1443, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 72, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 72, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-5.7891e-07, grad_fn=<MeanBackward0>) tensor(0.0003, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(72, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(72, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1179, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(-4.4829e-06, grad_fn=<MeanBackward0>) tensor(7.7161e-06, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(72, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(72, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1179, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(1, 1), stride=(1, 1)) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(1, 1), stride=(1, 1)) tensor(0.0004, grad_fn=<MeanBackward0>) tensor(0.0002, grad_fn=<VarBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(1, 1), stride=(1, 1)) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(1, 1), stride=(1, 1)) tensor(0.0002, grad_fn=<MeanBackward0>) tensor(0.0002, grad_fn=<VarBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(72, 72, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) tensor(-2.3052e-05, grad_fn=<MeanBackward0>) tensor(2.1433e-05, grad_fn=<VarBackward0>)\n",
      "Conv2d(72, 48, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(72, 48, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-4.5313e-05, grad_fn=<MeanBackward0>) tensor(0.0003, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(48, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1443, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(48, 144, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(48, 144, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-2.2923e-05, grad_fn=<MeanBackward0>) tensor(0.0001, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0833, grad_fn=<MeanBackward0>) tensor(5.5899e-17, grad_fn=<VarBackward0>)\n",
      "Conv2d(144, 144, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(144, 144, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), bias=False) tensor(4.5455e-07, grad_fn=<MeanBackward0>) tensor(1.9290e-06, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0833, grad_fn=<MeanBackward0>) tensor(5.5899e-17, grad_fn=<VarBackward0>)\n",
      "Conv2d(144, 96, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(144, 96, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(8.4591e-05, grad_fn=<MeanBackward0>) tensor(7.2336e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1021, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(96, 288, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(96, 288, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(3.3817e-06, grad_fn=<MeanBackward0>) tensor(3.6170e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0589, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(288, 288, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(288, 288, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(2.6957e-07, grad_fn=<MeanBackward0>) tensor(4.8228e-07, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0589, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(288, 96, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(288, 96, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-1.4012e-05, grad_fn=<MeanBackward0>) tensor(3.6170e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1021, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(96, 288, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(96, 288, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0762e-05, grad_fn=<MeanBackward0>) tensor(3.6170e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0589, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(288, 288, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(288, 288, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False) tensor(1.1218e-07, grad_fn=<MeanBackward0>) tensor(4.8228e-07, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(288, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0589, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(288, 96, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(288, 96, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(5.9593e-05, grad_fn=<MeanBackward0>) tensor(3.6167e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.1021, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) tensor(5.5471e-05, grad_fn=<MeanBackward0>) tensor(0.0001, grad_fn=<VarBackward0>)\n",
      "Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) tensor(2.7100e-05, grad_fn=<MeanBackward0>) tensor(0.0001, grad_fn=<VarBackward0>)\n",
      "Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) tensor(1., grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) tensor(2.5146e-06, grad_fn=<MeanBackward0>) tensor(1.2056e-05, grad_fn=<VarBackward0>)\n",
      "Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "Conv2d(96, 576, kernel_size=(1, 1), stride=(1, 1), bias=False) tensor(-1.1704e-05, grad_fn=<MeanBackward0>) tensor(1.8085e-05, grad_fn=<VarBackward0>)\n",
      "BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm2d(576, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0417, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n",
      "BatchNorm1d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(1.0000, grad_fn=<LinalgVectorNormBackward0>)\n",
      "BatchNorm1d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) tensor(0.0295, grad_fn=<MeanBackward0>) tensor(0., grad_fn=<VarBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "for module in pl_model.pytorch_model.modules():\n",
    "    #print(module)\n",
    "    #if isinstance(module, nn.Parameter) :\n",
    "    #    print(module.data)\n",
    "    #continue    \n",
    "    if not hasattr(module, \"weight\") :\n",
    "        continue\n",
    "    print(module, torch.linalg.norm(module.weight/torch.linalg.norm(module.weight)))\n",
    "    print(module, torch.mean(module.weight/torch.linalg.norm(module.weight)), torch.var(module.weight/torch.linalg.norm(module.weight)))\n",
    "    continue\n",
    "    if isinstance(module, nn.Conv2d):\n",
    "    \n",
    "        for iteration in range(1):\n",
    "            print(module.weight.shape)\n",
    "            weight_mean = module.weight.mean(dim=1, keepdim=True).mean(dim=2, keepdim=True).mean(dim=3, keepdim=True)\n",
    "            std = torch.sqrt(torch.var(module.weight.view(module.weight.size(0), -1), dim=1) + 1e-12).view(-1, 1, 1, 1) + 1e-5\n",
    "    \n",
    "            module.weight = torch.nn.Parameter(module.weight - weight_mean)\n",
    "            module.weight = torch.nn.Parameter(module.weight / std.expand_as(module.weight))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pl_model.pytorch_model.mapper.data = nn.functional.normalize(pl_model.pytorch_model.mapper.data, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "2283c03fd36d1fa019abbf902ee0991b5e2edf470abe8ab5e1e2a09291b1fce4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}