PANDA/panda-recov-submission.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6eedbaee",
   "metadata": {
    "papermill": {
     "duration": 0.005222,
     "end_time": "2024-02-08T14:55:59.127267",
     "exception": false,
     "start_time": "2024-02-08T14:55:59.122045",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Description\n",
    "This kernel performs inference for PANDA dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f927617d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:55:59.138661Z",
     "iopub.status.busy": "2024-02-08T14:55:59.138296Z",
     "iopub.status.idle": "2024-02-08T14:57:02.266755Z",
     "shell.execute_reply": "2024-02-08T14:57:02.265781Z"
    },
    "papermill": {
     "duration": 63.136813,
     "end_time": "2024-02-08T14:57:02.269071",
     "exception": false,
     "start_time": "2024-02-08T14:55:59.132258",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing /kaggle/timm-0.5.4\r\n",
      "  Preparing metadata (setup.py) ... \u001B[?25l-\b \bdone\r\n",
      "\u001B[?25hRequirement already satisfied: torch>=1.4 in /opt/conda/lib/python3.10/site-packages (from timm==0.5.4) (2.1.2)\r\n",
      "Requirement already satisfied: torchvision in /opt/conda/lib/python3.10/site-packages (from timm==0.5.4) (0.16.2)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.4->timm==0.5.4) (3.13.1)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.4->timm==0.5.4) (4.9.0)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.4->timm==0.5.4) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.4->timm==0.5.4) (3.2.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.4->timm==0.5.4) (3.1.2)\r\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.4->timm==0.5.4) (2023.12.2)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from torchvision->timm==0.5.4) (1.24.4)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from torchvision->timm==0.5.4) (2.31.0)\r\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.10/site-packages (from torchvision->timm==0.5.4) (9.5.0)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.4->timm==0.5.4) (2.1.3)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->torchvision->timm==0.5.4) (3.3.2)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->torchvision->timm==0.5.4) (3.6)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->torchvision->timm==0.5.4) (1.26.18)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->torchvision->timm==0.5.4) (2023.11.17)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.4->timm==0.5.4) (1.3.0)\r\n",
      "Building wheels for collected packages: timm\r\n",
      "  Building wheel for timm (setup.py) ... \u001B[?25l-\b \bdone\r\n",
      "\u001B[?25h  Created wheel for timm: filename=timm-0.5.4-py3-none-any.whl size=431521 sha256=ca99ab16c3d014897f580d21c49a0d77f57e00fdc22eb8c1d0d0b7ec45c4e868\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/cd/16/62/903e9b342497a308c0b62f74fe47e09f2315c459cd4e39e86e\r\n",
      "Successfully built timm\r\n",
      "Installing collected packages: timm\r\n",
      "  Attempting uninstall: timm\r\n",
      "    Found existing installation: timm 0.9.12\r\n",
      "    Uninstalling timm-0.9.12:\r\n",
      "      Successfully uninstalled timm-0.9.12\r\n",
      "Successfully installed timm-0.5.4\r\n",
      "Processing /kaggle/input/nystrom-attn/einops-0.7.0-py3-none-any.whl\r\n",
      "Installing collected packages: einops\r\n",
      "Successfully installed einops-0.7.0\r\n",
      "Processing /kaggle/input/nystrom-attn/nystrom_attention-0.0.11-py3-none-any.whl\r\n",
      "Requirement already satisfied: einops>=0.3 in /opt/conda/lib/python3.10/site-packages (from nystrom-attention==0.0.11) (0.7.0)\r\n",
      "Requirement already satisfied: torch>=1.6 in /opt/conda/lib/python3.10/site-packages (from nystrom-attention==0.0.11) (2.1.2)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->nystrom-attention==0.0.11) (3.13.1)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->nystrom-attention==0.0.11) (4.9.0)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->nystrom-attention==0.0.11) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->nystrom-attention==0.0.11) (3.2.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->nystrom-attention==0.0.11) (3.1.2)\r\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.6->nystrom-attention==0.0.11) (2023.12.2)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6->nystrom-attention==0.0.11) (2.1.3)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6->nystrom-attention==0.0.11) (1.3.0)\r\n",
      "Installing collected packages: nystrom-attention\r\n",
      "Successfully installed nystrom-attention-0.0.11\r\n"
     ]
    }
   ],
   "source": [
    "!cp -r /kaggle/input/customtimm/timm-0.5.4 /kaggle\n",
    "!pip install /kaggle/timm-0.5.4\n",
    "!pip install --no-index /kaggle/input/nystrom-attn/einops-0.7.0-py3-none-any.whl\n",
    "!pip install --no-index /kaggle/input/nystrom-attn/nystrom_attention-0.0.11-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98027983",
   "metadata": {
    "papermill": {
     "duration": 0.006356,
     "end_time": "2024-02-08T14:57:02.282184",
     "exception": false,
     "start_time": "2024-02-08T14:57:02.275828",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Install Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b4ec3918",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:02.296541Z",
     "iopub.status.busy": "2024-02-08T14:57:02.296227Z",
     "iopub.status.idle": "2024-02-08T14:57:11.742386Z",
     "shell.execute_reply": "2024-02-08T14:57:11.741256Z"
    },
    "papermill": {
     "duration": 9.456126,
     "end_time": "2024-02-08T14:57:11.744754",
     "exception": false,
     "start_time": "2024-02-08T14:57:02.288628",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import cv2\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "# import fastai\n",
    "# from fastai.vision import *\n",
    "import os\n",
    "from mish_activation import *\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import skimage.io\n",
    "from skimage import color\n",
    "from scipy.ndimage.morphology import binary_fill_holes\n",
    "from skimage.transform import rescale,resize\n",
    "# sys.path.insert(0, '../input/semisupervised-imagenet-models/semi-supervised-ImageNet1K-models-master/')\n",
    "# from hubconf import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch, torchvision\n",
    "import torch.nn as nn\n",
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "from torch.utils.data import Dataset\n",
    "from timm.models.layers.helpers import to_2tuple\n",
    "import timm\n",
    "import torch.nn as nn\n",
    "from nystrom_attention import NystromAttention\n",
    "from openslide import OpenSlide"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d79a05be",
   "metadata": {
    "papermill": {
     "duration": 0.006316,
     "end_time": "2024-02-08T14:57:11.757650",
     "exception": false,
     "start_time": "2024-02-08T14:57:11.751334",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83f608da",
   "metadata": {
    "papermill": {
     "duration": 0.006104,
     "end_time": "2024-02-08T14:57:11.769995",
     "exception": false,
     "start_time": "2024-02-08T14:57:11.763891",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### CtransPath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "97325a38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:11.783884Z",
     "iopub.status.busy": "2024-02-08T14:57:11.783557Z",
     "iopub.status.idle": "2024-02-08T14:57:13.683865Z",
     "shell.execute_reply": "2024-02-08T14:57:13.682960Z"
    },
    "papermill": {
     "duration": 1.909868,
     "end_time": "2024-02-08T14:57:13.686056",
     "exception": false,
     "start_time": "2024-02-08T14:57:11.776188",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class ConvStem(nn.Module):\n",
    "\n",
    "    def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=768, norm_layer=None, flatten=True):\n",
    "        super().__init__()\n",
    "\n",
    "        assert patch_size == 4\n",
    "        assert embed_dim % 8 == 0\n",
    "\n",
    "        img_size = to_2tuple(img_size)\n",
    "        patch_size = to_2tuple(patch_size)\n",
    "        self.img_size = img_size\n",
    "        self.patch_size = patch_size\n",
    "        self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1])\n",
    "        self.num_patches = self.grid_size[0] * self.grid_size[1]\n",
    "        self.flatten = flatten\n",
    "\n",
    "\n",
    "        stem = []\n",
    "        input_dim, output_dim = 3, embed_dim // 8\n",
    "        for l in range(2):\n",
    "            stem.append(nn.Conv2d(input_dim, output_dim, kernel_size=3, stride=2, padding=1, bias=False))\n",
    "            stem.append(nn.BatchNorm2d(output_dim))\n",
    "            stem.append(nn.ReLU(inplace=True))\n",
    "            input_dim = output_dim\n",
    "            output_dim *= 2\n",
    "        stem.append(nn.Conv2d(input_dim, embed_dim, kernel_size=1))\n",
    "        self.proj = nn.Sequential(*stem)\n",
    "\n",
    "        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()\n",
    "\n",
    "    def forward(self, x):\n",
    "        B, C, H, W = x.shape\n",
    "        assert H == self.img_size[0] and W == self.img_size[1], \\\n",
    "            f\"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]}).\"\n",
    "        x = self.proj(x)\n",
    "        if self.flatten:\n",
    "            x = x.flatten(2).transpose(1, 2)  # BCHW -> BNC\n",
    "        x = self.norm(x)\n",
    "        return x\n",
    "\n",
    "def ctranspath():\n",
    "    model = timm.create_model('swin_tiny_patch4_window7_224', embed_layer=ConvStem, pretrained=False)\n",
    "    return model\n",
    "\n",
    "\n",
    "featextractor = ctranspath()\n",
    "featextractor.head = nn.Identity()\n",
    "td = torch.load(r'/kaggle/input/model-weights/ctranspath.pth')\n",
    "featextractor.load_state_dict(td['model'], strict=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8381991",
   "metadata": {
    "papermill": {
     "duration": 0.006407,
     "end_time": "2024-02-08T14:57:13.699111",
     "exception": false,
     "start_time": "2024-02-08T14:57:13.692704",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### MIL model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f5512d09",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:13.713368Z",
     "iopub.status.busy": "2024-02-08T14:57:13.713070Z",
     "iopub.status.idle": "2024-02-08T14:57:13.765181Z",
     "shell.execute_reply": "2024-02-08T14:57:13.764306Z"
    },
    "papermill": {
     "duration": 0.061634,
     "end_time": "2024-02-08T14:57:13.767222",
     "exception": false,
     "start_time": "2024-02-08T14:57:13.705588",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class TransLayer(nn.Module):\n",
    "\n",
    "    def __init__(self, norm_layer=nn.LayerNorm, dim=128):\n",
    "        super().__init__()\n",
    "        self.norm = norm_layer(dim)\n",
    "        self.attn = NystromAttention(\n",
    "            dim = dim,\n",
    "            dim_head = dim//8,\n",
    "            heads = 8,\n",
    "            num_landmarks = dim//2,    # number of landmarks\n",
    "            pinv_iterations = 6,    # number of moore-penrose iterations for approximating pinverse. 6 was recommended by the paper\n",
    "            residual = True,         # whether to do an extra residual with the value or not. supposedly faster convergence if turned on\n",
    "            dropout=0.1\n",
    "        )\n",
    "\n",
    "    def forward(self, x, return_attn=False):\n",
    "        if return_attn:\n",
    "            attn_out, attn_vals = self.attn(self.norm(x),return_attn=return_attn)\n",
    "            x = x + attn_out\n",
    "            return x, attn_vals\n",
    "        else:\n",
    "            attn_out = self.attn(self.norm(x),return_attn=return_attn)\n",
    "            x = x + attn_out\n",
    "            return x\n",
    "\n",
    "\n",
    "class PEG(nn.Module):\n",
    "    def __init__(self, dim=256, k=7):\n",
    "        super(PEG, self).__init__()\n",
    "        self.proj = nn.Conv2d(dim, dim, k, 1, k//2, groups=dim)\n",
    "        self.proj1 = nn.Conv2d(dim, dim, 5, 1, 5//2, groups=dim)\n",
    "        self.proj2 = nn.Conv2d(dim, dim, 3, 1, 3//2, groups=dim)\n",
    "\n",
    "\n",
    "    def forward(self, x, H, W):\n",
    "        B, N, C = x.shape\n",
    "        cls_token, feat_token = x[:, 0], x[:, 1:]\n",
    "        cnn_feat = feat_token.transpose(1, 2).view(B, C, H, W)\n",
    "        x = self.proj(cnn_feat)+cnn_feat+self.proj1(cnn_feat)+self.proj2(cnn_feat)\n",
    "        x = x.flatten(2).transpose(1, 2)\n",
    "        x = torch.cat((cls_token.unsqueeze(1), x), dim=1)\n",
    "        return x\n",
    "\n",
    "'''\n",
    "Code copied from https://github.com/szc19990412/HVTSurv/blob/main/models/TransMIL.py and slightly modified for this codebase\n",
    "'''\n",
    "class TransMIL_peg(nn.Module):\n",
    "    def __init__(self, n_classes,dim=128):\n",
    "        super(TransMIL_peg, self).__init__()\n",
    "        self.pos_layer = PEG(dim)\n",
    "        self._fc1 = nn.Sequential(nn.Linear(768, dim), nn.ReLU())\n",
    "        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n",
    "        self.n_classes = n_classes\n",
    "        self.layer1 = TransLayer(dim=dim)\n",
    "        self.layer2 = TransLayer(dim=dim)\n",
    "        self.norm = nn.LayerNorm(dim)\n",
    "        # self._fc2 = nn.Linear(dim, self.n_classes)\n",
    "        # self._fc2 = nn.Sequential(*[nn.Linear(dim,dim), nn.LayerNorm(dim), nn.ReLU(), nn.Linear(dim, n_classes)])\n",
    "        # self._fc2 = nn.Sequential(*[nn.Linear(dim,dim), nn.ReLU(), nn.Linear(dim, n_classes)])\n",
    "        self._fc2 = nn.Sequential(*[nn.Linear(dim,dim), nn.ReLU(), nn.Dropout(0.25), nn.Linear(dim, n_classes)])\n",
    "\n",
    "\n",
    "    def forward(self, x, return_attn=False):\n",
    "\n",
    "        h = x.float().unsqueeze(0) #[B, n, 768]\n",
    "        \n",
    "        #---->Dimensionality reduction first\n",
    "        h = self._fc1(h) #[B, n, 128]\n",
    "        \n",
    "        #---->padding\n",
    "        H = h.shape[1]\n",
    "        _H, _W = int(np.ceil(np.sqrt(H))), int(np.ceil(np.sqrt(H)))\n",
    "        add_length = _H * _W - H\n",
    "        h = torch.cat([h, h[:,:add_length,:]],dim = 1) #[B, N, 128]\n",
    "\n",
    "        #---->Add position encoding, after a transformer\n",
    "        B = h.shape[0]\n",
    "        cls_tokens = self.cls_token.expand(B, -1, -1).cuda()\n",
    "        h = torch.cat((cls_tokens, h), dim=1)\n",
    "\n",
    "        #---->Translayer x1\n",
    "        h = self.layer1(h) #[B, N, 128]\n",
    "\n",
    "        #---->PPEG\n",
    "        h = self.pos_layer(h, _H, _W) #[B, N, 128]\n",
    "        \n",
    "        #---->Translayer x2\n",
    "        if return_attn:\n",
    "            h, attn_vals = self.layer2(h,return_attn) #[B, N, 128]\n",
    "        else:\n",
    "            h = self.layer2(h)\n",
    "\n",
    "        h = self.norm(h)[:,0]\n",
    "\n",
    "        #---->predict output\n",
    "        logits = self._fc2(h)\n",
    "        if return_attn:\n",
    "            return logits, attn_vals\n",
    "        else:\n",
    "            return logits\n",
    "\n",
    "mil_model = TransMIL_peg(n_classes=5, dim=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2e36e4cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:13.781827Z",
     "iopub.status.busy": "2024-02-08T14:57:13.781172Z",
     "iopub.status.idle": "2024-02-08T14:57:13.788205Z",
     "shell.execute_reply": "2024-02-08T14:57:13.787373Z"
    },
    "papermill": {
     "duration": 0.016667,
     "end_time": "2024-02-08T14:57:13.790343",
     "exception": false,
     "start_time": "2024-02-08T14:57:13.773676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "mode = 'test'\n",
    "# mode = 'train'\n",
    "DATA = f'../input/prostate-cancer-grade-assessment/{mode}_images'\n",
    "TEST = f'../input/prostate-cancer-grade-assessment/{mode}.csv'\n",
    "# TEST = \"/kaggle/input/testing/testsplit.csv\"\n",
    "SAMPLE = '../input/prostate-cancer-grade-assessment/sample_submission.csv'\n",
    "\n",
    "MODEL_WEIGHT = \"fastrecov_trained_model.pt\"\n",
    "\n",
    "\n",
    "#at 0.5mpp, at 1mpp, sz*2\n",
    "sz = 224\n",
    "nworkers = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6bb3580",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:13.805735Z",
     "iopub.status.busy": "2024-02-08T14:57:13.805031Z",
     "iopub.status.idle": "2024-02-08T14:57:14.178264Z",
     "shell.execute_reply": "2024-02-08T14:57:14.177350Z"
    },
    "papermill": {
     "duration": 0.382734,
     "end_time": "2024-02-08T14:57:14.180416",
     "exception": false,
     "start_time": "2024-02-08T14:57:13.797682",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TransMIL_peg(\n",
       "  (pos_layer): PEG(\n",
       "    (proj): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
       "    (proj1): Conv2d(512, 512, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=512)\n",
       "    (proj2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=512)\n",
       "  )\n",
       "  (_fc1): Sequential(\n",
       "    (0): Linear(in_features=768, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "  )\n",
       "  (layer1): TransLayer(\n",
       "    (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "    (attn): NystromAttention(\n",
       "      (to_qkv): Linear(in_features=512, out_features=1536, bias=False)\n",
       "      (to_out): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (res_conv): Conv2d(8, 8, kernel_size=(33, 1), stride=(1, 1), padding=(16, 0), groups=8, bias=False)\n",
       "    )\n",
       "  )\n",
       "  (layer2): TransLayer(\n",
       "    (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "    (attn): NystromAttention(\n",
       "      (to_qkv): Linear(in_features=512, out_features=1536, bias=False)\n",
       "      (to_out): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (res_conv): Conv2d(8, 8, kernel_size=(33, 1), stride=(1, 1), padding=(16, 0), groups=8, bias=False)\n",
       "    )\n",
       "  )\n",
       "  (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "  (_fc2): Sequential(\n",
       "    (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): Dropout(p=0.25, inplace=False)\n",
       "    (3): Linear(in_features=512, out_features=5, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mil_model.load_state_dict(torch.load(MODEL_WEIGHT))\n",
    "featextractor.cuda()\n",
    "mil_model.cuda()\n",
    "featextractor.eval()\n",
    "mil_model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c1a9f60",
   "metadata": {
    "papermill": {
     "duration": 0.006417,
     "end_time": "2024-02-08T14:57:14.193886",
     "exception": false,
     "start_time": "2024-02-08T14:57:14.187469",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dfa26e49",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:14.211629Z",
     "iopub.status.busy": "2024-02-08T14:57:14.211229Z",
     "iopub.status.idle": "2024-02-08T14:57:14.232304Z",
     "shell.execute_reply": "2024-02-08T14:57:14.231437Z"
    },
    "papermill": {
     "duration": 0.034594,
     "end_time": "2024-02-08T14:57:14.235135",
     "exception": false,
     "start_time": "2024-02-08T14:57:14.200541",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def tile(path):\n",
    "    scan = OpenSlide(path)\n",
    "    img = skimage.io.MultiImage(path)[-1]\n",
    "    level_dimensions = scan.level_dimensions\n",
    "    image_array = np.asarray(scan.read_region((0, 0), len(level_dimensions)-1, level_dimensions[-1]).convert('RGB'))\n",
    "    shape = img.shape\n",
    "    \n",
    "    #get mask from image\n",
    "    threshold = 0.1\n",
    "    lab = color.rgb2lab(image_array)\n",
    "    mean = np.mean(lab[..., 1])\n",
    "    lab = lab[..., 1] > (1 + threshold ) * mean\n",
    "    mask = lab.astype(np.uint8)\n",
    "    fill_mask_kernel_size=9\n",
    "    mask = binary_fill_holes(mask)\n",
    "    kernel = np.ones((fill_mask_kernel_size, fill_mask_kernel_size), np.uint8)\n",
    "    mask = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_CLOSE, kernel)\n",
    "#     mask = resize(mask,img.shape[:-1])\n",
    "    mask = (mask>0)*1\n",
    "    downsample_factor = int(level_dimensions[0][0]/level_dimensions[-1][0])\n",
    "    \n",
    "    sz_big = sz*2 \n",
    "    lim0,lim1 = shape[0]-shape[0]%sz_big,shape[1]-shape[1]%sz_big \n",
    "    sz_mask = int(sz_big/downsample_factor)\n",
    "    img = img[:lim0,:lim1,:]\n",
    "    mask = mask[:int(lim0//downsample_factor),:int(lim1//downsample_factor)]\n",
    "    img = img.reshape(img.shape[0]//sz_big,sz_big,img.shape[1]//sz_big,sz_big,3)\n",
    "    mask = mask.reshape(mask.shape[0]//sz_mask,sz_mask,mask.shape[1]//sz_mask,sz_mask,1)\n",
    "    img = img.transpose(0,2,1,3,4).reshape(-1,sz_big,sz_big,3)\n",
    "    mask = mask.transpose(0,2,1,3,4).reshape(-1,sz_mask,sz_mask,1)\n",
    "    idxs = np.where(mask.reshape(mask.shape[0],-1).sum(-1)/float(sz_mask*sz_mask)>=0.8)[0]\n",
    "    assert mask.shape[0]==img.shape[0]\n",
    "    img = img[idxs]\n",
    "    #For 1MPP extraction\n",
    "    temp = []\n",
    "    for i in range(len(img)):\n",
    "        temp.append(rescale(img[i],0.5,channel_axis=-1))\n",
    "    temp = np.stack(temp)\n",
    "    return temp\n",
    "\n",
    "class PandaDataset(Dataset):\n",
    "    def __init__(self, path, test):\n",
    "        self.path = path\n",
    "        self.names = list(pd.read_csv(test).image_id)\n",
    "        self.mean = torch.tensor([0.485, 0.456, 0.406])\n",
    "        self.std = torch.tensor([0.229, 0.224, 0.225])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.names)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        name = self.names[idx]\n",
    "#         img = skimage.io.MultiImage(os.path.join(DATA,name+'.tiff'))[-1]\n",
    "        path = os.path.join(self.path,name+'.tiff')\n",
    "        #Can make it faster at this stage will decide accordingly\n",
    "#         tiles = torch.Tensor(tile(path)/255.0)\n",
    "        tiles = torch.Tensor(tile(path))\n",
    "        tiles = (tiles - self.mean)/self.std\n",
    "        return tiles.permute(0,3,1,2), name\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1124e246",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:14.254624Z",
     "iopub.status.busy": "2024-02-08T14:57:14.254210Z",
     "iopub.status.idle": "2024-02-08T14:57:14.260558Z",
     "shell.execute_reply": "2024-02-08T14:57:14.259593Z"
    },
    "papermill": {
     "duration": 0.019738,
     "end_time": "2024-02-08T14:57:14.263454",
     "exception": false,
     "start_time": "2024-02-08T14:57:14.243716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def inference(tiles):\n",
    "    dataloader = torch.utils.data.DataLoader(tiles, batch_size=256)\n",
    "    all_feats = []\n",
    "    with torch.no_grad():\n",
    "        for data in dataloader:\n",
    "            # print(data)\n",
    "            img = data.cuda()\n",
    "            feats = featextractor(img)\n",
    "            all_feats.extend(feats)\n",
    "#         all_feats = torch.stack(all_feats,dim=0)\n",
    "        logits = mil_model(feats)\n",
    "        pred_sig = torch.sigmoid(logits)    \n",
    "#         predicted = pred_sig.sum(dim=1).cpu()\n",
    "        predicted = pred_sig.sum(dim=1).round().cpu()\n",
    "    return predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4decb586",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-02-08T14:57:14.280371Z",
     "iopub.status.busy": "2024-02-08T14:57:14.280120Z",
     "iopub.status.idle": "2024-02-08T14:57:14.295055Z",
     "shell.execute_reply": "2024-02-08T14:57:14.294362Z"
    },
    "papermill": {
     "duration": 0.024224,
     "end_time": "2024-02-08T14:57:14.296825",
     "exception": false,
     "start_time": "2024-02-08T14:57:14.272601",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sub_df = pd.read_csv(SAMPLE)\n",
    "if os.path.exists(DATA):\n",
    "    ds = PandaDataset(DATA,TEST)\n",
    "    names,preds = [],[]\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for idx in tqdm(range(len(ds))):\n",
    "            name = ds[idx][1]\n",
    "            tiles = ds[idx][0]\n",
    "            prediction = inference(tiles)\n",
    "            names.append(name)\n",
    "            preds.append(prediction)\n",
    "    \n",
    "    preds = np.asarray(torch.cat(preds).numpy(),int)\n",
    "    sub_df = pd.DataFrame({'image_id': names, 'isup_grade': preds})\n",
    "    sub_df.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 1126921,
     "sourceId": 18647,
     "sourceType": "competition"
    },
    {
     "datasetId": 458222,
     "sourceId": 863262,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 625794,
     "sourceId": 1115430,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4257318,
     "sourceId": 7333623,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4257342,
     "sourceId": 7339790,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4265266,
     "sourceId": 7345435,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4257214,
     "sourceId": 7589259,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 22581004,
     "sourceType": "kernelVersion"
    }
   ],
   "isGpuEnabled": true,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 80.238821,
   "end_time": "2024-02-08T14:57:16.565503",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-02-08T14:55:56.326682",
   "version": "2.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}