Stacking&Blending.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dkF-fU21HjPK"
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook\n",
    "import gc\n",
    "import pickle\n",
    "from catboost import CatBoostClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "j541rie2KaXb"
   },
   "source": [
    "## Features from categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IQ8Hd-FDHmJd"
   },
   "outputs": [],
   "source": [
    "cat_cols = ['main_okved', 'region_code']\n",
    "cat_cols = {x: str for x in cat_cols} \n",
    "cat_cols['id'] = str\n",
    "vertices = pd.read_csv('vertices.csv', index_col=0,\n",
    "                       dtype=cat_cols)\n",
    "edges = pd.read_csv('edges.csv', dtype={'id_1': str, 'id_2': str})\n",
    "ids = pd.read_csv('ids.csv', dtype={'id':str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wtdslOEvIuuM"
   },
   "outputs": [],
   "source": [
    "def prepare_okved(x):\n",
    "    l = x.split('.')\n",
    "    if len(l) == 1:\n",
    "        return x + '.-1'\n",
    "    if len(l[1]) == 0:\n",
    "        return x + '-1'\n",
    "    else:\n",
    "        return x\n",
    "\n",
    "vertices.main_okved = vertices.main_okved.apply(prepare_okved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "oNV9cpHAI1rJ"
   },
   "outputs": [],
   "source": [
    "vertices['okved_first'] = vertices.main_okved.apply(lambda x: x.split('.')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vTa_mglpI3U3"
   },
   "outputs": [],
   "source": [
    "vertices['okved_second'] = vertices.main_okved.apply(lambda x: x.split('.')[0] +'.'+ x.split('.')[1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R2YUEL4MI40S"
   },
   "outputs": [],
   "source": [
    "cols = ['region_code', 'company_type']\n",
    "bases = ['main_okved', 'okved_first', 'okved_second']\n",
    "for base in bases:\n",
    "    for col in cols:\n",
    "        vertices[base + '_' + col] = vertices[base] + '_' + vertices[col]\n",
    "        vertices[cols[0] + '_' + cols[1] + '_' + base] = vertices[base] + '_' + vertices[cols[0]] + '_' + vertices[cols[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vV16bFPXI6YJ"
   },
   "outputs": [],
   "source": [
    "vertices['region_code' + '_' + 'company_type'] = vertices[cols[0]] + '_' + vertices[cols[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Tyjza-1SJi3M"
   },
   "outputs": [],
   "source": [
    "vertices.to_csv('new_vertices.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cANE-lR5Kimb"
   },
   "source": [
    "## Combination of SVD matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "falGMbzbJm3t"
   },
   "outputs": [],
   "source": [
    "transformed450 = np.load('transformed450R20_2.npy')\n",
    "components450 = np.load('components450R20_2.npy')\n",
    "transformed400 = np.load('transformed400.npy')\n",
    "transformed400 = np.load('components400.npy')\n",
    "transformed475 = np.load('transformed475R25.npy')\n",
    "components475 = np.load('components475R25.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RDDSGz5vKGj9"
   },
   "outputs": [],
   "source": [
    "def transform_svd_matr(transformed, components):\n",
    "    j = 0\n",
    "    svd = np.zeros((100, len(vertices)))\n",
    "    for id in ids.id:\n",
    "        matr_id = int(id) - 1\n",
    "        svd[j] = transformed[matr_id]@components \n",
    "        j += 1\n",
    "    return svd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "69lKThuIKYPM"
   },
   "outputs": [],
   "source": [
    "svd450 = transform_svd_matr(transformed450, components450)\n",
    "svd475 = transform_svd_matr(transformed475, components475)\n",
    "svd400 = transform_svd_matr(transformed400, components400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lk16X9ndK1om"
   },
   "outputs": [],
   "source": [
    "svd_sum = svd400 + svd450 + svd475\n",
    "np.save('svd_sum.npy', svd_sum)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QYYsV-E7K855"
   },
   "source": [
    "## Mean encoding of categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NfXfah-fLogu"
   },
   "outputs": [],
   "source": [
    "edges['value_on_n_trans'] = edges.value / edges.n_transactions\n",
    "edges_concat = pd.concat([edges, edges[['id_2', 'id_1', 'value', 'n_transactions', 'value_on_n_trans']].rename(\n",
    "    {'id_2': 'id_1', 'id_1': 'id_2'}, axis=1)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dbvWY_L0L4Pu"
   },
   "outputs": [],
   "source": [
    "cat_features = vertices.set_index('id').columns[vertices.set_index('id').dtypes == np.object]\n",
    "merged_data = vertices.rename({'id': 'id_1'}, axis=1).merge(edges_concat, on='id_1')\n",
    "global_means = merged_data[['value', 'n_transactions', 'value_on_n_trans']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bjlXY0oML8in"
   },
   "outputs": [],
   "source": [
    "for feature in cat_features:\n",
    "    if feature == 'okved_info':\n",
    "    continue\n",
    "    nrows = len(merged_data)\n",
    "    global_mean = pd.DataFrame()\n",
    "    tmp_df = merged_data[[feature, 'value', 'n_transactions', 'value_on_n_trans']]\n",
    "    grouped_by =  tmp_df.groupby(feature)\n",
    "    count = grouped_by.agg({'value' : 'count'})\n",
    "    count = count.reset_index().value\n",
    "    grouped_by = grouped_by.agg('mean')\n",
    "    alpha = nrows / (10*merged_data[feature].nunique())\n",
    "    grouped_by['value'] =(grouped_by['value'].values*count.values + global_means['value']*alpha)/(count.values+alpha)\n",
    "    grouped_by['n_transactions']  = (grouped_by['n_transactions'].values*count.values + global_means['n_transactions']*alpha)/(count.values+alpha)\n",
    "    grouped_by['value_on_n_trans'] = (grouped_by['value_on_n_trans'].values*count.values + global_means['value_on_n_trans']*alpha)/(count.values+alpha)\n",
    "\n",
    "    grouped_by = grouped_by.rename(lambda x: feature + '_' + x + '_mean', axis=1)\\\n",
    "                .reset_index()\n",
    "    vertices = vertices.merge(grouped_by, on=feature, how='left')\n",
    "    vertices[feature + '_' + 'value' + '_mean'] = vertices[feature + '_' + 'value' + '_mean'].fillna(global_means['value'])\n",
    "    vertices[feature + '_' + 'n_transactions' + '_mean'] = vertices[feature + '_' + 'n_transactions' + '_mean'].fillna(global_means['n_transactions'])\n",
    "    vertices[feature + '_' + 'value_on_n_trans' + '_mean'] = vertices[feature + '_' + 'value_on_n_trans' + '_mean'].fillna(global_means['value_on_n_trans'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YdXZihxDL_Fn"
   },
   "outputs": [],
   "source": [
    "vertices.to_csv('me_vertices.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VmeRhdpaMJyI"
   },
   "source": [
    "## Adding features from Node2Vec embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_Y5UjzqhMKDD"
   },
   "outputs": [],
   "source": [
    "with open('embs.pickle', 'rb') as f:\n",
    "    node_dict = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WfJkjcKMOEfE"
   },
   "outputs": [],
   "source": [
    "vertices_nodes = np.empty((len(vertices), 128))\n",
    "vertices_nodes[:] = np.nan\n",
    "for id in vertices.id:\n",
    "    matr_id = int(id) - 1 \n",
    "    if id in node_dict:\n",
    "        vertices_nodes[matr_id] = node_dict[id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZhKo1u21OOPt"
   },
   "outputs": [],
   "source": [
    "vertices = pd.concat([vertices, vertices_nodes], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tiXRoPMnOZHr"
   },
   "outputs": [],
   "source": [
    "vertices.to_csv('vertices_nodes.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NHZ5X7KjP4-V"
   },
   "source": [
    "## Training \n",
    "## Grouping variables (light)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qKBnRAsGQZEW"
   },
   "outputs": [],
   "source": [
    "grouped = edges_concat.groupby('id_1').agg({\n",
    "    'id_2': 'count',\n",
    "    'value': ['mean', 'sum'],\n",
    "    'n_transactions':  ['mean', 'sum'],\n",
    "    'value_on_n_trans': ['mean', 'sum']\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AklMb5yDR7cj"
   },
   "outputs": [],
   "source": [
    "grouped.columns = ['g_count', 'g_val_mean', 'g_val_sum', 'g_ntr_mean', 'g_ntr_sum', 'g_vot_mean', 'g_vot_sum']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yuVNTs3rSCNQ"
   },
   "outputs": [],
   "source": [
    "grouped = grouped.reset_index().rename({'id_1': 'id'}, axis=1).set_index('id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "V-NZ3lMvSJdm"
   },
   "source": [
    "Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Adr2M5hGSExQ"
   },
   "outputs": [],
   "source": [
    "vertices = vertices.merge(grouped, how='left', on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9rYoCkFMSGoC"
   },
   "outputs": [],
   "source": [
    "vertices = vertices.set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YZW7Xk9SSISG"
   },
   "outputs": [],
   "source": [
    "cat_features = vertices.columns[vertices.dtypes==np.object]\n",
    "node_features = [col for col in vertices if col.startswith('node_f_')]\n",
    "mn_enc_features = [col for col in vertices if col.endswith('_mean') and not col.startswith('g_')]\n",
    "g_features = [col for col in vertices if col.startswith('g_')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gH4NF8-mSRG6"
   },
   "outputs": [],
   "source": [
    "vertices = vertices.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jhTh2v9CSTCA"
   },
   "outputs": [],
   "source": [
    "def create_probas(vertices, cat_features=None): \n",
    "    j = 0 \n",
    "    probas = np.zeros((100, len(vertices)))\n",
    "    for i in tqdm_notebook(ids.id):\n",
    "\n",
    "        df1 = edges[edges['id_1'] == i].reset_index()\n",
    "        df2 = edges[edges['id_2'] == i].reset_index()\n",
    "\n",
    "        df = df1[['id_2', 'id_1']].rename(columns={'id_1':'id_2', 'id_2':'id_1'}).append(df2[['id_1', 'id_2']])\n",
    "        df['target'] = 1\n",
    "        \n",
    "        df = vertices.set_index('id').join(df.set_index('id_1')['target']).fillna(0)\n",
    "        X = df.drop(['target'], axis=1)\n",
    "        y = df['target']\n",
    "        if cat_features is None:\n",
    "            model = CatBoostClassifier(iterations=100, verbose=False, task_type='GPU',\n",
    "                                    class_weights=[1, (y==0).sum()/(y==1).sum()])\n",
    "        else:\n",
    "            model = CatBoostClassifier(iterations=100, verbose=False, task_type='GPU',\n",
    "                                    class_weights=[1, (y==0).sum()/(y==1).sum()], cat_features=cat_features)          \n",
    "        \n",
    "        model.fit(X, y)\n",
    "        preds = model.predict_proba(X)[:,1]\n",
    "        probas[j] = preds\n",
    "        j += 1\n",
    "    return probas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZNneS0xYSfWt"
   },
   "outputs": [],
   "source": [
    "grouped_probas = create_probas(vertices.set_index('id')[g_features].reset_index())\n",
    "cat_probas = create_probas(vertices.set_index('id')[cat_features].reset_index(), cat_features)\n",
    "node_probas = create_probas(vertices.set_index('id')[node_features].reset_index())\n",
    "mn_enc_probas = create_probas(vertices.set_index('id')[mn_enc_features].reset_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5UibuCNeTdDY"
   },
   "outputs": [],
   "source": [
    "j = 0 \n",
    "result = pd.DataFrame(columns=['id_1', 'id_2', 'proba'])\n",
    "for i in tqdm_notebook(ids.id):\n",
    "  \n",
    "    df1 = edges[edges['id_1'] == i].reset_index()\n",
    "    df2 = edges[edges['id_2'] == i].reset_index()\n",
    "\n",
    "    df = df1[['id_2', 'id_1']].rename(columns={'id_1':'id_2', 'id_2':'id_1'}).append(df2[['id_1', 'id_2']])\n",
    "    df['target'] = 1\n",
    "    \n",
    "    df = vertices.set_index('id').join(df.set_index('id_1')['target']).fillna(0)\n",
    "    X = pd.DataFrame(df.reset_index()['id']).set_index('id')\n",
    "\n",
    "    X['cat_probas'] = cat_probas[j]\n",
    "    X['node_probas'] = node_probas[j]\n",
    "    X['mn_enc_probas'] = mn_enc_probas[j]\n",
    "    X['grouped_probas'] = grouped_probas[j]\n",
    "\n",
    "    y = df['target']\n",
    "    model = CatBoostClassifier(iterations=50, verbose=False, task_type='GPU',\n",
    "                               class_weights=[1, (y==0).sum()/(y==1).sum()],\n",
    "                               )\n",
    "    \n",
    "    model.fit(X, y)\n",
    "\n",
    "    preds = model.predict_proba(X)[:,1]\n",
    "    tmp = pd.DataFrame({\n",
    "        'id_1': [i]*len(preds),\n",
    "        'id_2': vertices.id,\n",
    "        'proba': preds\n",
    "    })\n",
    "\n",
    "    tmp = tmp[df.reset_index().target != 1].sort_values(by='proba', ascending=False)[:800]\n",
    "    result = pd.concat([result, tmp])\n",
    "\n",
    "    j += 1\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ewoOdKwRT2wy"
   },
   "outputs": [],
   "source": [
    "edges_concat['ss'] = edges_concat.id_1 + '_' + edges_concat.id_2\n",
    "\n",
    "idx = result.id_1 < result.id_2\n",
    "result['ss'] = np.zeros(len(result)).astype(str)\n",
    "result.loc[idx, 'ss'] =  result.id_1[idx] + '_' + result.id_2[idx]\n",
    "result.loc[~idx, 'ss'] =  result.id_2[~idx] + '_' + result.id_1[~idx]\n",
    "result = result.drop_duplicates(subset='ss')\n",
    "idx = result['ss'].isin(edges_concat.ss)\n",
    "result = result[~idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "noRlOsGYT-KD"
   },
   "outputs": [],
   "source": [
    "result.to_csv('result_stacking.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D3V3F_-XULcw"
   },
   "source": [
    "## Getting edges from SVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O55V4VKCVDGx"
   },
   "outputs": [],
   "source": [
    "j = 0 \n",
    "result = pd.DataFrame(columns=['id_1', 'id_2', 'proba'])\n",
    "for i in tqdm_notebook(ids.id):\n",
    "    preds = svd_sum[j] \n",
    "\n",
    "    tmp = pd.DataFrame({\n",
    "        'id_1': [i]*len(preds),\n",
    "        'id_2': vertices.id,\n",
    "        'proba': preds\n",
    "    })\n",
    "\n",
    "    tmp = tmp.sort_values(by='proba', ascending=False)[:2300]\n",
    "    result = pd.concat([result, tmp])\n",
    "    j += 1\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "10wiu19UVVYw"
   },
   "outputs": [],
   "source": [
    "idx = result.id_1 < result.id_2\n",
    "result['ss'] = np.zeros(len(result)).astype(str)\n",
    "result.loc[idx, 'ss'] =  result.id_1[idx] + '_' + result.id_2[idx]\n",
    "result.loc[~idx, 'ss'] =  result.id_2[~idx] + '_' + result.id_1[~idx]\n",
    "result = result.drop_duplicates(subset='ss')\n",
    "idx = result['ss'].isin(edges_concat.ss)\n",
    "result = result[~idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zXPAFbUFVY7v"
   },
   "outputs": [],
   "source": [
    "result.to_csv('result_svd.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gGmpK8BLVb_J"
   },
   "source": [
    "## Getting edges for nodes with 0 connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0rF3jCL4VhsD"
   },
   "outputs": [],
   "source": [
    "cat_cols = ['main_okved', 'region_code', 'okved_first', 'okved_second', 'main_okved_region_code',\n",
    "            'okved_first_region_code', 'okved_second_region_code']\n",
    "cat_cols = {x: str for x in cat_cols} \n",
    "cat_cols['id'] = str\n",
    "vertices = pd.read_csv('me_vertices.csv',\n",
    "                          dtype=cat_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9R3-4AkHWaAr"
   },
   "outputs": [],
   "source": [
    "lone_ids = vertices.id[~vertices.id.isin(edges_concat.id_2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qu8cqZCgWlzH"
   },
   "outputs": [],
   "source": [
    "cat_features = vertices.set_index('id').columns[vertices.set_index('id').dtypes == np.object]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5yfEiAhkWfa_"
   },
   "outputs": [],
   "source": [
    "result = pd.DataFrame(columns=['id_1', 'id_2', 'proba'])\n",
    "j = 0\n",
    "\n",
    "for i in tqdm_notebook(ids.id):\n",
    "\n",
    "    df1 = edges[edges['id_1'] == i].reset_index()\n",
    "    df2 = edges[edges['id_2'] == i].reset_index()\n",
    "\n",
    "    df = df1[['id_2', 'id_1']].rename(columns={'id_1':'id_2', 'id_2':'id_1'}).append(df2[['id_1', 'id_2']])\n",
    "    df['target'] = 1\n",
    "    \n",
    "    df = vertices.set_index('id').join(df.set_index('id_1')['target']).fillna(0)\n",
    "    X = df.drop(['target'], axis=1)\n",
    "    y = df['target']\n",
    "    model = CatBoostClassifier(iterations=100, verbose=False, task_type='GPU',\n",
    "                               class_weights=[1, (y==0).sum()/(y==1).sum()], \n",
    "                               cat_features=cat_features)\n",
    "    \n",
    "    model.fit(X, y)\n",
    "    preds = model.predict_proba(X)[:,1]\n",
    "\n",
    "    tmp = pd.DataFrame({\n",
    "        'id_1': [i]*len(preds),\n",
    "        'id_2': vertices.id,\n",
    "        'proba': preds\n",
    "    })\n",
    "    tmp = tmp[tmp.id_2.isin(lone_ids)]\n",
    "    tmp = tmp.sort_values(by='proba', ascending=False)[:3000]\n",
    "    result = pd.concat([result, tmp])\n",
    "    j += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NGraz9XtWsa4"
   },
   "outputs": [],
   "source": [
    "idx = result.id_1 < result.id_2\n",
    "result['ss'] = np.zeros(len(result)).astype(str)\n",
    "result.loc[idx, 'ss'] =  result.id_1[idx] + '_' + result.id_2[idx]\n",
    "result.loc[~idx, 'ss'] =  result.id_2[~idx] + '_' + result.id_1[~idx]\n",
    "result = result.drop_duplicates(subset='ss')\n",
    "idx = result['ss'].isin(edges_concat.ss)\n",
    "result = result[~idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VP1mr6REXBsy"
   },
   "outputs": [],
   "source": [
    "result.to_csv('result_zeros.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eGdR8lfbXE8n"
   },
   "source": [
    "## Grouping features (hard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P_Y7bRaRXt4S"
   },
   "outputs": [],
   "source": [
    "cat_cols = ['main_okved', 'region_code', 'okved_first', 'okved_second', 'main_okved_region_code',\n",
    "            'okved_first_region_code', 'okved_second_region_code']\n",
    "cat_cols = {x: str for x in cat_cols} \n",
    "cat_cols['id'] = str\n",
    "vertices = pd.read_csv('new_vertices.csv',\n",
    "                          dtype=cat_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OpnhQOG0X1Qy"
   },
   "outputs": [],
   "source": [
    "grouped_features = edges_concat.groupby('id_1').agg(\n",
    "    {\n",
    "        'id_2': 'count',\n",
    "        'value': ['sum', 'min', 'max', 'count', 'var'],\n",
    "        'n_transactions': ['sum', 'min', 'max', 'count', 'var'],\n",
    "        'value_on_n_trans': ['sum', 'min', 'max', 'count', 'var']\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yecr6p_HX-Dr"
   },
   "outputs": [],
   "source": [
    "new_columns = []\n",
    "for col in grouped_features.columns:\n",
    "    new_columns.append('g_'+ col[0]+ '_' + col[1])\n",
    "grouped_features.columns = new_columns\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-UMj-inrYBIy"
   },
   "outputs": [],
   "source": [
    "grouped_features = pd.concat([\n",
    "           grouped_features,\n",
    "           grouped_features.apply(lambda x: x**2).rename(lambda x: x + '_sqr', axis=1),\n",
    "            grouped_features.apply(lambda x: x**0.5).rename(lambda x: x + '_sqrt', axis=1),\n",
    "           grouped_features.apply(np.log).rename(lambda x: x + '_ln', axis=1)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TWPyDNltYC8X"
   },
   "outputs": [],
   "source": [
    "cat_features = vertices.set_index('id').columns[vertices.set_index('id').dtypes == np.object]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JzOcwBKDYFJ6"
   },
   "outputs": [],
   "source": [
    "grouped_features = grouped_features.fillna(-10000)\n",
    "grouped_features = grouped_features.reset_index().rename({'id_1': 'id'}, axis=1)\n",
    "vertices = vertices[['id'] + list(cat_features)].merge(grouped_features, on='id', how='left').fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ff1f4W9YYJgy"
   },
   "outputs": [],
   "source": [
    "result = pd.DataFrame(columns=['id_1', 'id_2', 'proba'])\n",
    "\n",
    "for i in tqdm_notebook(ids.id):\n",
    "\n",
    "    df1 = edges[edges['id_1'] == i].reset_index()\n",
    "    df2 = edges[edges['id_2'] == i].reset_index()\n",
    "\n",
    "    df = df1[['id_2', 'id_1']].rename(columns={'id_1':'id_2', 'id_2':'id_1'}).append(df2[['id_1', 'id_2']])\n",
    "    df['target'] = 1\n",
    "    \n",
    "    df = vertices.set_index('id').join(df.set_index('id_1')['target']).fillna(0)\n",
    "    X = df.drop(['target'], axis=1)\n",
    "    y = df['target']\n",
    "    model = CatBoostClassifier(iterations=100, verbose=False, task_type='GPU',\n",
    "                               class_weights=[1, (y==0).sum()/(y==1).sum()], \n",
    "                               cat_features=cat_features)\n",
    "    \n",
    "    model.fit(X, y)\n",
    "    preds = model.predict_proba(X)[:,1]\n",
    "\n",
    "    tmp = pd.DataFrame({\n",
    "        'id_1': [i]*len(preds),\n",
    "        'id_2': vertices.id,\n",
    "        'proba': preds\n",
    "    })\n",
    "    tmp = tmp[df.reset_index().target != 1]\n",
    "    tmp = tmp.sort_values(by='proba', ascending=False)[:3000]\n",
    "    result = pd.concat([result, tmp])\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R-uuYGH8Yboo"
   },
   "outputs": [],
   "source": [
    "idx = result.id_1 < result.id_2\n",
    "result['ss'] = np.zeros(len(result)).astype(str)\n",
    "result.loc[idx, 'ss'] =  result.id_1[idx] + '_' + result.id_2[idx]\n",
    "result.loc[~idx, 'ss'] =  result.id_2[~idx] + '_' + result.id_1[~idx]\n",
    "result = result.drop_duplicates(subset='ss')\n",
    "idx = result['ss'].isin(edges_concat.ss)\n",
    "result = result[~idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O8OGFuLKYd1e"
   },
   "outputs": [],
   "source": [
    "result.to_csv('grouped_result.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CkpMSAcSYhTu"
   },
   "source": [
    "## Blending all the models we got"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BIfkNhK1Yita"
   },
   "outputs": [],
   "source": [
    "result_svd = pd.read_csv('result_svd.csv',  dtype={'id_1': str, 'id_2': str})\n",
    "result_stacking = pd.read_csv('result_cb.csv', dtype={'id_1': str, 'id_2': str})\n",
    "result_zeros = pd.read_csv('result_zeros.csv', dtype={'id_1': str, 'id_2': str})\n",
    "node_result = pd.read_csv('node2vec_result.csv', dtype={'id_1': str, 'id_2': str})\n",
    "grouped_result = pd.read_csv('grouped_result.csv', dtype={'id_1': str, 'id_2': str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OXuI5JzgZDep"
   },
   "outputs": [],
   "source": [
    "idx = node_result.id_1 < node_result.id_2\n",
    "node_result['ss'] = np.zeros(len(node_result)).astype(str)\n",
    "node_result.loc[idx, 'ss'] =  node_result.id_1[idx] + '_' + node_result.id_2[idx]\n",
    "node_result.loc[~idx, 'ss'] =  node_result.id_2[~idx] + '_' + node_result.id_1[~idx]\n",
    "node_result = node_result.drop_duplicates(subset='ss')\n",
    "idx = node_result['ss'].isin(edges_concat.ss)\n",
    "node_result = node_result[~idx]\n",
    "node_result = node_result.rename({'preds': 'proba'}, axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PvwON4wxZEkp"
   },
   "outputs": [],
   "source": [
    "node_result = node_result.sort_values(by='proba', ascending=False).head(150000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KgzkYS0RZjVe"
   },
   "outputs": [],
   "source": [
    "submit = pd.concat([\n",
    "    grouped_result[(grouped_result.ss.isin(result_stacking.ss)) | (grouped_result.ss.isin(result_svd.ss)) | (grouped_result.ss.isin(node_result.ss))],\n",
    "    result_svd[((result_svd.ss.isin(node_result.ss)) | (result_svd.ss.isin(result_stacking.ss))) & (~result_svd.ss.isin(grouped_result.ss))],\n",
    "    node_result[(node_result.ss.isin(result_stacking.ss)) & (~node_result.ss.isin(result_svd.ss)) & (~node_result.ss.isin(grouped_result.ss))],\n",
    "    result_zeros.sort_values(by ='proba', ascending=False)[:35000],\n",
    "    result_svd[~((result_svd.ss.isin(node_result.ss)) | (result_svd.ss.isin(result_stacking.ss))) | (~result_svd.ss.isin(grouped_result.ss))][:100000]\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "V0dAl9M5Zsfd"
   },
   "outputs": [],
   "source": [
    "subm = submit.reset_index().drop('index', axis=1).reset_index().sort_values(by='index').drop_duplicates('ss')[:100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uewUi7rPZvX6"
   },
   "outputs": [],
   "source": [
    "subm = subm[['id_1', 'id_2']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SKXGmmYbZw5-"
   },
   "outputs": [],
   "source": [
    "subm.to_csv('submit.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Untitled43.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}