examples/roux_stat_corr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "68803c02-8d28-4e0c-a1bd-4a228a64f618",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "<!-- <a href=\"https://github.com/rraadd88/roux/blob/master/examples/roux_stat_corr.ipynb\"><img align=\"right\" style=\"float:right;\" src=\"https://img.shields.io/badge/-source-cccccc?style=flat-square\"></a>\n",
    " -->\n",
    " \n",
    "## ✨ Correlations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df3054e2-adc5-4a66-bb17-92706b67d386",
   "metadata": {},
   "outputs": [],
   "source": [
    "# install extra requirements\n",
    "# !pip install roux[stat]\n",
    "# loading non-roux requirements\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcc3debc-f2ec-434d-acbd-56d5b5fd935a",
   "metadata": {},
   "source": [
    "**Demo data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4f9bba68-640b-49e5-bc25-baedb5fd9a09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bTM23LRWf5</th>\n",
       "      <td>1.449605</td>\n",
       "      <td>-0.010841</td>\n",
       "      <td>0.039636</td>\n",
       "      <td>0.250015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   A         B         C         D\n",
       "bTM23LRWf5  1.449605 -0.010841  0.039636  0.250015"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## random state\n",
    "# import numpy as np\n",
    "# np.random.RandomState(1)\n",
    "## demo dataframe\n",
    "# data=pd._testing.makeDataFrame()\n",
    "data=pd.DataFrame({'A': {'bTM23LRWf5': 1.44960517798113,\n",
    "  'j4Js0U8axL': -1.9621997327822367,\n",
    "  'pBrcUD42hv': 1.0861341452827697,\n",
    "  '60qMz8HhQT': 0.8055880798845501,\n",
    "  'HWaui22wMu': 0.4127760107922468,\n",
    "  'duxjOqPcxW': 0.11972130630756372,\n",
    "  'SEWVZ8T1mL': -1.667361366356503,\n",
    "  'WdrUUHvmd0': 0.4024414760941458,\n",
    "  'fHIfH7QwOy': -0.23384501526196033,\n",
    "  'oVGnp5WsY0': -0.16334255905051992,\n",
    "  'RdCYxIpAqp': -1.239237691200248,\n",
    "  'ZiWWPhepgR': 1.178638443002846,\n",
    "  '896LAcJYc5': -1.449051257146253,\n",
    "  'Fe2rVyFVFj': 1.1303757186983767,\n",
    "  'GmnxQ3B6hh': -2.143884671873999,\n",
    "  '1uX3BBPoql': 1.665153022325663,\n",
    "  'CPSjRvsbfv': -1.4287155823331705,\n",
    "  'fyEnMyVmHD': 1.1243328246312276,\n",
    "  '3Ggyl7iaUn': 2.2242076462149503,\n",
    "  '4GXsPSAaMm': -0.3241986161565716,\n",
    "  'biMwNIiGQk': 1.2462390801714365,\n",
    "  'n3SVYsXCyc': -0.4577481959565738,\n",
    "  'coV6fFO4zy': -2.6735235557610793,\n",
    "  '6vyhq2AreT': 1.0929729611561816,\n",
    "  '9Lf0uHK944': -0.8396199455656115,\n",
    "  'roh1hlJMmh': 1.4536502690084832,\n",
    "  'yQOwSAJBOj': -1.8187037898762812,\n",
    "  'L3sZRyZT6G': -0.30712442566269627,\n",
    "  '8SAjYvRqft': 0.078485814903189,\n",
    "  'jrBDr6ZV5j': 1.1328848329671382},\n",
    " 'B': {'bTM23LRWf5': -0.010841346360700922,\n",
    "  'j4Js0U8axL': 0.4153633268193885,\n",
    "  'pBrcUD42hv': 1.4578106558684567,\n",
    "  '60qMz8HhQT': -0.7586700866021712,\n",
    "  'HWaui22wMu': -2.442313049287723,\n",
    "  'duxjOqPcxW': 0.3537271104100038,\n",
    "  'SEWVZ8T1mL': -0.2505061201285424,\n",
    "  'WdrUUHvmd0': -0.24734077602893173,\n",
    "  'fHIfH7QwOy': 1.7399536849274078,\n",
    "  'oVGnp5WsY0': -0.2014258994998956,\n",
    "  'RdCYxIpAqp': -0.20842551620405722,\n",
    "  'ZiWWPhepgR': -0.8970602676058785,\n",
    "  '896LAcJYc5': 0.25722083013696556,\n",
    "  'Fe2rVyFVFj': -0.7202495482075049,\n",
    "  'GmnxQ3B6hh': 1.2152059064150982,\n",
    "  '1uX3BBPoql': 0.8117965902200238,\n",
    "  'CPSjRvsbfv': -1.1734071185543102,\n",
    "  'fyEnMyVmHD': 0.5635885805680235,\n",
    "  '3Ggyl7iaUn': 2.200009157193734,\n",
    "  '4GXsPSAaMm': 0.37799408048066235,\n",
    "  'biMwNIiGQk': 1.1280216565088532,\n",
    "  'n3SVYsXCyc': 2.359298591630858,\n",
    "  'coV6fFO4zy': 1.2752758235750379,\n",
    "  '6vyhq2AreT': 0.6654407061651554,\n",
    "  '9Lf0uHK944': -0.9492968533118858,\n",
    "  'roh1hlJMmh': 0.5973674565497872,\n",
    "  'yQOwSAJBOj': -0.5539614824008819,\n",
    "  'L3sZRyZT6G': -1.5933015583484251,\n",
    "  '8SAjYvRqft': 0.40173602902266825,\n",
    "  'jrBDr6ZV5j': 0.48463225714525776},\n",
    " 'C': {'bTM23LRWf5': 0.039635552886720764,\n",
    "  'j4Js0U8axL': -0.3254261032440854,\n",
    "  'pBrcUD42hv': 0.6930254864807848,\n",
    "  '60qMz8HhQT': 0.3308706301857392,\n",
    "  'HWaui22wMu': -0.9589270472782682,\n",
    "  'duxjOqPcxW': 0.42079551569096973,\n",
    "  'SEWVZ8T1mL': -0.06338571562392002,\n",
    "  'WdrUUHvmd0': 0.19231063399322657,\n",
    "  'fHIfH7QwOy': 0.5506782297448156,\n",
    "  'oVGnp5WsY0': -0.563853422883703,\n",
    "  'RdCYxIpAqp': -1.3038730179167821,\n",
    "  'ZiWWPhepgR': 0.7131915559070086,\n",
    "  '896LAcJYc5': -1.6398502271778639,\n",
    "  'Fe2rVyFVFj': -1.0194896922917631,\n",
    "  'GmnxQ3B6hh': 0.41972004370942434,\n",
    "  '1uX3BBPoql': 1.4108426835801315,\n",
    "  'CPSjRvsbfv': -1.6492900622616429,\n",
    "  'fyEnMyVmHD': -0.8014060673808331,\n",
    "  '3Ggyl7iaUn': 0.4117882700488701,\n",
    "  '4GXsPSAaMm': -0.9700690808188418,\n",
    "  'biMwNIiGQk': -1.3683123780635702,\n",
    "  'n3SVYsXCyc': -2.1453642648174203,\n",
    "  'coV6fFO4zy': 0.0829621450960061,\n",
    "  '6vyhq2AreT': -0.9821964202524643,\n",
    "  '9Lf0uHK944': -0.6442148004592266,\n",
    "  'roh1hlJMmh': 0.1156286611379154,\n",
    "  'yQOwSAJBOj': 0.36638443006849636,\n",
    "  'L3sZRyZT6G': -0.9948999397679968,\n",
    "  '8SAjYvRqft': -0.041433546565667966,\n",
    "  'jrBDr6ZV5j': 0.7942857916593715},\n",
    " 'D': {'bTM23LRWf5': 0.2500146347850516,\n",
    "  'j4Js0U8axL': -0.10857951775290281,\n",
    "  'pBrcUD42hv': 0.29937549222146165,\n",
    "  '60qMz8HhQT': -1.7441469956896005,\n",
    "  'HWaui22wMu': 1.3928603482500568,\n",
    "  'duxjOqPcxW': 1.3284947225698323,\n",
    "  'SEWVZ8T1mL': 0.7044711130511356,\n",
    "  'WdrUUHvmd0': 0.741386000194462,\n",
    "  'fHIfH7QwOy': 0.24174461935215943,\n",
    "  'oVGnp5WsY0': 0.2268933233990426,\n",
    "  'RdCYxIpAqp': -0.04478438246292375,\n",
    "  'ZiWWPhepgR': -0.0019278321558485734,\n",
    "  '896LAcJYc5': 0.8435815329630647,\n",
    "  'Fe2rVyFVFj': 0.7806074737674725,\n",
    "  'GmnxQ3B6hh': -1.561445895742816,\n",
    "  '1uX3BBPoql': -0.6159690165929371,\n",
    "  'CPSjRvsbfv': -0.2790014784334211,\n",
    "  'fyEnMyVmHD': -0.47251385723791145,\n",
    "  '3Ggyl7iaUn': -1.2346734261225418,\n",
    "  '4GXsPSAaMm': -0.3423494884311885,\n",
    "  'biMwNIiGQk': 0.012027664114381066,\n",
    "  'n3SVYsXCyc': 1.130860104214341,\n",
    "  'coV6fFO4zy': -0.6515128297731695,\n",
    "  '6vyhq2AreT': -0.3809282418023091,\n",
    "  '9Lf0uHK944': 0.8850202033247277,\n",
    "  'roh1hlJMmh': -1.8162223085809335,\n",
    "  'yQOwSAJBOj': -1.018275515809312,\n",
    "  'L3sZRyZT6G': -0.6983710482086773,\n",
    "  '8SAjYvRqft': 0.9716971763142563,\n",
    "  'jrBDr6ZV5j': -0.9185297157826621}})\n",
    "data.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c37151f6-03d9-4bff-85d0-cdd75cf39304",
   "metadata": {},
   "source": [
    "## Applicable methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88a4c643-22c2-48b4-b111-af6df3a696b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "methods=[\n",
    "## correlations\n",
    " 'pearson',\n",
    " 'spearman',\n",
    "## distances\n",
    " 'braycurtis',\n",
    " 'canberra',\n",
    " 'chebyshev',\n",
    " 'cityblock',\n",
    " 'correlation',\n",
    " 'cosine',\n",
    " 'dice',\n",
    " 'euclidean',\n",
    " 'hamming',\n",
    " 'jaccard',\n",
    " 'jensenshannon',\n",
    " 'kulsinski',\n",
    " 'matching',\n",
    " 'minkowski',\n",
    " 'rogerstanimoto',\n",
    " 'russellrao',\n",
    " 'sokalmichener',\n",
    " 'sokalsneath',\n",
    " 'sqeuclidean',\n",
    " 'yule'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f861f4fa-8329-4f97-9c69-de0cced904a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "## import the roux function\n",
    "from roux.stat.corr import get_corr\n",
    "\n",
    "correlations=[]\n",
    "for method in methods:\n",
    "    try:\n",
    "        correlations.append(get_corr('A','B',df=data,method=method,resample=True))\n",
    "    except:\n",
    "        print(method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ac9fda39-fc57-44c2-88a0-19a8fb594895",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rr</th>\n",
       "      <th>ci</th>\n",
       "      <th>ci_type</th>\n",
       "      <th>n</th>\n",
       "      <th>method</th>\n",
       "      <th>r</th>\n",
       "      <th>P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.102380</td>\n",
       "      <td>0.160545</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>pearson</td>\n",
       "      <td>0.104575</td>\n",
       "      <td>0.582349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.144696</td>\n",
       "      <td>0.143826</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.151947</td>\n",
       "      <td>0.422812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.827325</td>\n",
       "      <td>0.122815</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>braycurtis</td>\n",
       "      <td>0.822641</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15.225082</td>\n",
       "      <td>0.498441</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>canberra</td>\n",
       "      <td>19.031352</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.830858</td>\n",
       "      <td>0.471767</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>chebyshev</td>\n",
       "      <td>3.948799</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>29.727868</td>\n",
       "      <td>2.738131</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cityblock</td>\n",
       "      <td>37.159836</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.897620</td>\n",
       "      <td>0.160545</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>correlation</td>\n",
       "      <td>0.895425</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.898417</td>\n",
       "      <td>0.178304</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>0.897893</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-1.377530</td>\n",
       "      <td>2.214833</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>dice</td>\n",
       "      <td>-0.404251</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7.870261</td>\n",
       "      <td>0.816197</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>euclidean</td>\n",
       "      <td>8.817350</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>hamming</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>jaccard</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>inf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>jensenshannon</td>\n",
       "      <td>inf</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.797838</td>\n",
       "      <td>0.293023</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>kulsinski</td>\n",
       "      <td>0.841881</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>matching</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>7.870261</td>\n",
       "      <td>0.816197</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>minkowski</td>\n",
       "      <td>8.817350</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.333273</td>\n",
       "      <td>0.902209</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>rogerstanimoto</td>\n",
       "      <td>-0.182075</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.855074</td>\n",
       "      <td>0.249144</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>russellrao</td>\n",
       "      <td>0.855074</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-0.333273</td>\n",
       "      <td>0.902209</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>sokalmichener</td>\n",
       "      <td>-0.182075</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.929161</td>\n",
       "      <td>1.931385</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>sokalsneath</td>\n",
       "      <td>7.600368</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>62.196529</td>\n",
       "      <td>12.436707</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>sqeuclidean</td>\n",
       "      <td>77.745662</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>-0.358391</td>\n",
       "      <td>1.987379</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>yule</td>\n",
       "      <td>-0.152695</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rr         ci ci_type   n          method          r         P\n",
       "0    0.102380   0.160545     max  30         pearson   0.104575  0.582349\n",
       "1    0.144696   0.143826     max  30        spearman   0.151947  0.422812\n",
       "2    0.827325   0.122815     max  30      braycurtis   0.822641       NaN\n",
       "3   15.225082   0.498441     max  30        canberra  19.031352       NaN\n",
       "4    3.830858   0.471767     max  30       chebyshev   3.948799       NaN\n",
       "5   29.727868   2.738131     max  30       cityblock  37.159836       NaN\n",
       "6    0.897620   0.160545     max  30     correlation   0.895425       NaN\n",
       "7    0.898417   0.178304     max  30          cosine   0.897893       NaN\n",
       "8   -1.377530   2.214833     max  30            dice  -0.404251       NaN\n",
       "9    7.870261   0.816197     max  30       euclidean   8.817350       NaN\n",
       "10   1.000000   0.000000     max  30         hamming   1.000000       NaN\n",
       "11   1.000000   0.000000     max  30         jaccard   1.000000       NaN\n",
       "12        inf        NaN     max  30   jensenshannon        inf       NaN\n",
       "13   0.797838   0.293023     max  30       kulsinski   0.841881       NaN\n",
       "14   1.000000   0.000000     max  30        matching   1.000000       NaN\n",
       "15   7.870261   0.816197     max  30       minkowski   8.817350       NaN\n",
       "16  -0.333273   0.902209     max  30  rogerstanimoto  -0.182075       NaN\n",
       "17   0.855074   0.249144     max  30      russellrao   0.855074       NaN\n",
       "18  -0.333273   0.902209     max  30   sokalmichener  -0.182075       NaN\n",
       "19   0.929161   1.931385     max  30     sokalsneath   7.600368       NaN\n",
       "20  62.196529  12.436707     max  30     sqeuclidean  77.745662       NaN\n",
       "21  -0.358391   1.987379     max  30            yule  -0.152695       NaN"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(correlations)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80cd0d27-3db1-42f1-8c2c-0496025a83a0",
   "metadata": {},
   "source": [
    "## Pairwise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fabf9675-6b67-4fc5-a061-1fa75cd8dbd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "## import the roux function\n",
    "from roux.stat.corr import get_corrs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5dcffd1b-cf11-43c0-97fa-258292249f30",
   "metadata": {},
   "source": [
    "### All columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fc21c3ed-6eea-4ae1-92ba-c50e29e9df2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable1</th>\n",
       "      <th>variable2</th>\n",
       "      <th>r</th>\n",
       "      <th>P</th>\n",
       "      <th>n</th>\n",
       "      <th>method</th>\n",
       "      <th>Q</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>0.291212</td>\n",
       "      <td>0.118448</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.355345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C</td>\n",
       "      <td>D</td>\n",
       "      <td>-0.300556</td>\n",
       "      <td>0.106565</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.355345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C</td>\n",
       "      <td>B</td>\n",
       "      <td>0.249388</td>\n",
       "      <td>0.183839</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.367678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>D</td>\n",
       "      <td>B</td>\n",
       "      <td>-0.192881</td>\n",
       "      <td>0.307163</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.460745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>0.151947</td>\n",
       "      <td>0.422812</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.507374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>D</td>\n",
       "      <td>-0.104783</td>\n",
       "      <td>0.581596</td>\n",
       "      <td>30</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.581596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  variable1 variable2         r         P   n    method         Q\n",
       "0         A         C  0.291212  0.118448  30  spearman  0.355345\n",
       "3         C         D -0.300556  0.106565  30  spearman  0.355345\n",
       "4         C         B  0.249388  0.183839  30  spearman  0.367678\n",
       "5         D         B -0.192881  0.307163  30  spearman  0.460745\n",
       "2         A         B  0.151947  0.422812  30  spearman  0.507374\n",
       "1         A         D -0.104783  0.581596  30  spearman  0.581596"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "out=get_corrs(\n",
    "    data=data,\n",
    "    method='spearman',\n",
    ")\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b04179be-c9e2-4c32-98dc-684c5828a83d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for testing\n",
    "assert sorted(out['r'].round(3).tolist())==sorted([0.291, -0.301, 0.249, -0.193, 0.152, -0.105]), sorted(out['r'].round(3).tolist())\n",
    "assert sorted(out['Q'].round(3).tolist())==sorted([0.355, 0.355, 0.368, 0.461, 0.507, 0.582]), sorted(out['Q'].round(3).tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ba77b23-3dd9-4df4-bcf8-878c2246fe3d",
   "metadata": {},
   "source": [
    "### Subset of columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "04da385f-5ac8-4813-96e2-bb016a0db01a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable1</th>\n",
       "      <th>variable2</th>\n",
       "      <th>r</th>\n",
       "      <th>P</th>\n",
       "      <th>n</th>\n",
       "      <th>method</th>\n",
       "      <th>Q</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>0.104575</td>\n",
       "      <td>0.582349</td>\n",
       "      <td>30</td>\n",
       "      <td>pearson</td>\n",
       "      <td>0.582349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  variable1 variable2         r         P   n   method         Q\n",
       "0         A         B  0.104575  0.582349  30  pearson  0.582349"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_corrs(\n",
    "    data=data,\n",
    "    method='pearson',\n",
    "    cols=['A','B'],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "408b83ac-3112-4c31-b361-cb2c87b5fc2a",
   "metadata": {},
   "source": [
    "### Subset of columns with other subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b7bbf95c-5571-4836-a3bf-91a390681a8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable1</th>\n",
       "      <th>variable2</th>\n",
       "      <th>r</th>\n",
       "      <th>n</th>\n",
       "      <th>method</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>0.757831</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>D</td>\n",
       "      <td>1.085174</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B</td>\n",
       "      <td>C</td>\n",
       "      <td>0.897184</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B</td>\n",
       "      <td>D</td>\n",
       "      <td>1.187511</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  variable1 variable2         r   n  method\n",
       "0         A         C  0.757831  30  cosine\n",
       "1         A         D  1.085174  30  cosine\n",
       "2         B         C  0.897184  30  cosine\n",
       "3         B         D  1.187511  30  cosine"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "get_corrs(\n",
    "    data=data,\n",
    "    method='cosine',\n",
    "    cols=['A','B'],\n",
    "    cols_with=['C','D'],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f3a964c-d93e-41df-8dbe-131c24a97015",
   "metadata": {},
   "source": [
    "### Resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e154daf6-45b1-4608-b443-23947c53c206",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable1</th>\n",
       "      <th>variable2</th>\n",
       "      <th>rr</th>\n",
       "      <th>ci</th>\n",
       "      <th>ci_type</th>\n",
       "      <th>n</th>\n",
       "      <th>method</th>\n",
       "      <th>r</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>0.754690</td>\n",
       "      <td>0.171653</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>0.757831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>D</td>\n",
       "      <td>1.084900</td>\n",
       "      <td>0.168700</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>1.085174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>0.898417</td>\n",
       "      <td>0.178304</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>0.897893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C</td>\n",
       "      <td>D</td>\n",
       "      <td>1.306537</td>\n",
       "      <td>0.051698</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>1.305269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C</td>\n",
       "      <td>B</td>\n",
       "      <td>0.883498</td>\n",
       "      <td>0.369841</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>0.897184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>D</td>\n",
       "      <td>B</td>\n",
       "      <td>1.181568</td>\n",
       "      <td>0.128890</td>\n",
       "      <td>max</td>\n",
       "      <td>30</td>\n",
       "      <td>cosine</td>\n",
       "      <td>1.187511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  variable1 variable2        rr        ci ci_type   n  method         r\n",
       "0         A         C  0.754690  0.171653     max  30  cosine  0.757831\n",
       "1         A         D  1.084900  0.168700     max  30  cosine  1.085174\n",
       "2         A         B  0.898417  0.178304     max  30  cosine  0.897893\n",
       "3         C         D  1.306537  0.051698     max  30  cosine  1.305269\n",
       "4         C         B  0.883498  0.369841     max  30  cosine  0.897184\n",
       "5         D         B  1.181568  0.128890     max  30  cosine  1.187511"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "out=get_corrs(\n",
    "    data=data,\n",
    "    method='cosine',\n",
    "    resample=True,\n",
    ")\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "11e611c4-4acd-4ccb-93bc-4ea0c75edce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# for testing\n",
    "assert sorted(out['rr'].round(3).tolist())==sorted([0.755, 1.085, 0.898, 1.307, 0.883, 1.182]), sorted(out['rr'].round(3).tolist())\n",
    "assert sorted(out['ci'].round(3).tolist())==sorted([0.172, 0.169, 0.178, 0.052, 0.37, 0.129]), sorted(out['ci'].round(3).tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef9fe4a-0a24-4d3e-bd51-e58a614bcf9b",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feddf0aa-66c1-4d6f-835d-26124eb91f7c",
   "metadata": {},
   "source": [
    "### Drop over-represented values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1b9c3bb5-270c-4bc1-a21f-a2475a8decbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bTM23LRWf5</th>\n",
       "      <td>1.449605</td>\n",
       "      <td>-0.010841</td>\n",
       "      <td>0.039636</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   A         B         C    D\n",
       "bTM23LRWf5  1.449605 -0.010841  0.039636  0.0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApsAAACPCAYAAAC1ShDAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUkUlEQVR4nO3df2zU9R3H8de10Gs72zoUChd+FcFMw6wJWCb4h10qHROREBksbqudsoaUZaSLBpZhHW5hgc0xSSeyKLhxiFkCDMUfNK1KFlAnhj9gY2lJnazQAyH2F6492u/+cD289ii98v30++Oej+Sb9r7f6/f7/t73dd++7+5zdwHLsiwBAAAABqQ5XQAAAAD8i2YTAAAAxtBsAgAAwBiaTQAAABhDswkAAABjaDYBAABgDM0mAAAAjKHZBAAAgDE0mwAAADCGZhMAAADG0Gy6yB/+8AcFAgHNmTPH6VKQwnbs2KFAIBA3jRs3TsXFxXrjjTecLg8p7NSpU6qoqNC0adOUmZmp3NxczZs3T7///e/1+eefO10eUkD/82NmZqZCoZBKS0v17LPPqr293ekSXWmU0wXginA4rKlTp+qDDz5QY2Ojpk+f7nRJSGHr169XQUGBLMtSJBLRjh079O1vf1uvvvqqFi5c6HR5SDEHDhzQ0qVLFQwG9YMf/EAzZ85Ud3e3/va3v+nxxx/XiRMntG3bNqfLRIroOz9Go1G1tLTonXfe0erVq/XMM89o//79uuOOO5wu0VVoNl2iqalJhw8f1p49e1RRUaFwOKzq6mqny0IKW7BggWbPnh27/Oijjyo/P18vv/wyzSZGVFNTk5YvX64pU6aovr5eEyZMiC2rrKxUY2OjDhw44GCFSDX9z49r165VfX29Fi5cqEWLFumf//ynsrKyHKzQXXgZ3SXC4bC++tWv6v7779dDDz2kcDjsdElAnBtvvFFZWVkaNYrHqBhZGzduVEdHh1544YW4RrPP9OnT9ZOf/MSByoArvvnNb2rdunX697//rZ07dzpdjqvQbLpEOBzWkiVLlJGRoe9+97tqaGjQ3//+d6fLQgprbW3Vp59+qvPnz+vEiRNauXKlOjo69L3vfc/p0pBiXn31VU2bNk1z5851uhRgUN///vclSQcPHnS4EnfhKQoXOHr0qE6ePKktW7ZIku655x5NnDhR4XBYd911l8PVIVWVlJTEXQ4Gg3rxxRd13333OVQRUlFbW5uam5v14IMPOl0KcE0TJ05UXl6eTp065XQprkKz6QLhcFj5+fkqLi6WJAUCAS1btkw7d+7Ub3/7W6WnpztcIVJRTU2Nbr31VklSJBLRzp079dhjjyknJ0dLlixxuDqkira2NklSTk6Ow5UAQ3PDDTfwrvR+eBndYT09Pdq9e7eKi4vV1NSkxsZGNTY2as6cOYpEIqqrq3O6RKSooqIilZSUqKSkRA8//LAOHDig22+/XatWrVJ3d7fT5SFF5ObmShL/vOEZHR0dPDjqh2bTYfX19Tp79qx2796tGTNmxKbvfOc7ksQbheAaaWlpKi4u1tmzZ9XQ0OB0OUgRubm5CoVCOn78uNOlANf0n//8R62trXx0YT+8jO6wcDiscePGqaamZsCyPXv2aO/evdq6dSsfoQBXuHz5sqQvHrkDI2XhwoXatm2bjhw5orvvvtvpcoCr+vOf/yxJKi0tdbgSd6HZdNDnn3+uPXv2aOnSpXrooYcGLA+FQnr55Ze1f/9+LVu2zIEKgSui0agOHjyojIwM3XbbbU6XgxTyxBNPKBwO67HHHlN9fb3y8/Pjlp86dUqvvfYaH38ER9XX1+vpp59WQUGBHn74YafLcRWaTQft379f7e3tWrRoUcLl3/jGNzR27FiFw2GaTYy4N954QydPnpQknTt3Trt27VJDQ4PWrFkTG0cHjIRbbrlFu3bt0rJly3TbbbfFfYPQ4cOH9Ze//EWPPPKI02UihfSdHy9fvqxIJKL6+nrV1tZqypQp2r9/vzIzM50u0VUClmVZTheRqhYtWqTa2lpduHBB2dnZCa9TXl6ucDiss2fP6qabbhrhCpGKduzYofLy8rh5mZmZ+trXvqaKigpVVFQoEAg4VB1SWUNDgzZt2qTa2lqdOXNGwWBQd9xxh5YvX64VK1YoGAw6XSJ8rv/5MSMjQ2PGjNHXv/51LVy4UOXl5bw5KAGaTQAAABjDu9EBAABgDM0mAAAAjKHZBAAAgDE0mwAAADCGZhMAAADG0GwCAADAGNd9qHtvb6/OnDmjnJwcPssvxViWpfb2doVCIaWluedxEJlMTeQRbkMm4SbJ5NF1zeaZM2c0adIkp8uAg06fPq2JEyc6XUYMmUxt5BFuQybhJkPJo+uazb5P3j99+rSRr8Tr+37n+fPna/To0bav3wl+2ae2tjZNmjTJdd++YEcm/XCM/LAP0tD3w895NMHr+fBC/WTy2tx2HP1cTzJ5dF2z2fcUfG5urrFmMzs7W7m5ua448Hbw2z657WUYOzLph2Pkh32Qkt8PP+bRBK/nw0v1k8mrc9txTIV6hpJH9wz6AAAAgO+47plNJGfqmgMKplvaWCTNfOotdfV88Qjj41/f73Bl8Iupaw5I0oCckTF4WV+u+/TlG7AL/5+v4JlNAAAAGEOzCQAAAGNoNgEAAGAMzSYAAACModkEAACAMTSbAAAAMIZmEwAAAMbQbAIAAMAYmk0AAAAYQ7MJAAAAY2g2AQAAYAzNJgAAAIyh2QQAAIAxNJsAAAAwhmYTAAAAxtBsAgAAwBiaTQAAABhDswkAAABjaDYBAABgDM0mAAAAjKHZBAAAgDE0mwAAADAm6Wbz0KFDeuCBBxQKhRQIBLRv37645ZZl6cknn9SECROUlZWlkpISNTQ02FUvEIc8wm3IJADES7rZ7OzsVGFhoWpqahIu37hxo5599llt3bpV77//vr7yla+otLRU//3vf6+7WKA/8gi3IZMAEG9Usn+wYMECLViwIOEyy7K0efNm/fznP9eDDz4oSfrTn/6k/Px87du3T8uXL7++aoF+yCPchkwCQLykm83BNDU1qaWlRSUlJbF5eXl5mjNnjo4cOZLwRNrV1aWurq7Y5ba2NklSNBpVNBq1s7zYer/80+uC6ZaCadYXv///p+TN/bO75uHkUTKTSS/nLpgen6++n17cF2nox8LE/nnhHDlcXst4X65jlz2Q6+HUdujQIW3atElHjx7V2bNntXfvXi1evDi23LIsVVdX649//KM+++wzzZs3T88995xmzJhhY+VIdbY2my0tLZKk/Pz8uPn5+fmxZf1t2LBBv/jFLwbMP3jwoLKzs+0sL05tba2xdY+kjUVXfn96dm/s99dff92Baq7PpUuXbF3fcPIomc2kF3P35YxJV3LmxYx92bWOhd15lLx1jhwur2S8f677uLn+4WSyb1jHD3/4Qy1ZsmTA8r5hHS+99JIKCgq0bt06lZaW6h//+IcyMzPtKBuwt9kcjrVr16qqqip2ua2tTZMmTdL8+fOVm5tr+/ai0ahqa2t13333afTo0bavf6TNfOotBdMsPT27V+s+TFNXb0CSdPypUocrS17fMzZOM5FJL+du5lNvSdKAnHkxY9LQj4Wf82iC1zLel+s+ffl2c/3DySTDOuAGtjab48ePlyRFIhFNmDAhNj8SiejOO+9M+DfBYFDBYHDA/NGjRxu9w5te/0jp6glc+b03ELvsxX2zu+bh5FEym0kv5u7LGZOu5Mxr+9HftY6Fif3z0jlyuNxaV3/9c93HzfXbXZebhhrZxU3DOdw4zM3O2yeZddjabBYUFGj8+PGqq6uLnTjb2tr0/vvva+XKlXZuCrgm8gi3IZNwEzcONbKLG4ZDuHmYmx23TzLDOpJuNjs6OtTY2Bi73NTUpGPHjmnMmDGaPHmyVq9erV/+8peaMWNGbPxHKBSKG5AM2IU8wm3IJPzOzUM73DScw43D3Oy8fZIZ1pF0s/nhhx+quLg4drkvcGVlZdqxY4eeeOIJdXZ26kc/+pE+++wz3XPPPXrzzTcZaAwjyCPchkzCK9w41MgubqjFzcPc7BoWNlRJN5v33nuvLMu66vJAIKD169dr/fr1ya4aSBp5hNuQSXgFwzowUhx/NzoAADCDYR1wA5pNAAB8imEdcAOaTQAAfIphHXCDNKcLAAAAgH/RbAIAAMAYmk0AAAAYQ7MJAAAAY2g2AQAAYAzNJgAAAIyh2QQAAIAxNJsAAAAwhmYTAAAAxtBsAgAAwBiaTQAAABhDswkAAABjaDYBAABgDM0mAAAAjKHZBAAAgDE0mwAAADCGZhMAAADG0GwCAADAmFFOFwDAX6auOTBg3se/vt+BSgAAbsAzmwAAADCGZhMAAADG0GwCAADAGJpNAAAAGMMbhJC0RG8AkXgTCABInCOB/nhmEwAAAMbQbAIAAMAYmk0AAAAYw5hNF7raeB8vYuySd/gpd8BwzXzqLXX1BOLmmTxfcY5EKqDZBAAAKSmZB9mp8gDAxAMgXkYHAACAMTSbAAAAMIZmEwAAAMYYG7NZU1OjTZs2qaWlRYWFhdqyZYuKioqua53JjiNIdP2Gp+cnte5kpMp4Di8ykUcpcW6SzUEyubbjPjDY9UeaE/e7mU+9pY1FA98IMtK3iVvPkcPNbzDdirtd7bg9Td437KjDT7x4jkzE5P94uzKWiuNEjTyz+corr6iqqkrV1dX66KOPVFhYqNLSUp07d87E5oBBkUe4DZmEm5BHmGak2XzmmWe0YsUKlZeX6/bbb9fWrVuVnZ2tF1980cTmgEGRR7gNmYSbkEeYZvvL6N3d3Tp69KjWrl0bm5eWlqaSkhIdOXJkwPW7urrU1dUVu9za2ipJunjxoqLRaHyxlzsTbvPChQsJ5ye6/oULF3Tp0iVduHBBo0ePvua6k3G1OpKVbC2jei1dutSrUdE09fQGbK0l4faSOA7JXLe9vV2SZFnWdVQXL9k8StefyUT7Fo1GE+buauu42nrsuA8ku+7Y8n45S/Z+l0x9yUg266OinQPuL4nWYyKPkjfOkcnoW8dQ8zGcdQ+lxuvd90TnUbtxjvzCcDM2FCb/xw+2zURGXe687lzZ/b98sP9FfYaayaTyaNmsubnZkmQdPnw4bv7jjz9uFRUVDbh+dXW1JYmJKTadPn3asTySSab+k515HE4mySNT/4lzJJObpqHk0fEPdV+7dq2qqqpil3t7e3Xx4kXddNNNCgTsf3TZ1tamSZMm6fTp08rNzbV9/U7wyz5ZlqX29naFQiFH6zCRST8cIz/sgzT0/fBzHk3wej68UD+ZvDa3HUc/15NMHm1vNm+++Walp6crEonEzY9EIho/fvyA6weDQQWDwbh5N954o91lDZCbm+uKA28nP+xTXl6eretLNo+S2Uz64Rj5YR+koe2H3XmUvHOOHC6v58Pt9fv9HGkXtx1Hv9Yz1Dza/gahjIwMzZo1S3V1dbF5vb29qqur091332335oBBkUe4DZmEm5BHjAQjL6NXVVWprKxMs2fPVlFRkTZv3qzOzk6Vl5eb2BwwKPIItyGTcBPyCNOMNJvLli3T+fPn9eSTT6qlpUV33nmn3nzzTeXn55vYXFKCwaCqq6sHvATgZX7cJzu5IY9+OEZ+2AfJHfvhhkzazQ236/Xwev3Xw095dNtxpJ4vBCzL5s/1AAAAAP6P70YHAACAMTSbAAAAMIZmEwAAAMbQbAIAAMCYlG02P/74Yz366KMqKChQVlaWbrnlFlVXV6u7u9vp0pJSU1OjqVOnKjMzU3PmzNEHH3zgdEkYhJdz5+WsbdiwQXfddZdycnI0btw4LV68WP/617+cLsu3fvWrX2nu3LnKzs523Yd9X42X843EnM6hmzJ16NAhPfDAAwqFQgoEAtq3b9+Ibj9lm82TJ0+qt7dXzz//vE6cOKHf/e532rp1q372s585XdqQvfLKK6qqqlJ1dbU++ugjFRYWqrS0VOfOnXO6NFyFV3Pn9ay9++67qqys1Hvvvafa2lpFo1HNnz9fnZ2dTpfmS93d3Vq6dKlWrlzpdClD4vV8IzEnc+i2THV2dqqwsFA1NTWObF/X/Pb0FLJx40aroKDA6TKGrKioyKqsrIxd7unpsUKhkLVhwwYHq0KyvJA7v2Xt3LlzliTr3XffdboUX9u+fbuVl5fndBnX5Ld8I54TOXRzpiRZe/fuHdFtpuwzm4m0trZqzJgxTpcxJN3d3Tp69KhKSkpi89LS0lRSUqIjR444WBmS5fbc+TFrra2tkuTq2x0jw4/5hrPI1EA0m//X2NioLVu2qKKiwulShuTTTz9VT0/PgG94yM/PV0tLi0NVIVleyJ3fstbb26vVq1dr3rx5mjlzptPlwGF+yzecR6YG8l2zuWbNGgUCgUGnkydPxv1Nc3OzvvWtb2np0qVasWKFQ5XDy8idd1RWVur48ePavXu306V4ynAyDtiNHHqTke9Gd9JPf/pTPfLII4NeZ9q0abHfz5w5o+LiYs2dO1fbtm0zXJ19br75ZqWnpysSicTNj0QiGj9+vENVpS4/585PWVu1apVee+01HTp0SBMnTnS6HE9JNuNe4ad8pwIv5JBMDeS7ZnPs2LEaO3bskK7b3Nys4uJizZo1S9u3b1damnee6M3IyNCsWbNUV1enxYsXS/ri5cG6ujqtWrXK2eJSkJ9z54esWZalH//4x9q7d6/eeecdFRQUOF2S5ySTcS/xQ75TiRdySKYG8l2zOVTNzc269957NWXKFP3mN7/R+fPnY8u88sijqqpKZWVlmj17toqKirR582Z1dnaqvLzc6dJwFV7NndezVllZqV27dumvf/2rcnJyYuOm8vLylJWV5XB1/vPJJ5/o4sWL+uSTT9TT06Njx45JkqZPn64bbrjB2eIS8Hq+kZiTOXRbpjo6OtTY2Bi73NTUpGPHjmnMmDGaPHmy+QJG9L3vLrJ9+3ZLUsLJS7Zs2WJNnjzZysjIsIqKiqz33nvP6ZIwCC/nzstZu9ptvn37dqdL86WysrKEt/fbb7/tdGlX5eV8IzGnc+imTL399tsJb4uysrIR2X7AsizLeEcLAACAlOTuwWIAAADwNJpNAAAAGEOzCQAAAGNoNgEAAGAMzSYAAACModkEAACAMTSbAAAAMIZmEwAAAMbQbAIAAMAYmk0AAAAYQ7MJAAAAY2g2AQAAYMz/AGV+/FtYUkWSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 800x100 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## demo data with many zeros\n",
    "data2=data.copy()\n",
    "for c in data2:\n",
    "    data2.loc[data2.sample(frac=0.5).index,c] = 0\n",
    "_=data2.hist(bins=20,figsize=[8,1],layout=[1,4])\n",
    "data2.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c518d760-be69-4846-8afd-4791ae64bf4d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable1</th>\n",
       "      <th>variable2</th>\n",
       "      <th>rr</th>\n",
       "      <th>ci</th>\n",
       "      <th>ci_type</th>\n",
       "      <th>n</th>\n",
       "      <th>method</th>\n",
       "      <th>r</th>\n",
       "      <th>P</th>\n",
       "      <th>Q</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A</td>\n",
       "      <td>C</td>\n",
       "      <td>0.095477</td>\n",
       "      <td>0.208302</td>\n",
       "      <td>max</td>\n",
       "      <td>24</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.100917</td>\n",
       "      <td>0.638926</td>\n",
       "      <td>0.906049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A</td>\n",
       "      <td>D</td>\n",
       "      <td>0.088154</td>\n",
       "      <td>0.074587</td>\n",
       "      <td>max</td>\n",
       "      <td>22</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.070554</td>\n",
       "      <td>0.755041</td>\n",
       "      <td>0.906049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "      <td>0.121447</td>\n",
       "      <td>0.334205</td>\n",
       "      <td>max</td>\n",
       "      <td>21</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.125581</td>\n",
       "      <td>0.587541</td>\n",
       "      <td>0.906049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C</td>\n",
       "      <td>D</td>\n",
       "      <td>-0.204256</td>\n",
       "      <td>0.187645</td>\n",
       "      <td>max</td>\n",
       "      <td>22</td>\n",
       "      <td>spearman</td>\n",
       "      <td>-0.206997</td>\n",
       "      <td>0.355333</td>\n",
       "      <td>0.906049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>D</td>\n",
       "      <td>B</td>\n",
       "      <td>-0.265778</td>\n",
       "      <td>0.193924</td>\n",
       "      <td>max</td>\n",
       "      <td>20</td>\n",
       "      <td>spearman</td>\n",
       "      <td>-0.273282</td>\n",
       "      <td>0.243687</td>\n",
       "      <td>0.906049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C</td>\n",
       "      <td>B</td>\n",
       "      <td>0.027264</td>\n",
       "      <td>0.086304</td>\n",
       "      <td>max</td>\n",
       "      <td>23</td>\n",
       "      <td>spearman</td>\n",
       "      <td>0.022680</td>\n",
       "      <td>0.918186</td>\n",
       "      <td>0.918186</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  variable1 variable2        rr        ci ci_type   n    method         r  \\\n",
       "0         A         C  0.095477  0.208302     max  24  spearman  0.100917   \n",
       "1         A         D  0.088154  0.074587     max  22  spearman  0.070554   \n",
       "2         A         B  0.121447  0.334205     max  21  spearman  0.125581   \n",
       "3         C         D -0.204256  0.187645     max  22  spearman -0.206997   \n",
       "5         D         B -0.265778  0.193924     max  20  spearman -0.273282   \n",
       "4         C         B  0.027264  0.086304     max  23  spearman  0.022680   \n",
       "\n",
       "          P         Q  \n",
       "0  0.638926  0.906049  \n",
       "1  0.755041  0.906049  \n",
       "2  0.587541  0.906049  \n",
       "3  0.355333  0.906049  \n",
       "5  0.243687  0.906049  \n",
       "4  0.918186  0.918186  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out=get_corrs(\n",
    "    data=data2,\n",
    "    method='spearman',\n",
    "    resample=True,\n",
    "    preprocess_kws=dict(drop_same_value=0),\n",
    ")\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b50332c7-89b0-4be5-a8e4-f3d12a169aba",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Documentation\n",
    "[`roux.stat.corr`](https://github.com/rraadd88/roux#module-rouxstatcorr)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}