3. BioPortal (khop-matrices-lcc).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "# !{sys.executable} -m pip install tldextract"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style='text-align:center;'>\n",
    "<h1>BioPortal</h1>\n",
    "<h2>k-hop matrices</h2>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "__author__ = \"Lisette Espin-Noboa\"\n",
    "__copyright__ = \"Copyright 2018, HopRank\"\n",
    "__credits__ = [\"Florian Lemmerich\", \"Markus Strohmaier\", \"Simon Walk\", \"Mark Musen\"]\n",
    "__license__ = \"GPL\"\n",
    "__version__ = \"1.0.3\"\n",
    "__maintainer__ = \"Lisette Espin-Noboa\"\n",
    "__email__ = \"Lisette.Espin@gesis.org\"\n",
    "__status__ = \"Developing\"\n",
    "\n",
    "########################################################################################\n",
    "# Warnings\n",
    "########################################################################################\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", message=\"numpy.dtype size changed\")\n",
    "warnings.filterwarnings(\"ignore\", message=\"numpy.ufunc size changed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Dependencies</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################################\n",
    "# System\n",
    "########################################################################################\n",
    "import os\n",
    "import importlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import networkx as nx\n",
    "from collections import Counter\n",
    "from joblib import Parallel, delayed\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt \n",
    "from collections import defaultdict\n",
    "\n",
    "########################################################################################\n",
    "# Local Dependencies\n",
    "########################################################################################\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "from org.gesis.libs.bioportal.ontology import Ontology\n",
    "from org.gesis.libs.bioportal.transition import Transition\n",
    "from org.gesis.libs.bioportal.clickstream import NAVITYPES\n",
    "from org.gesis.libs.bioportal.clickstream import load_clickstream\n",
    "from org.gesis.libs.bioportal.clickstream import preprocess_clickstream\n",
    "from org.gesis.libs.bioportal.clickstream import preprocess_sessions\n",
    "from org.gesis.libs.bioportal.clickstream import clean_clickstream\n",
    "from org.gesis.libs.bioportal.submission import get_submissions\n",
    "from org.gesis.libs.utils import log\n",
    "from org.gesis.libs.utils import weighted_to_undirected\n",
    "from org.gesis.libs.utils import to_symmetric\n",
    "\n",
    "########################################################################################\n",
    "# Plot's style\n",
    "########################################################################################\n",
    "sns.set_context(context=\"paper\", font_scale=1.5)\n",
    "sns.set_style(\"whitegrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Constants</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SM = 'submission'\n",
    "SUBMISSIONS_FN = '/bigdata/lespin/datasets/bioportal/submissions.json'\n",
    "\n",
    "ON = 'ontology'\n",
    "ONTOLOGY_ROOT = '/bigdata/lespin/datasets/bioportal/ontologies'\n",
    "ONTOLOGY_GRAPH_OUTPUT = '/bigdata/lespin/bioportal/ontologies/graph'\n",
    "ONTOLOGY_ADJACENCY_OUTPUT = '/bigdata/lespin/bioportal/ontologies/matrix'\n",
    "ONTOLOGY_NODES_OUTPUT = '/bigdata/lespin/bioportal/ontologies/nodes'\n",
    "ONTOLOGY_HOPS_OUTPUT = '/bigdata/lespin/bioportal/ontologies/hops'\n",
    "\n",
    "CS = 'clickstream'\n",
    "CS_FN = 'BP_webpage_requests_<YEAR><POSTFIX>.csv.bz2'\n",
    "CLICKSTREAM_ROOT = '/bigdata/lespin/datasets/bioportal/clickstream/'\n",
    "CLICKSTREAM_DF = '/bigdata/lespin/bioportal/clickstreams'\n",
    "\n",
    "TR = 'transitions'\n",
    "TRANS_GRAPH_OUTPUT = '/bigdata/lespin/bioportal/transitions/graph'\n",
    "TRANS_ADJACENCY_OUTPUT = '/bigdata/lespin/bioportal/transitions/matrix'\n",
    "TRANS_NODES_OUTPUT = '/bigdata/lespin/bioportal/transitions/nodes'\n",
    "LCC_TRANS_GRAPH_OUTPUT = '/bigdata/lespin/bioportal/transitions_in_lcc/graph'\n",
    "LCC_TRANS_ADJACENCY_OUTPUT = '/bigdata/lespin/bioportal/transitions_in_lcc/matrix'\n",
    "LCC_TRANS_NODES_OUTPUT = '/bigdata/lespin/bioportal/transitions_in_lcc/nodes'\n",
    "\n",
    "SU = 'summary'\n",
    "LOG_OUTPUT = '/bigdata/lespin/bioportal/logs'\n",
    "PLOTS_OUTPUT = '/bigdata/lespin/bioportal/plots'\n",
    "SUMMARY_OUTPUT = '/bigdata/lespin/bioportal/summary'\n",
    "YEAR = '2015'\n",
    "\n",
    "LAG_BREAK_IN_MINUTES = 60\n",
    "MIN_SESSION_LENGTH = 2\n",
    "MIN_REQ = 1000\n",
    "MIN_TRANSITIONS = 1000\n",
    "MAXKHOPS = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style='text-align:center;'>\n",
    "<h1>k-hop Neighborhoods</h1>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading submissions...\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/datasets/bioportal/submissions.json loaded!\n",
      "2019-02-15 18:09:56\t- 642 ontologies\n",
      "2019-02-15 18:09:56\t- 13 years\n",
      "loading valid ontologies...\n",
      "LCC: True\n",
      "generating k-hop matrices for 12 ontologies...\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_WHO-ART_2015.npz loaded!\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:09:56\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_5HOP.npz'\n",
      "2019-02-15 18:09:56\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_5HOP.npz NOT loaded!\n",
      "2019-02-15 18:09:56\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:09:56\ttocsr and int8...\n",
      "2019-02-15 18:09:56\tsubstracting 4hop from 5hop...\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:09:56\tsubstracting 3hop from 5hop...\n",
      "2019-02-15 18:09:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:09:56\tsubstracting 2hop from 5hop...\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:09:57\tsubstracting 1hop from 5hop...\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_WHO-ART_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t>0...\n",
      "2019-02-15 18:09:57\teliminating 0s...\n",
      "2019-02-15 18:09:57\tsetting diagonal to zero...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/ssd/lespin/virtualenv/python3.5/lib/python3.5/site-packages/scipy/sparse/compressed.py:746: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  SparseEfficiencyWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-02-15 18:09:57\teliminating 0s...\n",
      "2019-02-15 18:09:57\tto csr int...\n",
      "2019-02-15 18:09:57\tdone 5-hop!\n",
      "2019-02-15 18:09:57\tWHO-ART-2015-6: 5-hop has reached zero!\n",
      "last k: 4\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_CHMO_2015.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_17HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_20HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_21HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_22HOP.npz loaded!\n",
      "2019-02-15 18:09:57\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_23HOP.npz'\n",
      "2019-02-15 18:09:57\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_23HOP.npz NOT loaded!\n",
      "2019-02-15 18:09:57\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:09:57\ttocsr and int8...\n",
      "2019-02-15 18:09:58\tsubstracting 22hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_22HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 21hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_21HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 20hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_20HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 19hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 18hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 17hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_17HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 16hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 15hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 14hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 13hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 12hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 11hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 10hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 9hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:09:58\tsubstracting 8hop from 23hop...\n",
      "2019-02-15 18:09:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 7hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 6hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 5hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 4hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 3hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 2hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:09:59\tsubstracting 1hop from 23hop...\n",
      "2019-02-15 18:09:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CHMO_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:10:00\t>0...\n",
      "2019-02-15 18:10:00\teliminating 0s...\n",
      "2019-02-15 18:10:00\tsetting diagonal to zero...\n",
      "2019-02-15 18:10:00\teliminating 0s...\n",
      "2019-02-15 18:10:00\tto csr int...\n",
      "2019-02-15 18:10:00\tdone 23-hop!\n",
      "2019-02-15 18:10:00\tCHMO-2015-18: 23-hop has reached zero!\n",
      "last k: 22\n",
      "2019-02-15 18:10:00\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_HL7_2015.npz loaded!\n",
      "2019-02-15 18:10:00\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:10:00\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:10:00\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:10:00\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:10:01\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:10:01\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_17HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:10:02\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_20HOP.npz'\n",
      "2019-02-15 18:10:02\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_20HOP.npz NOT loaded!\n",
      "2019-02-15 18:10:02\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:10:02\ttocsr and int8...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-02-15 18:10:02\tsubstracting 19hop from 20hop...\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/hops/LCC_HL7_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:10:02\tHL7-2015-6: 20-hop has reached zero!\n",
      "last k: 19\n",
      "2019-02-15 18:10:02\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_ICD10_2015.npz loaded!\n",
      "2019-02-15 18:10:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:10:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:10:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:10:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:10:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:10:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:10:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:10:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:10:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:10:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:10:06\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_11HOP.npz'\n",
      "2019-02-15 18:10:06\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_11HOP.npz NOT loaded!\n",
      "2019-02-15 18:10:06\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:10:06\ttocsr and int8...\n",
      "2019-02-15 18:10:06\tsubstracting 10hop from 11hop...\n",
      "2019-02-15 18:10:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:10:07\tsubstracting 9hop from 11hop...\n",
      "2019-02-15 18:10:07\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:10:07\tsubstracting 8hop from 11hop...\n",
      "2019-02-15 18:10:08\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:10:10\tsubstracting 7hop from 11hop...\n",
      "2019-02-15 18:10:11\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:10:13\tsubstracting 6hop from 11hop...\n",
      "2019-02-15 18:10:13\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:10:15\tsubstracting 5hop from 11hop...\n",
      "2019-02-15 18:10:15\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:10:17\tsubstracting 4hop from 11hop...\n",
      "2019-02-15 18:10:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:10:19\tsubstracting 3hop from 11hop...\n",
      "2019-02-15 18:10:19\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:10:21\tsubstracting 2hop from 11hop...\n",
      "2019-02-15 18:10:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:10:23\tsubstracting 1hop from 11hop...\n",
      "2019-02-15 18:10:23\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD10_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:10:24\t>0...\n",
      "2019-02-15 18:10:25\teliminating 0s...\n",
      "2019-02-15 18:10:25\tsetting diagonal to zero...\n",
      "2019-02-15 18:10:26\teliminating 0s...\n",
      "2019-02-15 18:10:26\tto csr int...\n",
      "2019-02-15 18:10:26\tdone 11-hop!\n",
      "2019-02-15 18:10:26\tICD10-2015-6: 11-hop has reached zero!\n",
      "last k: 10\n",
      "2019-02-15 18:10:26\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_CPT_2015.npz loaded!\n",
      "2019-02-15 18:10:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:10:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:10:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:10:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:10:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:10:27\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:10:27\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:10:28\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:10:30\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:10:31\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_16HOP.npz'\n",
      "2019-02-15 18:10:31\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_16HOP.npz NOT loaded!\n",
      "2019-02-15 18:10:31\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:10:31\ttocsr and int8...\n",
      "2019-02-15 18:10:31\tsubstracting 15hop from 16hop...\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:10:31\tsubstracting 14hop from 16hop...\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:10:31\tsubstracting 13hop from 16hop...\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:10:31\tsubstracting 12hop from 16hop...\n",
      "2019-02-15 18:10:31\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:10:32\tsubstracting 11hop from 16hop...\n",
      "2019-02-15 18:10:32\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:10:32\tsubstracting 10hop from 16hop...\n",
      "2019-02-15 18:10:33\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:10:34\tsubstracting 9hop from 16hop...\n",
      "2019-02-15 18:10:35\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:10:37\tsubstracting 8hop from 16hop...\n",
      "2019-02-15 18:10:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:10:40\tsubstracting 7hop from 16hop...\n",
      "2019-02-15 18:10:41\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:10:44\tsubstracting 6hop from 16hop...\n",
      "2019-02-15 18:10:44\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:10:47\tsubstracting 5hop from 16hop...\n",
      "2019-02-15 18:10:47\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:10:50\tsubstracting 4hop from 16hop...\n",
      "2019-02-15 18:10:50\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:10:52\tsubstracting 3hop from 16hop...\n",
      "2019-02-15 18:10:52\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:10:55\tsubstracting 2hop from 16hop...\n",
      "2019-02-15 18:10:55\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:10:58\tsubstracting 1hop from 16hop...\n",
      "2019-02-15 18:10:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_CPT_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:11:01\t>0...\n",
      "2019-02-15 18:11:03\teliminating 0s...\n",
      "2019-02-15 18:11:03\tsetting diagonal to zero...\n",
      "2019-02-15 18:11:03\teliminating 0s...\n",
      "2019-02-15 18:11:03\tto csr int...\n",
      "2019-02-15 18:11:03\tdone 16-hop!\n",
      "2019-02-15 18:11:03\tCPT-2015-6: 16-hop has reached zero!\n",
      "last k: 15\n",
      "2019-02-15 18:11:03\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_MESH_2015.npz loaded!\n",
      "2019-02-15 18:11:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:11:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:11:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:11:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:11:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:11:05\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_6HOP.npz loaded!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-02-15 18:11:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:11:07\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:11:09\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:11:11\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:11:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:11:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:11:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:11:15\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:11:15\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_17HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_20HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_21HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_22HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_23HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_24HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_25HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_26HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_27HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_28HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_29HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_30HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_31HOP.npz loaded!\n",
      "2019-02-15 18:11:16\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_32HOP.npz'\n",
      "2019-02-15 18:11:16\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_32HOP.npz NOT loaded!\n",
      "2019-02-15 18:11:16\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:11:16\ttocsr and int8...\n",
      "2019-02-15 18:11:16\tsubstracting 31hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_31HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 30hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_30HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 29hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_29HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 28hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_28HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 27hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_27HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 26hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_26HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 25hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_25HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 24hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_24HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 23hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_23HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 22hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_22HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 21hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_21HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 20hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_20HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 19hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 18hop from 32hop...\n",
      "2019-02-15 18:11:16\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:11:16\tsubstracting 17hop from 32hop...\n",
      "2019-02-15 18:11:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_17HOP.npz loaded!\n",
      "2019-02-15 18:11:17\tsubstracting 16hop from 32hop...\n",
      "2019-02-15 18:11:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:11:17\tsubstracting 15hop from 32hop...\n",
      "2019-02-15 18:11:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:11:18\tsubstracting 14hop from 32hop...\n",
      "2019-02-15 18:11:18\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:11:19\tsubstracting 13hop from 32hop...\n",
      "2019-02-15 18:11:19\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:11:21\tsubstracting 12hop from 32hop...\n",
      "2019-02-15 18:11:22\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:11:23\tsubstracting 11hop from 32hop...\n",
      "2019-02-15 18:11:24\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:11:27\tsubstracting 10hop from 32hop...\n",
      "2019-02-15 18:11:28\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:11:32\tsubstracting 9hop from 32hop...\n",
      "2019-02-15 18:11:33\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:11:37\tsubstracting 8hop from 32hop...\n",
      "2019-02-15 18:11:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:11:43\tsubstracting 7hop from 32hop...\n",
      "2019-02-15 18:11:44\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:11:48\tsubstracting 6hop from 32hop...\n",
      "2019-02-15 18:11:49\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:11:53\tsubstracting 5hop from 32hop...\n",
      "2019-02-15 18:11:54\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:11:58\tsubstracting 4hop from 32hop...\n",
      "2019-02-15 18:11:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:12:03\tsubstracting 3hop from 32hop...\n",
      "2019-02-15 18:12:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:12:08\tsubstracting 2hop from 32hop...\n",
      "2019-02-15 18:12:08\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:12:13\tsubstracting 1hop from 32hop...\n",
      "2019-02-15 18:12:13\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MESH_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:12:17\t>0...\n",
      "2019-02-15 18:12:19\teliminating 0s...\n",
      "2019-02-15 18:12:19\tsetting diagonal to zero...\n",
      "2019-02-15 18:12:20\teliminating 0s...\n",
      "2019-02-15 18:12:20\tto csr int...\n",
      "2019-02-15 18:12:20\tdone 32-hop!\n",
      "2019-02-15 18:12:20\tMESH-2015-10: 32-hop has reached zero!\n",
      "last k: 31\n",
      "2019-02-15 18:12:20\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_ICD9CM_2015.npz loaded!\n",
      "2019-02-15 18:12:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:12:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:12:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:12:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:12:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:12:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:12:22\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_7HOP.npz loaded!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-02-15 18:12:25\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:12:29\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:12:33\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:12:35\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:12:36\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:12:36\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_13HOP.npz'\n",
      "2019-02-15 18:12:36\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_13HOP.npz NOT loaded!\n",
      "2019-02-15 18:12:36\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:12:36\ttocsr and int8...\n",
      "2019-02-15 18:12:36\tsubstracting 12hop from 13hop...\n",
      "2019-02-15 18:12:36\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:12:36\tsubstracting 11hop from 13hop...\n",
      "2019-02-15 18:12:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:12:39\tsubstracting 10hop from 13hop...\n",
      "2019-02-15 18:12:44\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:12:48\tsubstracting 9hop from 13hop...\n",
      "2019-02-15 18:12:52\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:12:59\tsubstracting 8hop from 13hop...\n",
      "2019-02-15 18:13:01\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:13:09\tsubstracting 7hop from 13hop...\n",
      "2019-02-15 18:13:09\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:13:17\tsubstracting 6hop from 13hop...\n",
      "2019-02-15 18:13:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:13:26\tsubstracting 5hop from 13hop...\n",
      "2019-02-15 18:13:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:13:34\tsubstracting 4hop from 13hop...\n",
      "2019-02-15 18:13:34\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:13:42\tsubstracting 3hop from 13hop...\n",
      "2019-02-15 18:13:42\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:13:49\tsubstracting 2hop from 13hop...\n",
      "2019-02-15 18:13:49\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:13:57\tsubstracting 1hop from 13hop...\n",
      "2019-02-15 18:13:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_ICD9CM_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:14:06\t>0...\n",
      "2019-02-15 18:14:10\teliminating 0s...\n",
      "2019-02-15 18:14:10\tsetting diagonal to zero...\n",
      "2019-02-15 18:14:11\teliminating 0s...\n",
      "2019-02-15 18:14:11\tto csr int...\n",
      "2019-02-15 18:14:11\tdone 13-hop!\n",
      "2019-02-15 18:14:11\tICD9CM-2015-8: 13-hop has reached zero!\n",
      "last k: 12\n",
      "2019-02-15 18:14:11\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_MEDDRA_2015.npz loaded!\n",
      "2019-02-15 18:14:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:14:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:14:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:14:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:14:13\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:14:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:14:18\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:14:32\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:14:32\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_9HOP.npz'\n",
      "2019-02-15 18:14:32\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_9HOP.npz NOT loaded!\n",
      "2019-02-15 18:14:32\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:14:40\ttocsr and int8...\n",
      "2019-02-15 18:14:42\tsubstracting 8hop from 9hop...\n",
      "2019-02-15 18:14:53\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:15:00\tsubstracting 7hop from 9hop...\n",
      "2019-02-15 18:15:01\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:15:07\tsubstracting 6hop from 9hop...\n",
      "2019-02-15 18:15:11\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:15:20\tsubstracting 5hop from 9hop...\n",
      "2019-02-15 18:15:20\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:15:29\tsubstracting 4hop from 9hop...\n",
      "2019-02-15 18:15:29\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:15:37\tsubstracting 3hop from 9hop...\n",
      "2019-02-15 18:15:37\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:15:45\tsubstracting 2hop from 9hop...\n",
      "2019-02-15 18:15:45\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:15:53\tsubstracting 1hop from 9hop...\n",
      "2019-02-15 18:15:53\t/bigdata/lespin/bioportal/ontologies/hops/LCC_MEDDRA_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:16:00\t>0...\n",
      "2019-02-15 18:16:05\teliminating 0s...\n",
      "2019-02-15 18:16:05\tsetting diagonal to zero...\n",
      "2019-02-15 18:16:06\teliminating 0s...\n",
      "2019-02-15 18:16:06\tto csr int...\n",
      "2019-02-15 18:16:06\tdone 9-hop!\n",
      "2019-02-15 18:16:06\tMEDDRA-2015-6: 9-hop has reached zero!\n",
      "last k: 8\n",
      "2019-02-15 18:16:06\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_NDFRT_2015.npz loaded!\n",
      "2019-02-15 18:16:07\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:16:07\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:16:07\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:16:08\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:16:10\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:16:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:16:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:16:17\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:16:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:16:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:16:30\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:16:33\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:16:36\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:16:37\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_17HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_20HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_21HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_22HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_23HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_24HOP.npz loaded!\n",
      "2019-02-15 18:16:38\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_25HOP.npz'\n",
      "2019-02-15 18:16:38\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_25HOP.npz NOT loaded!\n",
      "2019-02-15 18:16:38\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:16:38\ttocsr and int8...\n",
      "2019-02-15 18:16:39\tsubstracting 24hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_24HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 23hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_23HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 22hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_22HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 21hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_21HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 20hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_20HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 19hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_19HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 18hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_18HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 17hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_17HOP.npz loaded!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-02-15 18:16:39\tsubstracting 16hop from 25hop...\n",
      "2019-02-15 18:16:39\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_16HOP.npz loaded!\n",
      "2019-02-15 18:16:39\tsubstracting 15hop from 25hop...\n",
      "2019-02-15 18:16:40\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_15HOP.npz loaded!\n",
      "2019-02-15 18:16:40\tsubstracting 14hop from 25hop...\n",
      "2019-02-15 18:16:41\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_14HOP.npz loaded!\n",
      "2019-02-15 18:16:42\tsubstracting 13hop from 25hop...\n",
      "2019-02-15 18:16:44\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:16:46\tsubstracting 12hop from 25hop...\n",
      "2019-02-15 18:16:50\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:16:55\tsubstracting 11hop from 25hop...\n",
      "2019-02-15 18:16:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:17:07\tsubstracting 10hop from 25hop...\n",
      "2019-02-15 18:17:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:17:22\tsubstracting 9hop from 25hop...\n",
      "2019-02-15 18:17:25\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:17:39\tsubstracting 8hop from 25hop...\n",
      "2019-02-15 18:17:41\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:17:55\tsubstracting 7hop from 25hop...\n",
      "2019-02-15 18:17:58\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:18:13\tsubstracting 6hop from 25hop...\n",
      "2019-02-15 18:18:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:18:31\tsubstracting 5hop from 25hop...\n",
      "2019-02-15 18:18:33\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:18:50\tsubstracting 4hop from 25hop...\n",
      "2019-02-15 18:18:51\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:19:07\tsubstracting 3hop from 25hop...\n",
      "2019-02-15 18:19:07\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:19:25\tsubstracting 2hop from 25hop...\n",
      "2019-02-15 18:19:25\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:19:42\tsubstracting 1hop from 25hop...\n",
      "2019-02-15 18:19:42\t/bigdata/lespin/bioportal/ontologies/hops/LCC_NDFRT_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:19:59\t>0...\n",
      "2019-02-15 18:20:09\teliminating 0s...\n",
      "2019-02-15 18:20:09\tsetting diagonal to zero...\n",
      "2019-02-15 18:20:10\teliminating 0s...\n",
      "2019-02-15 18:20:10\tto csr int...\n",
      "2019-02-15 18:20:10\tdone 25-hop!\n",
      "2019-02-15 18:20:10\tNDFRT-2015-7: 25-hop has reached zero!\n",
      "last k: 24\n",
      "2019-02-15 18:20:10\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_OMIM_2015.npz loaded!\n",
      "2019-02-15 18:20:11\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:20:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:20:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:20:42\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:20:57\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:21:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:21:06\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_7HOP.npz'\n",
      "2019-02-15 18:21:06\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_7HOP.npz NOT loaded!\n",
      "2019-02-15 18:21:06\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:21:10\ttocsr and int8...\n",
      "2019-02-15 18:21:10\tsubstracting 6hop from 7hop...\n",
      "2019-02-15 18:21:18\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:21:22\tsubstracting 5hop from 7hop...\n",
      "2019-02-15 18:21:35\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:21:46\tsubstracting 4hop from 7hop...\n",
      "2019-02-15 18:22:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:22:36\tsubstracting 3hop from 7hop...\n",
      "2019-02-15 18:22:36\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:22:58\tsubstracting 2hop from 7hop...\n",
      "2019-02-15 18:23:00\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:23:25\tsubstracting 1hop from 7hop...\n",
      "2019-02-15 18:23:25\t/bigdata/lespin/bioportal/ontologies/hops/LCC_OMIM_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:23:48\t>0...\n",
      "2019-02-15 18:24:00\teliminating 0s...\n",
      "2019-02-15 18:24:00\tsetting diagonal to zero...\n",
      "2019-02-15 18:24:02\teliminating 0s...\n",
      "2019-02-15 18:24:02\tto csr int...\n",
      "2019-02-15 18:24:02\tdone 7-hop!\n",
      "2019-02-15 18:24:02\tOMIM-2015-8: 7-hop has reached zero!\n",
      "last k: 6\n",
      "2019-02-15 18:24:02\t/bigdata/lespin/bioportal/ontologies/matrix/LCC_LOINC_2015.npz loaded!\n",
      "2019-02-15 18:24:04\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_1HOP.npz loaded!\n",
      "2019-02-15 18:24:15\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_2HOP.npz loaded!\n",
      "2019-02-15 18:24:23\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:26:43\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:27:52\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:31:06\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:31:59\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:32:12\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:32:14\t[Errno 2] No such file or directory: '/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_14HOP.npz'\n",
      "2019-02-15 18:32:14\tERROR: /bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_14HOP.npz NOT loaded!\n",
      "2019-02-15 18:32:14\tmultiplying: product.dot(m)...\n",
      "2019-02-15 18:32:14\ttocsr and int8...\n",
      "2019-02-15 18:32:14\tsubstracting 13hop from 14hop...\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_13HOP.npz loaded!\n",
      "2019-02-15 18:32:14\tsubstracting 12hop from 14hop...\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_12HOP.npz loaded!\n",
      "2019-02-15 18:32:14\tsubstracting 11hop from 14hop...\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_11HOP.npz loaded!\n",
      "2019-02-15 18:32:14\tsubstracting 10hop from 14hop...\n",
      "2019-02-15 18:32:14\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_10HOP.npz loaded!\n",
      "2019-02-15 18:32:14\tsubstracting 9hop from 14hop...\n",
      "2019-02-15 18:32:15\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_9HOP.npz loaded!\n",
      "2019-02-15 18:32:16\tsubstracting 8hop from 14hop...\n",
      "2019-02-15 18:32:26\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_8HOP.npz loaded!\n",
      "2019-02-15 18:32:34\tsubstracting 7hop from 14hop...\n",
      "2019-02-15 18:33:22\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_7HOP.npz loaded!\n",
      "2019-02-15 18:33:53\tsubstracting 6hop from 14hop...\n",
      "2019-02-15 18:37:11\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_6HOP.npz loaded!\n",
      "2019-02-15 18:40:01\tsubstracting 5hop from 14hop...\n",
      "2019-02-15 18:41:08\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_5HOP.npz loaded!\n",
      "2019-02-15 18:45:24\tsubstracting 4hop from 14hop...\n",
      "2019-02-15 18:47:21\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_4HOP.npz loaded!\n",
      "2019-02-15 18:53:00\tsubstracting 3hop from 14hop...\n",
      "2019-02-15 18:53:09\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_3HOP.npz loaded!\n",
      "2019-02-15 18:59:14\tsubstracting 2hop from 14hop...\n",
      "2019-02-15 18:59:27\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_2HOP.npz loaded!\n",
      "2019-02-15 19:05:56\tsubstracting 1hop from 14hop...\n",
      "2019-02-15 19:05:56\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_1HOP.npz loaded!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-02-15 19:12:05\t>0...\n",
      "2019-02-15 19:16:57\teliminating 0s...\n",
      "2019-02-15 19:16:57\tsetting diagonal to zero...\n",
      "2019-02-15 19:17:03\teliminating 0s...\n",
      "2019-02-15 19:17:03\tto csr int...\n",
      "2019-02-15 19:17:03\tdone 14-hop!\n",
      "2019-02-15 19:17:03\tLOINC-2015-9: 14-hop has reached zero!\n",
      "last k: 13\n"
     ]
    }
   ],
   "source": [
    "print('loading submissions...')\n",
    "submissions_dict = get_submissions(SUBMISSIONS_FN)\n",
    "\n",
    "print('loading valid ontologies...')\n",
    "fn = os.path.join(SUMMARY_OUTPUT,'summary_ontologies_transitions_{}.{}'.format(YEAR,'csv'))\n",
    "df_onto = pd.read_csv(fn, index_col=False)\n",
    "tmp = df_onto.loc[df_onto[\"T'\"] >= MIN_TRANSITIONS].copy()\n",
    "\n",
    "lcc = True\n",
    "print('LCC: {}'.format(lcc))\n",
    "\n",
    "print('generating k-hop matrices for {} ontologies...'.format(tmp.ontology.nunique()))\n",
    "for index,row in tmp.sort_values(\"E'\", ascending=True).iterrows():\n",
    "    onto_name = row['ontology']\n",
    "    \n",
    "    if onto_name in ['SNOMEDCT']:\n",
    "        continue\n",
    "        \n",
    "    submission = submissions_dict[onto_name][YEAR]\n",
    "    onto = Ontology(name=onto_name, year=YEAR, submission_id=submission['submissionId'], root_folder=ONTOLOGY_ROOT)\n",
    "    onto.load_adjacency(path=ONTOLOGY_ADJACENCY_OUTPUT, lcc=lcc)\n",
    "    k = onto.create_hops_matrices(path=ONTOLOGY_HOPS_OUTPUT, maxk=MAXKHOPS, lcc=lcc)   \n",
    "    df_onto.loc[index,'d'] = k\n",
    "    print('last k: {}'.format(k))\n",
    "    del(onto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>ontology</th>\n",
       "      <th>N</th>\n",
       "      <th>E</th>\n",
       "      <th>k</th>\n",
       "      <th>cc</th>\n",
       "      <th>T</th>\n",
       "      <th>N'</th>\n",
       "      <th>E'</th>\n",
       "      <th>k'</th>\n",
       "      <th>T'</th>\n",
       "      <th>d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>SNOMEDCT</td>\n",
       "      <td>315684</td>\n",
       "      <td>467027</td>\n",
       "      <td>2.958826</td>\n",
       "      <td>60</td>\n",
       "      <td>110859</td>\n",
       "      <td>315205</td>\n",
       "      <td>466607</td>\n",
       "      <td>2.960657</td>\n",
       "      <td>110441</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>CPT</td>\n",
       "      <td>13219</td>\n",
       "      <td>13235</td>\n",
       "      <td>2.002421</td>\n",
       "      <td>3</td>\n",
       "      <td>44815</td>\n",
       "      <td>13092</td>\n",
       "      <td>13110</td>\n",
       "      <td>2.002750</td>\n",
       "      <td>44651</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>MEDDRA</td>\n",
       "      <td>66506</td>\n",
       "      <td>31863</td>\n",
       "      <td>0.958199</td>\n",
       "      <td>43493</td>\n",
       "      <td>71580</td>\n",
       "      <td>22889</td>\n",
       "      <td>31738</td>\n",
       "      <td>2.773210</td>\n",
       "      <td>42746</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>NDFRT</td>\n",
       "      <td>35019</td>\n",
       "      <td>34504</td>\n",
       "      <td>1.970587</td>\n",
       "      <td>522</td>\n",
       "      <td>24662</td>\n",
       "      <td>32074</td>\n",
       "      <td>32080</td>\n",
       "      <td>2.000374</td>\n",
       "      <td>22452</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>LOINC</td>\n",
       "      <td>174513</td>\n",
       "      <td>152683</td>\n",
       "      <td>1.749818</td>\n",
       "      <td>73518</td>\n",
       "      <td>7905</td>\n",
       "      <td>100871</td>\n",
       "      <td>152558</td>\n",
       "      <td>3.024814</td>\n",
       "      <td>6349</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>ICD9CM</td>\n",
       "      <td>22534</td>\n",
       "      <td>22531</td>\n",
       "      <td>1.999734</td>\n",
       "      <td>3</td>\n",
       "      <td>4485</td>\n",
       "      <td>22407</td>\n",
       "      <td>22406</td>\n",
       "      <td>1.999911</td>\n",
       "      <td>4434</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>WHO-ART</td>\n",
       "      <td>1852</td>\n",
       "      <td>2997</td>\n",
       "      <td>3.236501</td>\n",
       "      <td>3</td>\n",
       "      <td>2826</td>\n",
       "      <td>1725</td>\n",
       "      <td>2872</td>\n",
       "      <td>3.329855</td>\n",
       "      <td>2811</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>MESH</td>\n",
       "      <td>165166</td>\n",
       "      <td>24182</td>\n",
       "      <td>0.292821</td>\n",
       "      <td>145652</td>\n",
       "      <td>3842</td>\n",
       "      <td>16947</td>\n",
       "      <td>21596</td>\n",
       "      <td>2.548652</td>\n",
       "      <td>2623</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>ICD10</td>\n",
       "      <td>12446</td>\n",
       "      <td>11256</td>\n",
       "      <td>1.808774</td>\n",
       "      <td>1190</td>\n",
       "      <td>2328</td>\n",
       "      <td>11132</td>\n",
       "      <td>11131</td>\n",
       "      <td>1.999820</td>\n",
       "      <td>2288</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>CHMO</td>\n",
       "      <td>2966</td>\n",
       "      <td>3071</td>\n",
       "      <td>2.070802</td>\n",
       "      <td>3</td>\n",
       "      <td>1423</td>\n",
       "      <td>2964</td>\n",
       "      <td>3071</td>\n",
       "      <td>2.072200</td>\n",
       "      <td>1423</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>HL7</td>\n",
       "      <td>10319</td>\n",
       "      <td>10600</td>\n",
       "      <td>2.054463</td>\n",
       "      <td>1049</td>\n",
       "      <td>1395</td>\n",
       "      <td>9146</td>\n",
       "      <td>10475</td>\n",
       "      <td>2.290619</td>\n",
       "      <td>1374</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>OMIM</td>\n",
       "      <td>81821</td>\n",
       "      <td>39359</td>\n",
       "      <td>0.962076</td>\n",
       "      <td>44110</td>\n",
       "      <td>27437</td>\n",
       "      <td>37587</td>\n",
       "      <td>39234</td>\n",
       "      <td>2.087637</td>\n",
       "      <td>1291</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>CRISP</td>\n",
       "      <td>9040</td>\n",
       "      <td>11551</td>\n",
       "      <td>2.555531</td>\n",
       "      <td>3</td>\n",
       "      <td>831</td>\n",
       "      <td>8913</td>\n",
       "      <td>11426</td>\n",
       "      <td>2.563895</td>\n",
       "      <td>829</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>MAMO</td>\n",
       "      <td>101</td>\n",
       "      <td>103</td>\n",
       "      <td>2.039604</td>\n",
       "      <td>1</td>\n",
       "      <td>681</td>\n",
       "      <td>101</td>\n",
       "      <td>103</td>\n",
       "      <td>2.039604</td>\n",
       "      <td>681</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>ICPC</td>\n",
       "      <td>875</td>\n",
       "      <td>1443</td>\n",
       "      <td>3.298286</td>\n",
       "      <td>60</td>\n",
       "      <td>446</td>\n",
       "      <td>691</td>\n",
       "      <td>1318</td>\n",
       "      <td>3.814761</td>\n",
       "      <td>423</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>TAO</td>\n",
       "      <td>3372</td>\n",
       "      <td>4983</td>\n",
       "      <td>2.955516</td>\n",
       "      <td>146</td>\n",
       "      <td>209</td>\n",
       "      <td>3227</td>\n",
       "      <td>4983</td>\n",
       "      <td>3.088317</td>\n",
       "      <td>208</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>SNMI</td>\n",
       "      <td>42194</td>\n",
       "      <td>39457</td>\n",
       "      <td>1.870266</td>\n",
       "      <td>2739</td>\n",
       "      <td>6091</td>\n",
       "      <td>3536</td>\n",
       "      <td>3535</td>\n",
       "      <td>1.999434</td>\n",
       "      <td>116</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>NDDF</td>\n",
       "      <td>27574</td>\n",
       "      <td>125</td>\n",
       "      <td>0.009067</td>\n",
       "      <td>27449</td>\n",
       "      <td>3181</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "      <td>1.978261</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>RXNORM</td>\n",
       "      <td>118555</td>\n",
       "      <td>125</td>\n",
       "      <td>0.002109</td>\n",
       "      <td>118430</td>\n",
       "      <td>29416</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "      <td>1.978261</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>VANDF</td>\n",
       "      <td>25408</td>\n",
       "      <td>125</td>\n",
       "      <td>0.009839</td>\n",
       "      <td>25283</td>\n",
       "      <td>479</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "      <td>1.978261</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>RCD</td>\n",
       "      <td>11325</td>\n",
       "      <td>6893</td>\n",
       "      <td>1.217307</td>\n",
       "      <td>4446</td>\n",
       "      <td>667</td>\n",
       "      <td>49</td>\n",
       "      <td>53</td>\n",
       "      <td>2.163265</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     #  ontology       N       E         k      cc       T      N'      E'  \\\n",
       "0    1  SNOMEDCT  315684  467027  2.958826      60  110859  315205  466607   \n",
       "1    2       CPT   13219   13235  2.002421       3   44815   13092   13110   \n",
       "2    3    MEDDRA   66506   31863  0.958199   43493   71580   22889   31738   \n",
       "3    4     NDFRT   35019   34504  1.970587     522   24662   32074   32080   \n",
       "4    5     LOINC  174513  152683  1.749818   73518    7905  100871  152558   \n",
       "5    6    ICD9CM   22534   22531  1.999734       3    4485   22407   22406   \n",
       "6    7   WHO-ART    1852    2997  3.236501       3    2826    1725    2872   \n",
       "7    8      MESH  165166   24182  0.292821  145652    3842   16947   21596   \n",
       "8    9     ICD10   12446   11256  1.808774    1190    2328   11132   11131   \n",
       "9   10      CHMO    2966    3071  2.070802       3    1423    2964    3071   \n",
       "10  11       HL7   10319   10600  2.054463    1049    1395    9146   10475   \n",
       "11  12      OMIM   81821   39359  0.962076   44110   27437   37587   39234   \n",
       "12  13     CRISP    9040   11551  2.555531       3     831    8913   11426   \n",
       "13  14      MAMO     101     103  2.039604       1     681     101     103   \n",
       "14  15      ICPC     875    1443  3.298286      60     446     691    1318   \n",
       "15  16       TAO    3372    4983  2.955516     146     209    3227    4983   \n",
       "16  17      SNMI   42194   39457  1.870266    2739    6091    3536    3535   \n",
       "17  18      NDDF   27574     125  0.009067   27449    3181      92      91   \n",
       "18  19    RXNORM  118555     125  0.002109  118430   29416      92      91   \n",
       "19  20     VANDF   25408     125  0.009839   25283     479      92      91   \n",
       "20  21       RCD   11325    6893  1.217307    4446     667      49      53   \n",
       "\n",
       "          k'      T'   d  \n",
       "0   2.960657  110441   0  \n",
       "1   2.002750   44651  15  \n",
       "2   2.773210   42746   8  \n",
       "3   2.000374   22452  24  \n",
       "4   3.024814    6349  13  \n",
       "5   1.999911    4434  12  \n",
       "6   3.329855    2811   4  \n",
       "7   2.548652    2623  31  \n",
       "8   1.999820    2288  10  \n",
       "9   2.072200    1423  22  \n",
       "10  2.290619    1374  19  \n",
       "11  2.087637    1291   6  \n",
       "12  2.563895     829   0  \n",
       "13  2.039604     681   0  \n",
       "14  3.814761     423   0  \n",
       "15  3.088317     208   0  \n",
       "16  1.999434     116   0  \n",
       "17  1.978261      24   0  \n",
       "18  1.978261      10   0  \n",
       "19  1.978261       4   0  \n",
       "20  2.163265       0   0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_onto.d = df_onto.d.fillna(0).astype(np.int)\n",
    "df_onto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>ontology</th>\n",
       "      <th>N</th>\n",
       "      <th>E</th>\n",
       "      <th>k</th>\n",
       "      <th>cc</th>\n",
       "      <th>T</th>\n",
       "      <th>N'</th>\n",
       "      <th>E'</th>\n",
       "      <th>k'</th>\n",
       "      <th>d'</th>\n",
       "      <th>T'</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>SNOMEDCT</td>\n",
       "      <td>315684</td>\n",
       "      <td>467027</td>\n",
       "      <td>2.958826</td>\n",
       "      <td>60</td>\n",
       "      <td>110859</td>\n",
       "      <td>315205</td>\n",
       "      <td>466607</td>\n",
       "      <td>2.960657</td>\n",
       "      <td>0</td>\n",
       "      <td>110441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>CPT</td>\n",
       "      <td>13219</td>\n",
       "      <td>13235</td>\n",
       "      <td>2.002421</td>\n",
       "      <td>3</td>\n",
       "      <td>44815</td>\n",
       "      <td>13092</td>\n",
       "      <td>13110</td>\n",
       "      <td>2.002750</td>\n",
       "      <td>15</td>\n",
       "      <td>44651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>MEDDRA</td>\n",
       "      <td>66506</td>\n",
       "      <td>31863</td>\n",
       "      <td>0.958199</td>\n",
       "      <td>43493</td>\n",
       "      <td>71580</td>\n",
       "      <td>22889</td>\n",
       "      <td>31738</td>\n",
       "      <td>2.773210</td>\n",
       "      <td>8</td>\n",
       "      <td>42746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>NDFRT</td>\n",
       "      <td>35019</td>\n",
       "      <td>34504</td>\n",
       "      <td>1.970587</td>\n",
       "      <td>522</td>\n",
       "      <td>24662</td>\n",
       "      <td>32074</td>\n",
       "      <td>32080</td>\n",
       "      <td>2.000374</td>\n",
       "      <td>24</td>\n",
       "      <td>22452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>LOINC</td>\n",
       "      <td>174513</td>\n",
       "      <td>152683</td>\n",
       "      <td>1.749818</td>\n",
       "      <td>73518</td>\n",
       "      <td>7905</td>\n",
       "      <td>100871</td>\n",
       "      <td>152558</td>\n",
       "      <td>3.024814</td>\n",
       "      <td>13</td>\n",
       "      <td>6349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #  ontology       N       E         k     cc       T      N'      E'  \\\n",
       "0  1  SNOMEDCT  315684  467027  2.958826     60  110859  315205  466607   \n",
       "1  2       CPT   13219   13235  2.002421      3   44815   13092   13110   \n",
       "2  3    MEDDRA   66506   31863  0.958199  43493   71580   22889   31738   \n",
       "3  4     NDFRT   35019   34504  1.970587    522   24662   32074   32080   \n",
       "4  5     LOINC  174513  152683  1.749818  73518    7905  100871  152558   \n",
       "\n",
       "         k'  d'      T'  \n",
       "0  2.960657   0  110441  \n",
       "1  2.002750  15   44651  \n",
       "2  2.773210   8   42746  \n",
       "3  2.000374  24   22452  \n",
       "4  3.024814  13    6349  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_onto.rename(columns={\"d\":\"d'\"},inplace=True)\n",
    "df_onto = df_onto[['#','ontology','N','E','k','cc','T', \"N'\",\"E'\",\"k'\",\"d'\",\"T'\"]]\n",
    "df_onto.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bigdata/lespin/bioportal/summary/lcc_summary_ontologies_transitions_2015.csv saved!\n",
      "/bigdata/lespin/bioportal/summary/lcc_summary_ontologies_transitions_2015.latex saved!\n"
     ]
    }
   ],
   "source": [
    "fn = os.path.join(SUMMARY_OUTPUT,'lcc_summary_ontologies_transitions_{}.{}'.format(YEAR,'csv'))\n",
    "df_onto.to_csv(fn, index=None)\n",
    "print('{} saved!'.format(fn))\n",
    "\n",
    "fn = os.path.join(SUMMARY_OUTPUT,'lcc_summary_ontologies_transitions_{}.{}'.format(YEAR,'latex'))\n",
    "txt = df_onto.to_latex(index=False, float_format=lambda x: '%.2f' % x, bold_rows=[0])\n",
    "with open(fn, 'w') as f:\n",
    "    f.write(txt)\n",
    "    f.write('\\n')\n",
    "print('{} saved!'.format(fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}