find_nets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Os dados que vamos importar representam todas as mátriculas de alunos de Computação em alguma disciplina desde 2003.\n",
    "Cada observação dos dados tem as seguintes características:\n",
    "- PERIODO\n",
    "- COD_DISCIPLINA\n",
    "- NOME_DISCIPLINE\n",
    "- MAT_ALUNO.\n",
    "\n",
    "Ao todo foram analisados (as) mais de 1700 alunos (as)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matriculas_cc = pd.read_csv('dados/matriculas_cc.csv', sep=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculando quantas vezes cada par de alunos (as) estudou juntos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PERIODO, CODIGO, DISCIPLINA, MATRICULA = 0, 1, 2, 3\n",
    "\n",
    "network = {}\n",
    "for group, obs in matriculas_cc.groupby(['PERIODO', 'CODIGO']):\n",
    "    \n",
    "    for value_x in obs.get_values():\n",
    "        mat_x = value_x[MATRICULA]\n",
    "        \n",
    "        for value_y in obs.get_values():\n",
    "            mat_y = value_y[MATRICULA]\n",
    "            \n",
    "            if mat_x != mat_y:\n",
    "                \n",
    "                if mat_x not in network:\n",
    "                    network[mat_x] = {}\n",
    "                \n",
    "                if mat_y not in network[mat_x]:\n",
    "                    network[mat_x][mat_y] = 0\n",
    "                \n",
    "                network[mat_x][mat_y] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Para cada matrícula, filtram-se os (as) alunos (as) com quem mais estudou-se nas mesmas turmas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "same_num_connections = {}\n",
    "\n",
    "for mat, connections in network.iteritems():\n",
    "    \n",
    "    my_connections_avg = np.mean(connections.values())\n",
    "    my_connections_max = max(connections.values())\n",
    "    my_n_connections = (my_connections_avg + my_connections_max) / 2\n",
    "        \n",
    "    my_same_n_connections = set(filter(lambda x: connections[x] >= my_n_connections, connections))\n",
    "    same_num_connections[mat] = my_same_n_connections"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As reais turmas de alunos (as) que sempre estudaram juntos e calculado usando o algoritmo Depth-First Search para grafos. Rodamos o DFS para encontrar componentes, em que cada componente corresponde justamente a um grupo de alunos (as) que se matricularam muitas vezes juntos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_on_group(set_mother, me):\n",
    "    \n",
    "    if set_mother not in same_num_connections[me]:\n",
    "        return False\n",
    "    \n",
    "    n_containing = len(list(filter(lambda adj: me in same_num_connections[adj], same_num_connections[set_mother])))\n",
    "    return float(n_containing) / len(same_num_connections[set_mother]) >= 0.9    \n",
    "    \n",
    "def dfs(m):\n",
    "    visited[m] = True\n",
    "    my_components.append(m)\n",
    "    for adj in same_num_connections[m]:\n",
    "        if not visited[adj] and is_on_group(m, adj):\n",
    "            dfs(adj)\n",
    "            \n",
    "components = []\n",
    "visited = {m:False for m in network}\n",
    "for mat, values in same_num_connections.iteritems():\n",
    "    if not visited[mat]:\n",
    "        my_components = []\n",
    "        dfs(mat)\n",
    "        components.append(my_components)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Constructing .csv to simple network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nós também salvaremos os (as) alunos (as) que formaram algum grupo de tamanho 2 ou maior. Precisaremos saber dessa informação mais à frente, quando formos construir os dados para networks que possam ser melhor customizadas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clusters = []\n",
    "has_cluster = []\n",
    "\n",
    "for component in components:\n",
    "    \n",
    "    if len(component) > 1:\n",
    "        for i in range(len(component)-1, -1, -1):\n",
    "            \n",
    "            new_row = {'source': component[i],\n",
    "                       'target': component[i-1]}\n",
    "            clusters.append(new_row)\n",
    "            \n",
    "            has_cluster.append(component[i])\n",
    "            has_cluster.append(component[i-1])\n",
    "\n",
    "has_cluster = set(has_cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output = pd.DataFrame(columns=('source', 'target'))\n",
    "output = pd.DataFrame(clusters)\n",
    "output.to_csv('dados/samplenetwork.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Constructing datas (.csv's) to more customizing networks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "alunos_cc = pd.read_csv('dados/alunos_cc.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "i_group = 0\n",
    "i_row_index = 0\n",
    "nodes = []\n",
    "row_index = {}\n",
    "MATRICULA = 1\n",
    "\n",
    "for group, obs in alunos_cc.groupby(['PERIODO_INGRESSO']):\n",
    "    \n",
    "    i_group += 1\n",
    "    \n",
    "    for row_value in obs.get_values():\n",
    "        \n",
    "        matricula = row_value[MATRICULA]\n",
    "        \n",
    "        if matricula in has_cluster:\n",
    "            row_index[matricula] = i_row_index\n",
    "            new_row = {'name': group, 'group': i_group, 'size': 1}\n",
    "            nodes.append(new_row)\n",
    "            i_row_index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "links = []\n",
    "\n",
    "for component in components:\n",
    "    if len(component) > 1:\n",
    "        for i in range(len(component)-1, -1, -1):\n",
    "            mat_x = component[i]\n",
    "            mat_y = component[i-1]\n",
    "            node_x = row_index[mat_x]\n",
    "            node_y = row_index[mat_y]\n",
    "            new_row = {'source': node_x, 'target': node_y, 'value': 1}\n",
    "            links.append(new_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = pd.DataFrame(columns=('name', 'group', 'value'))\n",
    "output = pd.DataFrame(nodes)\n",
    "output.to_csv('dados/network_nodes.csv', index=False)\n",
    "\n",
    "output = pd.DataFrame(columns=('source', 'target', 'value'))\n",
    "output = pd.DataFrame(links)\n",
    "output.to_csv('dados/network_links.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}