Depression-Detector.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"http://knoesis.org/resources/images/knoesis_depression_logo.jpg\" alt=\"Knoesis Depression Project Logo\" style=\"float:right;width: 250px;\"/>\n",
    "\n",
    "# Social-media Depression Detector (SDD)\n",
    "\n",
    "#### This notebook executes the code developed to detect depression using the ssToT method introduced in our ASONAM 2017 paper titled \"Semi-Supervised Approach to Monitoring Clinical Depressive Symptoms in Social Media\"\n",
    "\n",
    "This software is open-source, released under the terms of GPL-3.0 and CreativesForGood licenses.\n",
    "\n",
    "##### Author: Hussein S. Al-Olimat (github.com/halolimat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re, json, string, datetime, random, itertools\n",
    "from collections import OrderedDict, defaultdict\n",
    "\n",
    "# You should install the following libraries\n",
    "import wordsegment #https://pypi.python.org/pypi/wordsegment\n",
    "from nltk import TweetTokenizer #http://www.nltk.org/api/nltk.tokenize.html\n",
    "import tweepy #https://github.com/tweepy/tweepy\n",
    "from textblob import TextBlob #https://textblob.readthedocs.io/en/dev/\n",
    "from gensim import corpora #https://radimrehurek.com/gensim/\n",
    "import pandas as pd #http://pandas.pydata.org/\n",
    "import numpy as NP #http://www.numpy.org/\n",
    "import matplotlib.pyplot as plt #https://matplotlib.org/\n",
    "\n",
    "# You should install pSSLDA in order to be able to run this program and import these libraries\n",
    "#     follow the instruction in: https://github.com/davidandrzej/pSSLDA\n",
    "import FastLDA\n",
    "from pSSLDA import infer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "##### Preparing the depression lexicon to seed the LDA topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read Depression PHQ-9 Lexicon (DPL) from json file\n",
    "with open(\"depression_lexicon.json\") as f:\n",
    "    seed_terms = json.load(f)\n",
    "\n",
    "# read all seed terms into a list removing the underscore from all seeds\n",
    "all_seeds_raw = [seed.replace(\"_\",\" \").encode('utf-8') for seed in list(itertools.chain.from_iterable([seed_terms[signal] for signal in seed_terms.keys()]))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "##### Preparing other lexicons and resources to be used in filtering and preprocessing the tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "# Other lexicons and resources\n",
    "emojies = [\":‑)\", \":)\", \":D\", \":o)\", \":]\", \":3\", \":c)\", \":>\", \"=]\", \"8)\", \"=)\", \":}\", \":^)\", \":っ)\", \":‑D\", \"8‑D\", \"8D\", \"x‑D\", \"xD\", \"X‑D\", \"XD\", \"=‑D\", \"=D\", \"=‑3\", \"=3\", \"B^D\", \":-))\", \">:[\", \":‑(\", \":(\", \":‑c\", \":c\", \":‑<\", \":っC\", \":<\", \":‑[\", \":[\", \":{\", \";(\", \":-||\", \":@\", \">:(\", \":'‑(\", \":'(\", \":'‑)\", \":')\", \"D:<\", \"D:\", \"D8\", \"D;\", \"D=\", \"DX\", \"v.v\", \"D‑':\", \">:O\", \":‑O\", \":O\", \":‑o\", \":o\", \"8‑0\", \"O_O\", \"o‑o\", \"O_o\", \"o_O\", \"o_o\", \"O-O\", \":*\", \":-*\", \":^*\", \"(\", \"}{'\", \")\", \";‑)\", \";)\", \"*-)\", \"*)\", \";‑]\", \";]\", \";D\", \";^)\", \":‑,\", \">:P\", \":‑P\", \":P\", \"X‑P\", \"x‑p\", \"xp\", \"XP\", \":‑p\", \":p\", \"=p\", \":‑Þ\", \":Þ\", \":þ\", \":‑þ\", \":‑b\", \":b\", \"d:\", \">:\\\\\", \">:/\", \":‑/\", \":‑.\", \":/\", \":\\\\\", \"=/\", \"=\\\\\", \":L\", \"=L\", \":S\", \">.<\", \":|\", \":‑|\", \":$\", \":‑X\", \":X\", \":‑#\", \":#\", \"O:‑)\", \"0:‑3\", \"0:3\", \"0:‑)\", \"0:)\", \"0;^)\", \">:)\", \">;)\", \">:‑)\", \"}:‑)\", \"}:)\", \"3:‑)\", \"3:)\", \"o/\\o\", \"^5\", \">_>^\", \"^<_<\", \"|;‑)\", \"|‑O\", \":‑J\", \":‑&\", \":&\", \"#‑)\", \"%‑)\", \"%)\", \":‑###..\", \":###..\", \"<:‑|\", \"<*)))‑{\", \"><(((*>\", \"><>\", \"\\o/\", \"*\\0/*\", \"@}‑;‑'‑‑‑\", \"@>‑‑>‑‑\", \"~(_8^(I)\", \"5:‑)\", \"~:‑\\\\\", \"//0‑0\\\\\\\\\", \"*<|:‑)\", \"=:o]\", \"7:^]\", \",:‑)\", \"</3\", \"<3\"]\n",
    "\n",
    "# Tweet tokenizer from NLTK: http://www.nltk.org/_modules/nltk/tokenize/casual.html#TweetTokenizer\n",
    "nltk_tok = TweetTokenizer(preserve_case=True, reduce_len=True, strip_handles=True)\n",
    "\n",
    "printable = set(string.printable)\n",
    "\n",
    "punctuation = list(string.punctuation)\n",
    "punctuation.remove(\"-\")\n",
    "punctuation.remove('_')\n",
    "\n",
    "long_stop_list = [\"a\", \"a's\", \"abaft\", \"able\", \"aboard\", \"about\", \"above\", \"abst\", \"accordance\", \"according\", \"accordingly\", \"across\", \"act\", \"actually\", \"added\", \"adj\", \"affected\", \"affecting\", \"affects\", \"afore\", \"aforesaid\", \"after\", \"afterwards\", \"again\", \"against\", \"agin\", \"ago\", \"ah\", \"ain't\", \"aint\", \"albeit\", \"all\", \"allow\", \"allows\", \"almost\", \"alone\", \"along\", \"alongside\", \"already\", \"also\", \"although\", \"always\", \"am\", \"american\", \"amid\", \"amidst\", \"among\", \"amongst\", \"an\", \"and\", \"anent\", \"announce\", \"another\", \"any\", \"anybody\", \"anyhow\", \"anymore\", \"anyone\", \"anything\", \"anyway\", \"anyways\", \"anywhere\", \"apart\", \"apparently\", \"appear\", \"appreciate\", \"appropriate\", \"approximately\", \"are\", \"aren\", \"aren't\", \"arent\", \"arise\", \"around\", \"as\", \"aside\", \"ask\", \"asking\", \"aslant\", \"associated\", \"astride\", \"at\", \"athwart\", \"auth\", \"available\", \"away\", \"awfully\", \"b\", \"back\", \"bar\", \"barring\", \"be\", \"became\", \"because\", \"become\", \"becomes\", \"becoming\", \"been\", \"before\", \"beforehand\", \"begin\", \"beginning\", \"beginnings\", \"begins\", \"behind\", \"being\", \"believe\", \"below\", \"beneath\", \"beside\", \"besides\", \"best\", \"better\", \"between\", \"betwixt\", \"beyond\", \"biol\", \"both\", \"brief\", \"briefly\", \"but\", \"by\", \"c\", \"c'mon\", \"c's\", \"ca\", \"came\", \"can\", \"can't\", \"cannot\", \"cant\", \"cause\", \"causes\", \"certain\", \"certainly\", \"changes\", \"circa\", \"clearly\", \"close\", \"co\", \"com\", \"come\", \"comes\", \"concerning\", \"consequently\", \"consider\", \"considering\", \"contain\", \"containing\", \"contains\", \"corresponding\", \"cos\", \"could\", \"couldn\", \"couldn't\", \"couldnt\", \"couldst\", \"course\", \"currently\", \"d\", \"dare\", \"dared\", \"daren\", \"dares\", \"daring\", \"date\", \"definitely\", \"described\", \"despite\", \"did\", \"didn\", \"didn't\", \"different\", \"directly\", \"do\", \"does\", \"doesn\", \"doesn't\", \"doing\", \"don\", \"don't\", \"done\", \"dost\", \"doth\", \"down\", \"downwards\", \"due\", \"during\", \"durst\", \"e\", \"each\", \"early\", \"ed\", \"edu\", \"effect\", \"eg\", \"eight\", \"eighty\", \"either\", \"else\", \"elsewhere\", \"em\", \"end\", \"ending\", \"english\", \"enough\", \"entirely\", \"er\", \"ere\", \"especially\", \"et\", \"et-al\", \"etc\", \"even\", \"ever\", \"every\", \"everybody\", \"everyone\", \"everything\", \"everywhere\", \"ex\", \"exactly\", \"example\", \"except\", \"excepting\", \"f\", \"failing\", \"far\", \"few\", \"ff\", \"fifth\", \"first\", \"five\", \"fix\", \"followed\", \"following\", \"follows\", \"for\", \"former\", \"formerly\", \"forth\", \"found\", \"four\", \"from\", \"further\", \"furthermore\", \"g\", \"gave\", \"get\", \"gets\", \"getting\", \"give\", \"given\", \"gives\", \"giving\", \"go\", \"goes\", \"going\", \"gone\", \"gonna\", \"got\", \"gotta\", \"gotten\", \"greetings\", \"h\", \"had\", \"hadn\", \"hadn't\", \"happens\", \"hard\", \"hardly\", \"has\", \"hasn\", \"hasn't\", \"hast\", \"hath\", \"have\", \"haven\", \"haven't\", \"having\", \"he\", \"he'd\", \"he'll\", \"he's\", \"hed\", \"hello\", \"help\", \"hence\", \"her\", \"here\", \"here's\", \"hereafter\", \"hereby\", \"herein\", \"heres\", \"hereupon\", \"hers\", \"herself\", \"hes\", \"hi\", \"hid\", \"high\", \"him\", \"himself\", \"his\", \"hither\", \"home\", \"hopefully\", \"how\", \"how's\", \"howbeit\", \"however\", \"hundred\", \"i\", \"i'd\", \"i'll\", \"i'm\", \"i've\", \"id\", \"ie\", \"if\", \"ignored\", \"ill\", \"im\", \"immediate\", \"immediately\", \"importance\", \"important\", \"in\", \"inasmuch\", \"inc\", \"indeed\", \"index\", \"indicate\", \"indicated\", \"indicates\", \"information\", \"inner\", \"inside\", \"insofar\", \"instantly\", \"instead\", \"into\", \"invention\", \"inward\", \"is\", \"isn\", \"isn't\", \"it\", \"it'd\", \"it'll\", \"it's\", \"itd\", \"its\", \"itself\", \"j\", \"just\", \"k\", \"keep\", \"keeps\", \"kept\", \"kg\", \"km\", \"know\", \"known\", \"knows\", \"l\", \"large\", \"largely\", \"last\", \"lately\", \"later\", \"latter\", \"latterly\", \"least\", \"left\", \"less\", \"lest\", \"let\", \"let's\", \"lets\", \"like\", \"liked\", \"likely\", \"likewise\", \"line\", \"little\", \"living\", \"ll\", \"long\", \"look\", \"looking\", \"looks\", \"ltd\", \"m\", \"made\", \"mainly\", \"make\", \"makes\", \"many\", \"may\", \"maybe\", \"mayn\", \"me\", \"mean\", \"means\", \"meantime\", \"meanwhile\", \"merely\", \"mg\", \"mid\", \"midst\", \"might\", \"mightn\", \"million\", \"mine\", \"minus\", \"miss\", \"ml\", \"more\", \"moreover\", \"most\", \"mostly\", \"mr\", \"mrs\", \"much\", \"mug\", \"must\", \"mustn\", \"mustn't\", \"my\", \"myself\", \"n\", \"na\", \"name\", \"namely\", \"nay\", \"nd\", \"near\", \"nearly\", \"neath\", \"necessarily\", \"necessary\", \"need\", \"needed\", \"needing\", \"needn\", \"needs\", \"neither\", \"never\", \"nevertheless\", \"new\", \"next\", \"nigh\", \"nigher\", \"nighest\", \"nine\", \"ninety\", \"nisi\", \"no\", \"nobody\", \"non\", \"none\", \"nonetheless\", \"noone\", \"nor\", \"normally\", \"nos\", \"not\", \"noted\", \"nothing\", \"notwithstanding\", \"novel\", \"now\", \"nowhere\", \"o\", \"obtain\", \"obtained\", \"obviously\", \"of\", \"off\", \"often\", \"oh\", \"ok\", \"okay\", \"old\", \"omitted\", \"on\", \"once\", \"one\", \"ones\", \"oneself\", \"only\", \"onto\", \"open\", \"or\", \"ord\", \"other\", \"others\", \"otherwise\", \"ought\", \"oughtn\", \"our\", \"ours\", \"ourselves\", \"out\", \"outside\", \"over\", \"overall\", \"owing\", \"own\", \"p\", \"page\", \"pages\", \"part\", \"particular\", \"particularly\", \"past\", \"pending\", \"per\", \"perhaps\", \"placed\", \"please\", \"plus\", \"poorly\", \"possible\", \"possibly\", \"potentially\", \"pp\", \"predominantly\", \"present\", \"presumably\", \"previously\", \"primarily\", \"probably\", \"promptly\", \"proud\", \"provided\", \"provides\", \"providing\", \"public\", \"put\", \"q\", \"qua\", \"que\", \"quickly\", \"quite\", \"qv\", \"r\", \"ran\", \"rather\", \"rd\", \"re\", \"readily\", \"real\", \"really\", \"reasonably\", \"recent\", \"recently\", \"ref\", \"refs\", \"regarding\", \"regardless\", \"regards\", \"related\", \"relatively\", \"research\", \"respecting\", \"respectively\", \"resulted\", \"resulting\", \"results\", \"right\", \"round\", \"run\", \"s\", \"said\", \"same\", \"sans\", \"save\", \"saving\", \"saw\", \"say\", \"saying\", \"says\", \"sec\", \"second\", \"secondly\", \"section\", \"see\", \"seeing\", \"seem\", \"seemed\", \"seeming\", \"seems\", \"seen\", \"self\", \"selves\", \"sensible\", \"sent\", \"serious\", \"seriously\", \"seven\", \"several\", \"shall\", \"shalt\", \"shan\", \"shan't\", \"she\", \"she'd\", \"she'll\", \"she's\", \"shed\", \"shell\", \"shes\", \"short\", \"should\", \"shouldn\", \"shouldn't\", \"show\", \"showed\", \"shown\", \"showns\", \"shows\", \"significant\", \"significantly\", \"similar\", \"similarly\", \"since\", \"six\", \"slightly\", \"small\", \"so\", \"some\", \"somebody\", \"somehow\", \"someone\", \"somethan\", \"something\", \"sometime\", \"sometimes\", \"somewhat\", \"somewhere\", \"soon\", \"sorry\", \"special\", \"specifically\", \"specified\", \"specify\", \"specifying\", \"still\", \"stop\", \"strongly\", \"sub\", \"substantially\", \"successfully\", \"such\", \"sufficiently\", \"suggest\", \"summat\", \"sup\", \"supposing\", \"sure\", \"t\", \"t's\", \"take\", \"taken\", \"taking\", \"tell\", \"tends\", \"th\", \"than\", \"thank\", \"thanks\", \"thanx\", \"that\", \"that'll\", \"that's\", \"that've\", \"thats\", \"the\", \"thee\", \"their\", \"theirs\", \"them\", \"themselves\", \"then\", \"thence\", \"there\", \"there'll\", \"there's\", \"there've\", \"thereafter\", \"thereby\", \"thered\", \"therefore\", \"therein\", \"thereof\", \"therere\", \"theres\", \"thereto\", \"thereupon\", \"these\", \"they\", \"they'd\", \"they'll\", \"they're\", \"they've\", \"theyd\", \"theyre\", \"thine\", \"think\", \"third\", \"this\", \"tho\", \"thorough\", \"thoroughly\", \"those\", \"thou\", \"though\", \"thoughh\", \"thousand\", \"three\", \"thro\", \"throug\", \"through\", \"throughout\", \"thru\", \"thus\", \"thyself\", \"til\", \"till\", \"tip\", \"to\", \"today\", \"together\", \"too\", \"took\", \"touching\", \"toward\", \"towards\", \"tried\", \"tries\", \"true\", \"truly\", \"try\", \"trying\", \"ts\", \"twas\", \"tween\", \"twere\", \"twice\", \"twill\", \"twixt\", \"two\", \"twould\", \"u\", \"un\", \"under\", \"underneath\", \"unfortunately\", \"unless\", \"unlike\", \"unlikely\", \"until\", \"unto\", \"up\", \"upon\", \"ups\", \"us\", \"use\", \"used\", \"useful\", \"usefully\", \"usefulness\", \"uses\", \"using\", \"usually\", \"v\", \"value\", \"various\", \"ve\", \"versus\", \"very\", \"via\", \"vice\", \"vis-a-vis\", \"viz\", \"vol\", \"vols\", \"vs\", \"w\", \"wanna\", \"want\", \"wanting\", \"wants\", \"was\", \"wasn\", \"wasn't\", \"wasnt\", \"way\", \"we\", \"we'd\", \"we'll\", \"we're\", \"we've\", \"wed\", \"welcome\", \"well\", \"went\", \"were\", \"weren\", \"weren't\", \"werent\", \"wert\", \"what\", \"what'll\", \"what's\", \"whatever\", \"whats\", \"when\", \"when's\", \"whence\", \"whencesoever\", \"whenever\", \"where\", \"where's\", \"whereafter\", \"whereas\", \"whereby\", \"wherein\", \"wheres\", \"whereupon\", \"wherever\", \"whether\", \"which\", \"whichever\", \"whichsoever\", \"while\", \"whilst\", \"whim\", \"whither\", \"who\", \"who'll\", \"who's\", \"whod\", \"whoever\", \"whole\", \"whom\", \"whomever\", \"whore\", \"whos\", \"whose\", \"whoso\", \"whosoever\", \"why\", \"why's\", \"widely\", \"will\", \"willing\", \"wish\", \"with\", \"within\", \"without\", \"won't\", \"wonder\", \"wont\", \"words\", \"world\", \"would\", \"wouldn\", \"wouldn't\", \"wouldnt\", \"wouldst\", \"www\", \"x\", \"y\", \"ye\", \"yes\", \"yet\", \"you\", \"you'd\", \"you'll\", \"you're\", \"you've\", \"youd\", \"your\", \"youre\", \"yours\", \"yourself\", \"yourselves\", \"z\", \"zero\"]\n",
    "stoplist = long_stop_list + punctuation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "---\n",
    "\n",
    "##### Now, to detect depression for a single Twitter user we should crawl all their tweets using the Twitter search API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This function is taken from https://github.com/marado/tweet_dumper\n",
    "def get_all_tweets(screen_name):\n",
    "    \n",
    "    #Twitter API credentials - https://apps.twitter.com/\n",
    "    consumer_key=''\n",
    "    consumer_secret=''\n",
    "    access_key=''\n",
    "    access_secret=''\n",
    "    \n",
    "    if (consumer_key == \"\"):\n",
    "        print \"You need to set up the script first. Edit it and add your keys.\"\n",
    "        return\n",
    "\n",
    "    # Twitter only allows access to a users most recent 3240 tweets with this method\n",
    "\n",
    "    # authorize twitter, initialize tweepy\n",
    "    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "    auth.set_access_token(access_key, access_secret)\n",
    "    api = tweepy.API(auth)\n",
    "\n",
    "    # initialize a list to hold all the tweepy Tweets\n",
    "    alltweets = []\n",
    "\n",
    "    # make initial request for most recent tweets (200 is the maximum allowed count)\n",
    "    new_tweets = api.user_timeline(screen_name=screen_name, count=200)\n",
    "\n",
    "    # save most recent tweets\n",
    "    alltweets.extend(new_tweets)\n",
    "\n",
    "    # save the id of the oldest tweet less one\n",
    "    oldest = alltweets[-1].id - 1\n",
    "\n",
    "    # keep grabbing tweets until there are no tweets left to grab\n",
    "    while len(new_tweets) > 0:\n",
    "        print \"getting tweets before %s\" % (oldest)\n",
    "\n",
    "        # all subsiquent requests use the max_id param to prevent duplicates\n",
    "        new_tweets = api.user_timeline(screen_name=screen_name, count=200, max_id=oldest)\n",
    "\n",
    "        # save most recent tweets\n",
    "        alltweets.extend(new_tweets)\n",
    "\n",
    "        # update the id of the oldest tweet less one\n",
    "        oldest = alltweets[-1].id - 1\n",
    "\n",
    "        print \"...%s tweets downloaded so far\" % (len(alltweets))\n",
    "        \n",
    "    # transform the tweepy tweets into a 2D array that will populate the csv\t\n",
    "    outtweets = [[tweet.id_str, tweet.created_at, tweet.text.encode(\"utf-8\")] for tweet in alltweets]\n",
    "    \n",
    "    return outtweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose the screen name here and run the cell to collect all tweets\n",
    "screen_name = \"\"\n",
    "\n",
    "if screen_name == \"\":\n",
    "    print \"You need to add a screen name first!\"\n",
    "\n",
    "account_tweets = get_all_tweets(screen_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "##### Now, we should preprocess tweets by filtering the text and recording the sentiments of each tweet\n",
    "\n",
    "Output format: ``` [tweet_ID, created_at, raw_text, cleaned_text, sentiment]```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess_text(tweet):\n",
    "\n",
    "    # this will replace seeds (as phrases) as unigrams. lack of > lack_of\n",
    "    for seed in all_seeds_raw:\n",
    "        if seed in tweet and \" \" in seed:\n",
    "            tweet = tweet.replace(seed, seed.replace(\" \", \"_\"))\n",
    "\n",
    "    # remove retweet handler\n",
    "    if tweet[:2] == \"RT\":\n",
    "        try:\n",
    "            colon_idx = tweet.index(\":\")\n",
    "            tweet = tweet[colon_idx+2:]\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "    # remove url from tweet\n",
    "    tweet = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '', tweet)\n",
    "\n",
    "    # remove non-ascii characters\n",
    "    tweet = filter(lambda x: x in printable, tweet)\n",
    "\n",
    "    # additional preprocessing\n",
    "    tweet = tweet.replace(\"\\n\", \" \").replace(\" https\",\"\").replace(\"http\",\"\")\n",
    "\n",
    "    # remove all mentions in tweet\n",
    "    mentions = re.findall(r\"@\\w+\", tweet)\n",
    "    for mention in mentions:\n",
    "        tweet = tweet.replace(mention, \"\")\n",
    "\n",
    "    # break usernames and hashtags +++++++++++++\n",
    "    for term in re.findall(r\"#\\w+\", tweet):\n",
    "\n",
    "        token = term[1:]\n",
    "\n",
    "        # remove any punctuations from the hashtag and mention\n",
    "        # ex: Troll_Cinema => TrollCinema\n",
    "        token = token.translate(None, ''.join(string.punctuation))\n",
    "\n",
    "        segments = wordsegment.segment(token)\n",
    "        segments = ' '.join(segments)\n",
    "\n",
    "        tweet = tweet.replace(term, segments)\n",
    "\n",
    "    # remove all punctuations from the tweet text\n",
    "    tweet = \"\".join([char for char in tweet if char not in punctuation])\n",
    "\n",
    "    # remove trailing spaces\n",
    "    tweet = tweet.strip()\n",
    "\n",
    "    # remove all tokens in the tweet where the token is\n",
    "    # a stop word or an emoji\n",
    "    tweet = [word.lower() for word in nltk_tok.tokenize(tweet) if word.lower() not in stoplist \n",
    "                and word.lower() not in emojies and len(word) > 1]\n",
    "\n",
    "    tweet = \" \".join(tweet)\n",
    "\n",
    "    # remove numbers\n",
    "    tweet = re.sub(r'[\\d-]+', 'NUM', tweet)\n",
    "    # padding NUM with spaces\n",
    "    tweet = tweet.replace(\"NUM\", \" NUM \")\n",
    "    # remove multiple spaces in tweet text\n",
    "    tweet = re.sub('\\s{2,}', ' ', tweet)\n",
    "\n",
    "    return tweet\n",
    "\n",
    "\n",
    "# input: [Tweet_ID, created_at, text]\n",
    "def preprocess(account_tweets):\n",
    "    \n",
    "    preprocessed_tweets = list()\n",
    "    \n",
    "    for index, tweet in enumerate(account_tweets):\n",
    "        \n",
    "        cleaned_text = preprocess_text(tweet[2])\n",
    "        sent_score = TextBlob(tweet[2].decode('ascii', errors=\"ignore\")).sentiment.polarity        \n",
    "    \n",
    "        # output: [tweet_ID, created_at, raw_text, cleaned_text, sentiment]\n",
    "        preprocessed_tweets.append([tweet[0], tweet[1], tweet[2], cleaned_text, sent_score])\n",
    "        \n",
    "        if index % 100 == 0:\n",
    "            print \".\",\n",
    "\n",
    "    return preprocessed_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_tweets = preprocess(account_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "##### Now, to emulate PHQ-9 questionare, we bucket tweets based on their creation time with a sliding window of 14 days. Each bucket will then be treated as a document when we run LDA."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_sliding_buckets_on_time(account_tweets):\n",
    "\n",
    "    size_of_bucket = 14 # days\n",
    "    \n",
    "    # convert list of lists to pandas dataframe\n",
    "    account_tweets = pd.DataFrame(account_tweets, columns=[\"tweet_ID\", \"created_at\", \"raw_text\", \n",
    "                                                           \"cleaned_text\", \"sentiment\"])\n",
    "    \n",
    "    # ensure that Created_at column is of type datetime\n",
    "    account_tweets['created_at'] = pd.to_datetime(account_tweets['created_at'])\n",
    "\n",
    "    min_date = account_tweets.created_at.min()\n",
    "    max_date = account_tweets.created_at.max()\n",
    "    max_date = max_date + datetime.timedelta(days=1)\n",
    "\n",
    "    min_date = min_date.replace(hour=0, minute=0, second=0)\n",
    "    max_date = max_date.replace(hour=0, minute=0, second=0)\n",
    "\n",
    "    new_min = min_date\n",
    "    new_max = min_date + datetime.timedelta(days=size_of_bucket)\n",
    "\n",
    "    # will contain the tweets grouped in buckets\n",
    "    bucketed_tweets = defaultdict(list)\n",
    "    \n",
    "    counter = 0\n",
    "    while True:\n",
    "        if new_max <= max_date:\n",
    "\n",
    "            mask = (account_tweets['created_at'] > new_min) & (account_tweets['created_at'] <= new_max)\n",
    "\n",
    "            df = account_tweets[mask]\n",
    "\n",
    "            for index, tweet in df.iterrows():\n",
    "\n",
    "                bucketed_tweets[counter].append(tweet)\n",
    "\n",
    "            new_min = new_min + datetime.timedelta(days=1)\n",
    "            new_max = new_max + datetime.timedelta(days=1)\n",
    "            counter += 1\n",
    "            \n",
    "            # just to show processing progress!\n",
    "            if counter % 50 == 0:\n",
    "                print \".\",\n",
    "\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    return bucketed_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucketed_tweets = build_sliding_buckets_on_time(preprocessed_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "##### Prepare the data for pSSLDA from the bucketed tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_data_for_pSSLDA(bucketed_tweets):\n",
    "\n",
    "    texts = list()\n",
    "\n",
    "    # each bucket is hashed on the start and end date\n",
    "    for bucket in bucketed_tweets:\n",
    "\n",
    "        all_bucket_tweets = \"\"\n",
    "\n",
    "        for tweet in bucketed_tweets[bucket]:\n",
    "\n",
    "            try:\n",
    "                all_bucket_tweets += tweet.cleaned_text + \" \"\n",
    "            except:\n",
    "                # some cleaned fields are None. therefore, ignore!\n",
    "                pass\n",
    "\n",
    "        texts.append(all_bucket_tweets.strip().replace(\"\\n\", \"\").split(\" \"))\n",
    "\n",
    "    # assign each word a unique ID\n",
    "    dictionary = corpora.Dictionary(texts)\n",
    "\n",
    "    # remove gaps in id sequence after words that were removed\n",
    "    dictionary.compactify()\n",
    "\n",
    "    voc_size = len(list(dictionary.keys()))\n",
    "\n",
    "    # replace token ids with the token text in each doc and return similar arry of tokens and docs\n",
    "    text_as_ids = list()\n",
    "\n",
    "    # to later be the docvec\n",
    "    doc_as_ids = list()\n",
    "\n",
    "    # number of docs here is the number of buckets\n",
    "    number_of_docs = len(bucketed_tweets)\n",
    "\n",
    "    for x in range(number_of_docs):\n",
    "\n",
    "        doc = texts[x]\n",
    "\n",
    "        for token in doc:\n",
    "            text_as_ids.append(dictionary.token2id[token])\n",
    "            doc_as_ids.append(x)\n",
    " \n",
    "    return text_as_ids, doc_as_ids, voc_size, dictionary.token2id, number_of_docs, bucketed_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# docs for us will be here multiple tweets\n",
    "pSSLDA_input = prepare_data_for_pSSLDA(bucketed_tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "##### Run pSSLDA allowing us to seed the LDA topics using our depression lexicon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# NOTE: topics and signals are used in interchangebly in this code, they both mean the same thing.\n",
    "\n",
    "# calculated the average sentiment of a token based on its occurence in a given set of tweets\n",
    "# terms sentiment is therefore taken from the tweet sentiment not targeted sentiment\n",
    "def get_avg_sentiment(bucketed_tweets, token):\n",
    "\n",
    "    term_tweets_sent_scores = get_tweets_by_term(bucketed_tweets, token)\n",
    "    \n",
    "    score = 0.0\n",
    "    count = 0\n",
    "\n",
    "    for sent_score in term_tweets_sent_scores:\n",
    "         score += float(sent_score)\n",
    "         count+=1\n",
    "\n",
    "    return score/count\n",
    "\n",
    "\n",
    "def get_tweets_by_term(bucketed_tweets, term):\n",
    "\n",
    "    term_tweets_sent_scores = list()\n",
    "\n",
    "    for bucket in bucketed_tweets:\n",
    "        for tweet in bucketed_tweets[bucket]:\n",
    "            try:\n",
    "                if term in tweet.cleaned_text:\n",
    "                    term_tweets_sent_scores.append(tweet.sentiment)\n",
    "            except:\n",
    "                # pass on empty text field\n",
    "                pass\n",
    "\n",
    "    return term_tweets_sent_scores\n",
    "\n",
    "\n",
    "def get_topics_terms(tup):\n",
    "\n",
    "    estphi = tup[0]\n",
    "    W = tup[1]\n",
    "    T = tup[2]\n",
    "    id2token = tup[3]\n",
    "\n",
    "    # This will contain the mappings of each term to each of our topics\n",
    "    # topic1 -> termX, termY ...\n",
    "    topics_dict = defaultdict(defaultdict)\n",
    "\n",
    "    print \"Reading Topics Terms: \"\n",
    "    \n",
    "    # find the topic where each term is part of\n",
    "    # W: vocabulary size\n",
    "    for index in range(W):\n",
    "        # projects one column of the matrix which contains the weight of the term in all of the topics\n",
    "        term_weights = estphi[:,index]\n",
    "\n",
    "        # will contain the largest weight which ->  topic it was assigned to\n",
    "        largest_weight = 0\n",
    "\n",
    "        for weight in term_weights:\n",
    "            if weight > largest_weight:\n",
    "                largest_weight = weight\n",
    "\n",
    "        # this will get the index of the topic with largest weight\n",
    "        term_topic = NP.argwhere(term_weights==largest_weight)[0][0]\n",
    "\n",
    "        topics_dict[term_topic][id2token[index]] = largest_weight\n",
    "\n",
    "        if index % 50 == 0:\n",
    "            print \".\",\n",
    "    \n",
    "    print \"Done Reading Topics Terms\"\n",
    "    \n",
    "    return topics_dict\n",
    "\n",
    "\n",
    "def get_all_terms_sentiments(id2token, w, bucketed_tweets):\n",
    "\n",
    "    seed_term_sentiment = defaultdict(float)\n",
    "\n",
    "    unique_w = list(set(w))\n",
    "\n",
    "    for wi in unique_w:\n",
    "        token = id2token[wi]\n",
    "\n",
    "        if token in seed_terms['signal_1']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_2']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_3']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_4']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_5']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_6']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_7']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_8']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_9']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "        elif token in seed_terms['signal_10']:\n",
    "            seed_term_sentiment[token] = get_avg_sentiment(bucketed_tweets, token)\n",
    "\n",
    "    return seed_term_sentiment\n",
    "\n",
    "# This is a modified version of the code in https://github.com/davidandrzej/pSSLDA/blob/master/example/example.py\n",
    "def run_pSSLDA(pSSLDA_input, parameters):\n",
    "    \n",
    "    print \"Running ssToT\"\n",
    "\n",
    "    token2id = pSSLDA_input[3]\n",
    "\n",
    "    # number of topics\n",
    "    T = parameters[\"topics_count\"]\n",
    "\n",
    "    (wordvec, docvec, zvec) = ([], [], [])\n",
    "\n",
    "    # vector of words per tweet\n",
    "    wordvec = pSSLDA_input[0]\n",
    "    docvec = pSSLDA_input[1]\n",
    "\n",
    "    # W = vocabulary size\n",
    "    W = pSSLDA_input[2]\n",
    "\n",
    "    (w, d) = (NP.array(wordvec, dtype = NP.int),\n",
    "              NP.array(docvec, dtype = NP.int))\n",
    "\n",
    "    # Create parameters\n",
    "    alpha = NP.ones((1,T)) * 1\n",
    "    beta = NP.ones((T,W)) * 0.01\n",
    "\n",
    "    # How many parallel samplers do we wish to use?\n",
    "    P = 10\n",
    "\n",
    "    # Random number seed\n",
    "    randseed =  random.randint(999,999999)# 194582\n",
    "\n",
    "    # Number of samples to take\n",
    "    numsamp = 500\n",
    "\n",
    "    # Do parallel inference\n",
    "    finalz = infer(w, d, alpha, beta, numsamp, randseed, P)\n",
    "\n",
    "    # number of documents = tweets\n",
    "    D = pSSLDA_input[4]\n",
    "\n",
    "    # Estimate phi and theta\n",
    "    (nw, nd) = FastLDA.countMatrices(w, W, d, D, finalz, T)\n",
    "    (estphi,esttheta) = FastLDA.estPhiTheta(nw, nd, alpha, beta)\n",
    "\n",
    "    # ======================================================================\n",
    "\n",
    "    # swap keys with values in the token2id => id2token\n",
    "    id2token = dict((v,k) for k,v in token2id.iteritems())\n",
    "\n",
    "    seed_term_sentiment = get_all_terms_sentiments(id2token, w, pSSLDA_input[5])\n",
    "\n",
    "    # ----------------------------------------------------------------------\n",
    "    \n",
    "    # Now, we add z-labels to *force* words into separate topics\n",
    "    \n",
    "    labelweight = 5.0\n",
    "\n",
    "    label0 = NP.zeros((T,), dtype=NP.float)\n",
    "    label0[0] = labelweight\n",
    "\n",
    "    label1 = NP.zeros((T,), dtype=NP.float)\n",
    "    label1[1] = labelweight\n",
    "\n",
    "    label2 = NP.zeros((T,), dtype=NP.float)\n",
    "    label2[2] = labelweight\n",
    "\n",
    "    label3 = NP.zeros((T,), dtype=NP.float)\n",
    "    label3[3] = labelweight\n",
    "\n",
    "    label4 = NP.zeros((T,), dtype=NP.float)\n",
    "    label4[4] = labelweight\n",
    "\n",
    "    label5 = NP.zeros((T,), dtype=NP.float)\n",
    "    label5[5] = labelweight\n",
    "\n",
    "    label6 = NP.zeros((T,), dtype=NP.float)\n",
    "    label6[6] = labelweight\n",
    "\n",
    "    label7 = NP.zeros((T,), dtype=NP.float)\n",
    "    label7[7] = labelweight\n",
    "\n",
    "    label8 = NP.zeros((T,), dtype=NP.float)\n",
    "    label8[8] = labelweight\n",
    "\n",
    "    label9 = NP.zeros((T,), dtype=NP.float)\n",
    "    label9[9] = labelweight\n",
    "\n",
    "    label10 = NP.zeros((T,), dtype=NP.float)\n",
    "    label10[10] = labelweight\n",
    "\n",
    "    label11 = NP.zeros((T,), dtype=NP.float)\n",
    "    label11[11] = labelweight\n",
    "\n",
    "    # signals ids\n",
    "    corpus_signals = [0,1,2,3,4,5,6,7,8,9]\n",
    "   \n",
    "    seed_terms_per_signal = defaultdict(lambda: defaultdict(int))\n",
    "\n",
    "    zlabels = []\n",
    "    for wi in w:\n",
    "\n",
    "        token = id2token[wi]\n",
    "\n",
    "        # if the word appears in topic 0\n",
    "        if token in seed_terms['signal_1'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label0)\n",
    "\n",
    "            seed_terms_per_signal['signal_1'][token]+=1\n",
    "\n",
    "            if 0 in corpus_signals:\n",
    "                corpus_signals.remove(0)\n",
    "\n",
    "\n",
    "        elif token in seed_terms['signal_2'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label1)\n",
    "\n",
    "            seed_terms_per_signal['signal_2'][token]+=1\n",
    "\n",
    "            if 1 in corpus_signals:\n",
    "                corpus_signals.remove(1)\n",
    "\n",
    "\n",
    "        elif token in seed_terms['signal_3'] and seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label2)\n",
    "\n",
    "            seed_terms_per_signal['signal_3'][token]+=1\n",
    "\n",
    "            if 2 in corpus_signals:\n",
    "                corpus_signals.remove(2)\n",
    "\n",
    "\n",
    "        elif token in seed_terms['signal_4'] and seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label3)\n",
    "            seed_terms_per_signal['signal_4'][token]+=1\n",
    "\n",
    "            if 3 in corpus_signals:\n",
    "                corpus_signals.remove(3)\n",
    "\n",
    "\n",
    "        elif token in seed_terms['signal_5'] and seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label4)\n",
    "\n",
    "            seed_terms_per_signal['signal_5'][token]+=1\n",
    "\n",
    "            if 4 in corpus_signals:\n",
    "                corpus_signals.remove(4)\n",
    "\n",
    "        elif token in seed_terms['signal_6'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label5)\n",
    "\n",
    "            seed_terms_per_signal['signal_6'][token]+=1\n",
    "\n",
    "            if 5 in corpus_signals:\n",
    "                corpus_signals.remove(5)\n",
    "\n",
    "        elif token in seed_terms['signal_7'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label6)\n",
    "\n",
    "            seed_terms_per_signal['signal_7'][token]+=1\n",
    "\n",
    "            if 6 in corpus_signals:\n",
    "                corpus_signals.remove(6)\n",
    "\n",
    "        elif token in seed_terms['signal_8'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label7)\n",
    "\n",
    "            seed_terms_per_signal['signal_8'][token]+=1\n",
    "\n",
    "            if 7 in corpus_signals:\n",
    "                corpus_signals.remove(7)\n",
    "\n",
    "        elif token in seed_terms['signal_9'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label8)\n",
    "\n",
    "            seed_terms_per_signal['signal_9'][token]+=1\n",
    "\n",
    "            if 8 in corpus_signals:\n",
    "                corpus_signals.remove(8)\n",
    "\n",
    "        elif token in seed_terms['signal_10'] and  seed_term_sentiment[token] <= 0:\n",
    "\n",
    "            zlabels.append(label9)\n",
    "\n",
    "            seed_terms_per_signal['signal_10'][token]+=1\n",
    "\n",
    "            if 9 in corpus_signals:\n",
    "                corpus_signals.remove(9)\n",
    "\n",
    "        else:\n",
    "            zlabels.append(None)\n",
    "\n",
    "\n",
    "    # --------------------------------------------------------------------\n",
    "\n",
    "    # Now inference will find topics with 0 and 1 in separate topics\n",
    "    finalz = infer(w, d, alpha, beta, numsamp, randseed, P, zlabels = zlabels)\n",
    "\n",
    "    # Re-estimate phi and theta\n",
    "    (nw, nd) = FastLDA.countMatrices(w, W, d, D, finalz, T)\n",
    "    (estphi,esttheta) = FastLDA.estPhiTheta(nw, nd, alpha, beta)\n",
    "\n",
    "    # --------------------------------------------------------------------\n",
    "    \n",
    "    # Find the sentiment of each topic cluster based on the tweets where each seed term appered in\n",
    "\n",
    "    tup = (estphi, W, T, id2token)\n",
    "    topics_terms = get_topics_terms(tup)\n",
    "    \n",
    "    # --------------------------------------------------------------------\n",
    "    \n",
    "    # TODO: refactor this subroutine to make it faster, use inverted index!\n",
    "    \n",
    "    sent_scores = defaultdict(list)\n",
    "\n",
    "    print \"Calculating topics sentiments: \"\n",
    "    \n",
    "    counter = 0\n",
    "    for topic in topics_terms:\n",
    "\n",
    "        topic_sent_scores = list()\n",
    "\n",
    "        for term in topics_terms[topic]:\n",
    "            term_tweets_sent_scores = get_tweets_by_term(pSSLDA_input[5], term)\n",
    "\n",
    "            for sent_score in term_tweets_sent_scores:\n",
    "                 topic_sent_scores.append(float(sent_score))\n",
    "\n",
    "        avg = sum(topic_sent_scores) / float(len(topic_sent_scores))\n",
    "\n",
    "        sent_scores[topic] = (topic_sent_scores, avg)\n",
    "        \n",
    "        counter+=1\n",
    "        print \".\",\n",
    "\n",
    "    # --------------------------------------------------------------------\n",
    "        \n",
    "    # post processing of topics. If the bucket has less than 30 tweets then\n",
    "    # discard the probabilities of that bucket\n",
    "\n",
    "    len_buckets = []\n",
    "    for bucket in pSSLDA_input[5]:\n",
    "        len_b = len(pSSLDA_input[5][bucket])\n",
    "        len_buckets.append(len_b)\n",
    "\n",
    "   \n",
    "    # threshold #1: if number of tweets in that bucket is less than x, then discard that bucket.\n",
    "    min_number_of_tweets_per_bucket = parameters[\"min_tweets_per_bucket\"]\n",
    "    \n",
    "    for x in range(len(len_buckets)):\n",
    "        if len_buckets[x] <= min_number_of_tweets_per_bucket:\n",
    "            esttheta[x, :] = 0\n",
    "\n",
    "    # this will replace zero to the probabilities of the topic by ID if no seed terms were found in the corpus\n",
    "    for topic_id in corpus_signals:\n",
    "        esttheta[:, topic_id] = 0\n",
    "\n",
    "    all_topics_seeds = list()\n",
    "    for signal in seed_terms_per_signal:\n",
    "        all_topics_seeds += seed_terms_per_signal[signal]\n",
    "\n",
    "    # topics to keep\n",
    "    seeds_in_top_k = defaultdict(int)\n",
    "\n",
    "    # number of seed terms that should be in the top topic terms\n",
    "    seeds_threshold = parameters[\"seeds_threshold\"]\n",
    "    # The number of terms in the topic that we will look into to search for seed terms\n",
    "    top_topic_terms = parameters[\"top_topic_terms\"]\n",
    "\n",
    "    for topic in topics_terms:\n",
    "        for x in range(len(topics_terms[topic])):\n",
    "            term = list(topics_terms[topic])[x]\n",
    "            if x < top_topic_terms:\n",
    "                if term in all_topics_seeds:\n",
    "                    seeds_in_top_k[topic] += 1\n",
    "\n",
    "    # this will replace zero to the probabilities of the topic by ID if no seed terms were found in the corpus\n",
    "    for x in range(len(esttheta[0])):\n",
    "        if x in seeds_in_top_k.keys():\n",
    "            if seeds_in_top_k[x] < seeds_threshold:\n",
    "                esttheta[:, x] = 0\n",
    "        else:\n",
    "            esttheta[:, x] = 0\n",
    "\n",
    "\n",
    "    return (estphi, W, T, id2token), esttheta, topics_terms, seed_terms_per_signal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' \n",
    "    topics_count: number of topics/signals to construct using pSSLDA\n",
    "    min_tweets_per_bucket: minimum number of tweets per bucket to constructs and accept a topic from it\n",
    "    seeds_threshold: number of seed terms in the top topic terms\n",
    "    top_topic_terms: the number of terms to consider when searching for seed terms\n",
    "'''\n",
    "parameters = {\"topics_count\": 15, \"min_tweets_per_bucket\": 20, \"seeds_threshold\": 2, \"top_topic_terms\": 25}\n",
    "\n",
    "# pSSLDA_input: 0> estphi_tup | 1> esttheta | 2> topics_terms | 3> seed_terms_per_signal\n",
    "pSSLDA_output = run_pSSLDA(pSSLDA_input, parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_depression(pSSLDA_output):\n",
    "    \n",
    "    try:\n",
    "\n",
    "        esttheta = pSSLDA_output[1]\n",
    "\n",
    "        print\n",
    "        print \"<<<<<<< Topics Probabilties Over Time >>>>>>>\"\n",
    "        print \n",
    "                \n",
    "        headers = [\"Time Period\", \"Signal-1\", \"Signal-2\", \"Signal-3\", \"Signal-4\", \"Signal-5\",\n",
    "                                  \"Signal-6\", \"Signal-7\", \"Signal-8\", \"Signal-9\", \"Signal-10\"]\n",
    "        \n",
    "        rows = list()\n",
    "        \n",
    "        counter = 0\n",
    "        for key in bucketed_tweets.keys():\n",
    "            \n",
    "            # list of series to dataframe\n",
    "            df = pd.DataFrame(bucketed_tweets[key])\n",
    "                        \n",
    "            bucket_date = str(df.created_at.min().strftime(\"%d/%m/%Y\")) + \" To \" + \\\n",
    "                          str(df.created_at.max().strftime(\"%d/%m/%Y\"))\n",
    "            \n",
    "            row = [bucket_date] + [esttheta[counter][x] for x in range(len(esttheta[counter])) if x < 10]\n",
    "            \n",
    "            rows.append(row)\n",
    "\n",
    "            # increment counter to get element from the result matrix\n",
    "            counter+=1\n",
    "\n",
    "        topics_probabilities = pd.DataFrame(rows, columns=headers)\n",
    "        \n",
    "        print topics_probabilities\n",
    "        \n",
    "        topics_probabilities.plot(kind='line')\n",
    "        plt.show()\n",
    "                \n",
    "        #--------------------------------------------------------------------------------------------\n",
    "\n",
    "        print\n",
    "        print \"<<<<<<< Topics Terms >>>>>>>\"\n",
    "        print \n",
    "\n",
    "        \n",
    "        headers = [\"Topic Number\", \"Topic Terms\"]\n",
    "        rows = list()\n",
    "        \n",
    "        for topic in pSSLDA_output[2]:\n",
    "\n",
    "            topic_nbr = topic+1\n",
    "            \n",
    "            rows.append([topic_nbr, \", \".join(pSSLDA_output[2][topic])])\n",
    "\n",
    "        topics_terms = pd.DataFrame(rows, columns=headers)\n",
    "                    \n",
    "        print topics_terms\n",
    "        \n",
    "        #--------------------------------------------------------------------------------------------\n",
    "\n",
    "        print\n",
    "        print \"<<<<<<< Seeded Terms Per Topic >>>>>>>\"\n",
    "        print\n",
    "\n",
    "        headers = [\"Topic Number\", \"Seed Terms:Count\"]\n",
    "        rows = list()\n",
    "        \n",
    "        # pSSLDA_output[3] = seed_terms_per_signal\n",
    "        for topic in pSSLDA_output[3]:\n",
    "            \n",
    "            seedTerms = [str(seedTerm)+\":\"+str(pSSLDA_output[3][topic][seedTerm]) \n",
    "                                         for seedTerm in pSSLDA_output[3][topic]]\n",
    "            \n",
    "            rows.append([topic, \", \".join(seedTerms)])\n",
    "        \n",
    "        topics_seeds = pd.DataFrame(rows, columns=headers)\n",
    "                    \n",
    "        print topics_seeds\n",
    "\n",
    "    except AssertionError:\n",
    "        print \"ERROR: number of tweets is insufficents for depression detection!\"\n",
    "    except Exception as e:\n",
    "        print \"ERROR >>> \", e\n",
    "        raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "detect_depression(pSSLDA_output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}