Gensim and Mallet.py.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code reference https://www.machinelearningplus.com/nlp/topic-modeling-gensim-python/ \n",
    "import string\n",
    "import re\n",
    "from pprint import pprint\n",
    "\n",
    "# NLTK \n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english') #this depends on each language\n",
    "\n",
    "# Gensim\n",
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.models import CoherenceModel\n",
    "\n",
    "# spacy for lemmatization\n",
    "import spacy\n",
    "\n",
    "# Plotting tools\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim  \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Enable logging for gensim - optional\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\",category=DeprecationWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_file(filename):\n",
    "    contents = open(filename, encoding=\"utf8\").read()\n",
    "    for ii in kTOKENIZER.tokenize(contents):\n",
    "        yield ii"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations\n",
    "\n",
    "data_words = list(sent_to_words(data))\n",
    "\n",
    "print(data_words[:1])b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the bigram and trigram models\n",
    "bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100) # higher threshold fewer phrases.\n",
    "trigram = gensim.models.Phrases(bigram[data_words], threshold=100)  \n",
    "\n",
    "# Faster way to get a sentence clubbed as a trigram/bigram\n",
    "bigram_mod = gensim.models.phrases.Phraser(bigram)\n",
    "trigram_mod = gensim.models.phrases.Phraser(trigram)\n",
    "\n",
    "# See trigram example\n",
    "print(trigram_mod[bigram_mod[data_words[0]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define functions for stopwords, bigrams, trigrams and lemmatization\n",
    "def remove_stopwords(texts):\n",
    "    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]\n",
    "\n",
    "def make_bigrams(texts):\n",
    "    return [bigram_mod[doc] for doc in texts]\n",
    "\n",
    "def make_trigrams(texts):\n",
    "    return [trigram_mod[bigram_mod[doc]] for doc in texts]\n",
    "\n",
    "def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):\n",
    "    \"\"\"https://spacy.io/api/annotation\"\"\"\n",
    "    texts_out = []\n",
    "    for sent in texts:\n",
    "        doc = nlp(\" \".join(sent)) \n",
    "        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])\n",
    "    return texts_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#To run this code it is necessary to get the spacy moidel: \"!python - m spacy download en\", last depends on the used language \n",
    "\n",
    "# Remove Stop Words\n",
    "data_words_nostops = remove_stopwords(data_words)\n",
    "\n",
    "# Form Bigrams\n",
    "data_words_bigrams = make_bigrams(data_words_nostops)\n",
    "\n",
    "# Initialize spacy 'es' model, keeping only tagger component (for efficiency)\n",
    "# python3 -m spacy download es\n",
    "nlp = spacy.load('en', disable=['parser', 'ner'])\n",
    "\n",
    "# Do lemmatization keeping only noun, adj, vb, adv\n",
    "data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])\n",
    "\n",
    "print(data_lemmatized[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Dictionary\n",
    "id2word = corpora.Dictionary(data_lemmatized)\n",
    "\n",
    "# Create Corpus\n",
    "texts = data_lemmatized\n",
    "\n",
    "# Term Document Frequency\n",
    "corpus = [id2word.doc2bow(text) for text in texts]\n",
    "\n",
    "# View\n",
    "print(corpus[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Human readable format of corpus (term-frequency)\n",
    "[[(id2word[id], freq) for id, freq in cp] for cp in corpus[:2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build LDA model\n",
    "lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,\n",
    "                                           id2word=id2word,\n",
    "                                           num_topics=20, \n",
    "                                           random_state=100,\n",
    "                                           update_every=1,\n",
    "                                           chunksize=100,\n",
    "                                           passes=4,\n",
    "                                           alpha='auto',\n",
    "                                           per_word_topics=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print the Keyword in the 10 topics\n",
    "pprint(lda_model.print_topics())\n",
    "doc_lda = lda_model[corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download File: http://mallet.cs.umass.edu/dist/mallet-2.0.8.zip\n",
    "mallet_path = 'mallet-2.0.8/bin/mallet' # update this path\n",
    "ldamallet = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=20, id2word=id2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show Topics\n",
    "pprint(ldamallet.show_topics(formatted=False))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}