12.ner/SequenceLabelingBiLSTM_TODO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook explores sequence labeling using the Twitter NER dataset from the [W-NUT 2016 shared task](https://noisy-text.github.io/2016/ner-shared-task.html#resource)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from keras.layers import Dense, Input, Embedding, TimeDistributed, Layer, Multiply, Concatenate, Dropout, LSTM, Bidirectional\n",
    "from keras.models import Model, Sequential\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping, Callback\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_embeddings(filename, max_vocab_size):\n",
    "\n",
    "    \"\"\" Load pre-trained word embeddings, reserving 0 for padding symbol and 1 for UNK \"\"\"\n",
    "    \n",
    "    vocab={}\n",
    "    embeddings=[]\n",
    "    with open(filename) as file:\n",
    "        \n",
    "        cols=file.readline().split(\" \")\n",
    "        num_words=int(cols[0])\n",
    "        size=int(cols[1])\n",
    "        embeddings.append(np.zeros(size))  # 0 = 0 padding if needed\n",
    "        embeddings.append(np.zeros(size))  # 1 = UNK\n",
    "        vocab[\"_0_\"]=0\n",
    "        vocab[\"_UNK_\"]=1\n",
    "        \n",
    "        for idx,line in enumerate(file):\n",
    "\n",
    "            if idx+2 >= max_vocab_size:\n",
    "                break\n",
    "\n",
    "            cols=line.rstrip().split(\" \")\n",
    "            val=np.array(cols[1:])\n",
    "            word=cols[0]\n",
    "            \n",
    "            embeddings.append(val)\n",
    "            vocab[word]=idx+2\n",
    "\n",
    "    return np.array(embeddings), vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_ids(sentences, word_vocab, label_vocab):\n",
    "    \n",
    "    \"\"\" Function to convert a list of sentences (where each sentence is a list of (word, tag) tuples)\n",
    "    into:\n",
    "    -- a list of padded sequences of word ids\n",
    "    -- a list of padded sequence of tag ids\n",
    "    -- a list of sequence lengths (the original token count for each sentence)\n",
    "    \n",
    "    Pads each sequence to the maximum sequence length observed in the sentences input\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    words_ids=[]\n",
    "    sent_lengths=[]\n",
    "    tags_ids=[]\n",
    "    \n",
    "    output_dim=len(label_vocab)+1\n",
    "    \n",
    "    max_length=0\n",
    "    for sentence in sentences:\n",
    "        if len(sentence) > max_length:\n",
    "            max_length=len(sentence)\n",
    "    \n",
    "    for sentence in sentences:\n",
    "        wids=[]\n",
    "        tids=[]\n",
    "        \n",
    "        for word, tag in sentence:\n",
    "            val = word_vocab[word.lower()] if word.lower() in word_vocab else 1\n",
    "            wids.append(val)\n",
    "            y = to_categorical(label_vocab[tag], num_classes=output_dim)\n",
    "            tids.append(y)\n",
    "        \n",
    "        \n",
    "        for i in range(len(wids),max_length):\n",
    "            wids.append(0)\n",
    "            tids.append(to_categorical(0, num_classes=output_dim))\n",
    "            \n",
    "        words_ids.append(wids)\n",
    "        tags_ids.append(tids)\n",
    "        sent_lengths.append(len(sentence))\n",
    " \n",
    "    return np.array(words_ids), np.array(tags_ids), np.array(sent_lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_tsv(filename):\n",
    "    \n",
    "    \"\"\" Read input in two-column TSV, one line per word, with sentences delimited by a blank line \"\"\"\n",
    "    \n",
    "    sentences=[]\n",
    "    sentence=[]\n",
    "    with open(filename) as file:\n",
    "        for line in file:\n",
    "            cols=line.rstrip().split(\"\\t\")\n",
    "            if len(cols) < 2:\n",
    "                if len(sentence) > 0:\n",
    "                    sentences.append(sentence)\n",
    "                sentence=[]\n",
    "                continue\n",
    "                \n",
    "            word=cols[0]\n",
    "            tag=cols[1]\n",
    "            \n",
    "            sentence.append((word, tag))\n",
    "            \n",
    "        if len(sentence) > 0:\n",
    "            sentences.append(sentence)\n",
    "            \n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tag_vocab(sentences):\n",
    "    tags={}\n",
    "    # 0 is for masking\n",
    "    tid=1\n",
    "    for sentence in sentences:\n",
    "        for word, tag in sentence:\n",
    "            if tag not in tags:\n",
    "                tags[tag]=tid\n",
    "                tid+=1\n",
    "    return tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=read_tsv(\"../data/twitter-ner/ner.train.txt\")\n",
    "devData=read_tsv(\"../data/twitter-ner/ner.dev.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_vocab=get_tag_vocab(data)\n",
    "rev_tags={}\n",
    "for t in tag_vocab:\n",
    "    rev_tags[tag_vocab[t]]=t\n",
    "\n",
    "print(tag_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings, word_vocab=load_embeddings(\"../data/glove.twitter.27B.100d.50K.txt.w2v.txt\", 100000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you are missing the twitter glove embeddings file, download [glove.twitter.27B.100d.50K.txt](https://drive.google.com/file/d/1Tk4S5u6mwwZwEd5H7bimNXzHnbqWw7_y/view) and store it in the data/ directory. Then run the cell below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Run ONLY if you got a missing file error in the cell above!\n",
    "This code reformats the glove file you just downloaded into a format that works with our code.\n",
    "\"\"\"\n",
    "\n",
    "from gensim.models import Word2Vec, KeyedVectors\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from gensim.test.utils import datapath\n",
    "\n",
    "def read_glove(filename):\n",
    "\n",
    "    # First we have to convert the Glove format into w2v format; this creates a new file\n",
    "    glove_in_w2v_format=\"%s.w2v.txt\" % filename\n",
    "    _ = glove2word2vec(filename, glove_in_w2v_format)\n",
    "    \n",
    "    glove = KeyedVectors.load_word2vec_format(glove_in_w2v_format, binary=False)\n",
    "    return glove\n",
    "\n",
    "read_glove(\"../data/glove.twitter.27B.100d.50K.txt\")\n",
    "embeddings, word_vocab=load_embeddings(\"../data/glove.twitter.27B.100d.50K.txt.w2v.txt\", 100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX, trainY, trainS=get_word_ids(data, word_vocab, tag_vocab)\n",
    "devX, devY, devS=get_word_ids(devData, word_vocab, tag_vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train a bidirectional LSTM for sequence labeling to make predictions about the NER tag for each word in a sentence.  Explore the effect of the lstm size and dropout rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_bilstm(embeddings, output_dim, lstm_size=25, dropout_rate=0.25):\n",
    "    \n",
    "    vocab_size, word_embedding_dim=embeddings.shape\n",
    "\n",
    "    word_sequence_input = Input(shape=(None,), dtype='int32')\n",
    "    sentence_lengths = Input(shape=(None,), dtype='int32')\n",
    "\n",
    "    word_embedding_layer = Embedding(vocab_size,\n",
    "                                    word_embedding_dim,\n",
    "                                    weights=[embeddings],\n",
    "                                    trainable=False, mask_zero=True)\n",
    "\n",
    "    embedded_sequences = word_embedding_layer(word_sequence_input)\n",
    "    bi_lstm = Bidirectional(LSTM(lstm_size, return_sequences=True, activation='relu', dropout=dropout_rate), merge_mode='concat')(embedded_sequences)\n",
    "    preds = TimeDistributed(Dense(output_dim, activation=\"softmax\"))(bi_lstm)\n",
    "\n",
    "    model = Model(inputs=[word_sequence_input, sentence_lengths], outputs=preds)\n",
    "\n",
    "    model.compile(loss=\"categorical_crossentropy\", optimizer='adam', metrics=[\"acc\"])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, modelName):\n",
    "    print (model.summary())\n",
    "\n",
    "    early_stopping = EarlyStopping(monitor='val_loss',\n",
    "    min_delta=0,\n",
    "    patience=10,\n",
    "    verbose=0, \n",
    "    mode='auto')\n",
    "\n",
    "    checkpoint = ModelCheckpoint(modelName, monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "    \n",
    "    model.fit([trainX, trainS], trainY, \n",
    "            validation_data=([devX, devS], devY),\n",
    "            epochs=30, batch_size=32,\n",
    "            callbacks=[checkpoint, early_stopping])\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train a model on the data and save the one that performs best on the validation data in `bilstm_sequence_labeling.hdf5`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=create_bilstm(embeddings, len(tag_vocab)+1)\n",
    "train(model, \"bilstm_sequence_labeling.hdf5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can explore the performance of the model by predicting the NER tags for a new sequence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=create_bilstm(embeddings, len(tag_vocab)+1)\n",
    "model.load_weights(\"bilstm_sequence_labeling.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(text, model, rev_tags):\n",
    "    text=text.split(\" \")\n",
    "    wids=[]\n",
    "    for t in text:\n",
    "        if t.lower() in word_vocab:\n",
    "            wids.append(word_vocab[t.lower()])\n",
    "        else:\n",
    "            wids.append(0)\n",
    "\n",
    "    wids=np.array(wids)\n",
    "    lengths=np.array([len(wids)])\n",
    "\n",
    "\n",
    "    # lengths=np.expand_dims(lengths, 0)\n",
    "    preds=model.predict([[wids], [lengths]])\n",
    "    y_classes = preds.argmax(axis=-1)\n",
    "\n",
    "\n",
    "    predicted=[rev_tags[t] for t in y_classes[0]]\n",
    "    for w, t in zip(text, predicted):\n",
    "        print(\"%s\\t%s\" % (w,t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text=\"Bill Gates is the founder of Microsoft\"\n",
    "predict(text, model, rev_tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q1: You'll notice above that the model gets a token-level validation accuracy around 95 simply due to the high presence of the majority class (\"O\").  That's not a very helpful metric  in this case. Implement F-score for NER.  Remember, the F-score for NER is based on *chunks*; for more, see section 11.3.2 in: of SLP3 [chapter 11](https://web.stanford.edu/~jurafsky/slp3/11.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculateF1(gold_sequences, predicted_sequences):\n",
    "    \n",
    "    \"\"\" Function to calculate the precision, recall and F-score over labeled chunks in the gold and predicted\n",
    "    input sequences.  Each input parameter contains a list of label sequences (one label for each word in the\n",
    "    sentence). In the following example, `gold_sequences` and `predicted_sequences` both contain two sentences\n",
    "    (the first has 7 words/tags, and the second has 3 words/tags):\n",
    "    \n",
    "    gold_sequences=[[\"B-PER\", \"I-PER\", \"O\", \"O\", \"O\", \"O\", \"B-ORG\"], [\"O\", \"O\", \"O\"]]\n",
    "    predicted_sequences=[[\"B-PER\", \"O\", \"O\", \"O\", \"B-PER\", \"O\", \"B-ORG\"], [\"O\", \"O\", \"O\"]]\n",
    "    \n",
    "    Returns tuple of (precision, recall, F-score)\n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    # your code here\n",
    "\n",
    "    return precision, recall, F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example from class on 4/4\n",
    "\n",
    "precision, recall, F1=calculateF1([[\"B-PER\", \"I-PER\", \"O\", \"O\", \"O\", \"O\", \"B-ORG\"], [\"O\", \"O\", \"O\"]], [[\"B-PER\", \"O\", \"O\", \"O\", \"B-PER\", \"O\", \"B-ORG\"], [\"O\", \"O\", \"O\"]])\n",
    "print(\"P: %.3f, R: %.3f, F: %.3f\" % (precision, recall, F1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keras by default calculates metrics like accuracy at the batch level (averaging the metric across batches).  F-score, however, is a metric properly calculated over an entire dataset; we can incorporate that into learning by defining a callback function that prints out the validation F-score at the end of each epoch.  Once you've implemented `calculateF1` above, execute the following cells to see the validation F-score while training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class F_score(Callback):\n",
    "    \n",
    "    def __init__(self, reverse_tag_vocab):\n",
    "        self.reverse_tag_vocab=reverse_tag_vocab\n",
    "        \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        \n",
    "        valX=self.validation_data[0]\n",
    "        valS=self.validation_data[1]\n",
    "        valY=self.validation_data[2]\n",
    "        \n",
    "        predictions=self.model.predict([valX, valS])\n",
    "        y_classes = predictions.argmax(axis=-1)\n",
    "        truth = valY.argmax(axis=-1)\n",
    "\n",
    "        preds=[]\n",
    "        golds=[]\n",
    "\n",
    "        s,w=y_classes.shape\n",
    "        for i in range(s):\n",
    "            sent_preds=[]\n",
    "            sent_golds=[]\n",
    "            for j in range(int(valS[i])):\n",
    "                sent_golds.append(self.reverse_tag_vocab[truth[i,j]])\n",
    "                sent_preds.append(self.reverse_tag_vocab[y_classes[i,j]])\n",
    "            preds.append(sent_preds)\n",
    "            golds.append(sent_golds)\n",
    "        \n",
    "        precision, recall, F1=calculateF1(golds, preds)\n",
    "        print(\"P: %.3f, R: %.3f, F: %.3f\" % (precision, recall, F1))\n",
    "    \n",
    "        return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, modelName):\n",
    "    print (model.summary())\n",
    "\n",
    "    early_stopping = EarlyStopping(monitor='val_loss',\n",
    "    min_delta=0,\n",
    "    patience=10,\n",
    "    verbose=0, \n",
    "    mode='auto')\n",
    "\n",
    "    f_score=F_score(rev_tags)\n",
    "    checkpoint = ModelCheckpoint(modelName, monitor='val_loss', verbose=0, save_best_only=True, mode='min')\n",
    "    \n",
    "    model.fit([trainX, trainS], trainY, \n",
    "            validation_data=([devX, devS], devY),\n",
    "            epochs=30, batch_size=32,\n",
    "            callbacks=[f_score, checkpoint, early_stopping])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=create_bilstm(embeddings, len(tag_vocab)+1)\n",
    "train(model, \"bilstm_sequence_labeling.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}