examples/colab/semantic_similarity_with_tf_hub_universal_encoder.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "RUymE2l9GZfO"
      },
      "source": [
        "##### Copyright 2018 The TensorFlow Hub Authors.\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "cellView": "code",
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "JMyTNwSJGGWg"
      },
      "outputs": [],
      "source": [
        "# Copyright 2018 The TensorFlow Hub Authors. All Rights Reserved.\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     http://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License.\n",
        "# =============================================================================="
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "co7MV6sX7Xto"
      },
      "source": [
        "# Universal Sentence Encoder\n",
        "\n",
        "\n",
        "\u003ctable align=\"left\"\u003e\u003ctd\u003e\n",
        "  \u003ca target=\"_blank\"  href=\"https://colab.research.google.com/github/tensorflow/hub/blob/master/examples/colab/semantic_similarity_with_tf_hub_universal_encoder.ipynb\"\u003e\n",
        "    \u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\n",
        "  \u003c/a\u003e\n",
        "\u003c/td\u003e\u003ctd\u003e\n",
        "  \u003ca target=\"_blank\"  href=\"https://github.com/tensorflow/hub/blob/master/examples/colab/semantic_similarity_with_tf_hub_universal_encoder.ipynb\"\u003e\n",
        "    \u003cimg width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n",
        "\u003c/td\u003e\u003c/table\u003e\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "eAVQGidpL8v5"
      },
      "source": [
        "This notebook illustrates how to access the Universal Sentence Encoder and use it for sentence similarity and sentence classification tasks.\n",
        "\n",
        "The Universal Sentence Encoder makes getting sentence level embeddings as easy as it has historically been to lookup the embeddings for individual words. The sentence embeddings can then be trivially used to compute sentence level meaning similarity as well as to enable better performance on downstream classification tasks using less supervised training data.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "pOTzp8O36CyQ"
      },
      "source": [
        "# Getting Started\n",
        "\n",
        "This section sets up the environment for access to the Universal Sentence Encoder on TF Hub and provides examples of applying the encoder to words, sentences, and paragraphs."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "lVjNK8shFKOC"
      },
      "outputs": [],
      "source": [
        "# Install the latest Tensorflow version.\n",
        "!pip3 install --quiet \"tensorflow\u003e=1.7\"\n",
        "# Install TF-Hub.\n",
        "!pip3 install --quiet tensorflow-hub\n",
        "!pip3 install --quiet seaborn"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "63Pd3nJnTl-i"
      },
      "source": [
        "More detailed information about installing Tensorflow can be found at [https://www.tensorflow.org/install/](https://www.tensorflow.org/install/)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "MSeY-MUQo2Ha"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import os\n",
        "import pandas as pd\n",
        "import re\n",
        "import seaborn as sns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "zwty8Z6mAkdV"
      },
      "outputs": [],
      "source": [
        "module_url = \"https://tfhub.dev/google/universal-sentence-encoder/2\" #@param [\"https://tfhub.dev/google/universal-sentence-encoder/2\", \"https://tfhub.dev/google/universal-sentence-encoder-large/3\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "Q8F4LNGFqOiq"
      },
      "outputs": [],
      "source": [
        "# Import the Universal Sentence Encoder's TF Hub module\n",
        "embed = hub.Module(module_url)\n",
        "\n",
        "# Compute a representation for each message, showing various lengths supported.\n",
        "word = \"Elephant\"\n",
        "sentence = \"I am a sentence for which I would like to get its embedding.\"\n",
        "paragraph = (\n",
        "    \"Universal Sentence Encoder embeddings also support short paragraphs. \"\n",
        "    \"There is no hard limit on how long the paragraph is. Roughly, the longer \"\n",
        "    \"the more 'diluted' the embedding will be.\")\n",
        "messages = [word, sentence, paragraph]\n",
        "\n",
        "# Reduce logging output.\n",
        "tf.logging.set_verbosity(tf.logging.ERROR)\n",
        "\n",
        "with tf.Session() as session:\n",
        "  session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
        "  message_embeddings = session.run(embed(messages))\n",
        "\n",
        "  for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):\n",
        "    print(\"Message: {}\".format(messages[i]))\n",
        "    print(\"Embedding size: {}\".format(len(message_embedding)))\n",
        "    message_embedding_snippet = \", \".join(\n",
        "        (str(x) for x in message_embedding[:3]))\n",
        "    print(\"Embedding: [{}, ...]\\n\".format(message_embedding_snippet))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "BnvjATdy64eR"
      },
      "source": [
        "# Semantic Textual Similarity Task Example\n",
        "\n",
        "The embeddings produced by the Universal Sentence Encoder are approximately normalized. The semantic similarity of two sentences can be trivially computed as the inner product of the encodings."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "h1FFCTKm7ba4"
      },
      "outputs": [],
      "source": [
        "def plot_similarity(labels, features, rotation):\n",
        "  corr = np.inner(features, features)\n",
        "  sns.set(font_scale=1.2)\n",
        "  g = sns.heatmap(\n",
        "      corr,\n",
        "      xticklabels=labels,\n",
        "      yticklabels=labels,\n",
        "      vmin=0,\n",
        "      vmax=1,\n",
        "      cmap=\"YlOrRd\")\n",
        "  g.set_xticklabels(labels, rotation=rotation)\n",
        "  g.set_title(\"Semantic Textual Similarity\")\n",
        "\n",
        "\n",
        "def run_and_plot(session_, input_tensor_, messages_, encoding_tensor):\n",
        "  message_embeddings_ = session_.run(\n",
        "      encoding_tensor, feed_dict={input_tensor_: messages_})\n",
        "  plot_similarity(messages_, message_embeddings_, 90)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "339tuJ5Pwqqv"
      },
      "source": [
        "## Similarity Visualized\n",
        "Here we show the similarity in a heat map. The final graph is a 9x9 matrix where each entry `[i, j]` is colored based on the inner product of the encodings for sentence `i` and `j`."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "cPMCaxrZwp7t"
      },
      "outputs": [],
      "source": [
        "messages = [\n",
        "    # Smartphones\n",
        "    \"I like my phone\",\n",
        "    \"My phone is not good.\",\n",
        "    \"Your cellphone looks great.\",\n",
        "\n",
        "    # Weather\n",
        "    \"Will it snow tomorrow?\",\n",
        "    \"Recently a lot of hurricanes have hit the US\",\n",
        "    \"Global warming is real\",\n",
        "\n",
        "    # Food and health\n",
        "    \"An apple a day, keeps the doctors away\",\n",
        "    \"Eating strawberries is healthy\",\n",
        "    \"Is paleo better than keto?\",\n",
        "\n",
        "    # Asking about age\n",
        "    \"How old are you?\",\n",
        "    \"what is your age?\",\n",
        "]\n",
        "\n",
        "similarity_input_placeholder = tf.placeholder(tf.string, shape=(None))\n",
        "similarity_message_encodings = embed(similarity_input_placeholder)\n",
        "with tf.Session() as session:\n",
        "  session.run(tf.global_variables_initializer())\n",
        "  session.run(tf.tables_initializer())\n",
        "  run_and_plot(session, similarity_input_placeholder, messages,\n",
        "               similarity_message_encodings)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "6FjdeCqPJeg-"
      },
      "source": [
        "## Evaluation: STS (Semantic Textual Similarity) Benchmark\n",
        "\n",
        "The [**STS Benchmark**](http://ixa2.si.ehu.es/stswiki/index.php/STSbenchmark) provides an intristic evaluation of the degree to which similarity scores computed using sentence embeddings align with human judgements. The benchmark requires systems to return similarity scores for a diverse selection of sentence pairs. [Pearson correlation](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) is then used to evaluate the quality of the machine similarity scores against human judgements."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "q5nuBbI1iFQR"
      },
      "source": [
        "### Download data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "VOs8ZfOnJeBF"
      },
      "outputs": [],
      "source": [
        "import pandas\n",
        "import scipy\n",
        "import math\n",
        "\n",
        "\n",
        "def load_sts_dataset(filename):\n",
        "  # Loads a subset of the STS dataset into a DataFrame. In particular both\n",
        "  # sentences and their human rated similarity score.\n",
        "  sent_pairs = []\n",
        "  with tf.gfile.GFile(filename, \"r\") as f:\n",
        "    for line in f:\n",
        "      ts = line.strip().split(\"\\t\")\n",
        "      # (sent_1, sent_2, similarity_score)\n",
        "      sent_pairs.append((ts[5], ts[6], float(ts[4])))\n",
        "  return pandas.DataFrame(sent_pairs, columns=[\"sent_1\", \"sent_2\", \"sim\"])\n",
        "\n",
        "\n",
        "def download_and_load_sts_data():\n",
        "  sts_dataset = tf.keras.utils.get_file(\n",
        "      fname=\"Stsbenchmark.tar.gz\",\n",
        "      origin=\"http://ixa2.si.ehu.es/stswiki/images/4/48/Stsbenchmark.tar.gz\",\n",
        "      extract=True)\n",
        "\n",
        "  sts_dev = load_sts_dataset(\n",
        "      os.path.join(os.path.dirname(sts_dataset), \"stsbenchmark\", \"sts-dev.csv\"))\n",
        "  sts_test = load_sts_dataset(\n",
        "      os.path.join(\n",
        "          os.path.dirname(sts_dataset), \"stsbenchmark\", \"sts-test.csv\"))\n",
        "\n",
        "  return sts_dev, sts_test\n",
        "\n",
        "\n",
        "sts_dev, sts_test = download_and_load_sts_data()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "MkqPOxH3EL1j"
      },
      "source": [
        "### Build Evaluation Graph"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "PeoO8z30smCS"
      },
      "outputs": [],
      "source": [
        "sts_input1 = tf.placeholder(tf.string, shape=(None))\n",
        "sts_input2 = tf.placeholder(tf.string, shape=(None))\n",
        "\n",
        "# For evaluation we use exactly normalized rather than\n",
        "# approximately normalized.\n",
        "sts_encode1 = tf.nn.l2_normalize(embed(sts_input1), axis=1)\n",
        "sts_encode2 = tf.nn.l2_normalize(embed(sts_input2), axis=1)\n",
        "cosine_similarities = tf.reduce_sum(tf.multiply(sts_encode1, sts_encode2), axis=1)\n",
        "clip_cosine_similarities = tf.clip_by_value(cosine_similarities, -1.0, 1.0)\n",
        "sim_scores = 1.0 - tf.acos(clip_cosine_similarities)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "8OKy8WhnKRe_"
      },
      "source": [
        "### Evaluate Sentence Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "5PwUwn0qb2RN"
      },
      "outputs": [],
      "source": [
        "sts_data = sts_dev #@param [\"sts_dev\", \"sts_test\"] {type:\"raw\"}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 0,
      "metadata": {
        "colab": {
          "autoexec": {
            "startup": false,
            "wait_interval": 0
          }
        },
        "colab_type": "code",
        "id": "W-q2r7jyZGb7"
      },
      "outputs": [],
      "source": [
        "text_a = sts_data['sent_1'].tolist()\n",
        "text_b = sts_data['sent_2'].tolist()\n",
        "dev_scores = sts_data['sim'].tolist()\n",
        "\n",
        "def run_sts_benchmark(session):\n",
        "  \"\"\"Returns the similarity scores\"\"\"\n",
        "  emba, embb, scores = session.run(\n",
        "      [sts_encode1, sts_encode2, sim_scores],\n",
        "      feed_dict={\n",
        "          sts_input1: text_a,\n",
        "          sts_input2: text_b\n",
        "      })\n",
        "  return scores\n",
        "\n",
        "\n",
        "with tf.Session() as session:\n",
        "  session.run(tf.global_variables_initializer())\n",
        "  session.run(tf.tables_initializer())\n",
        "  scores = run_sts_benchmark(session)\n",
        "\n",
        "pearson_correlation = scipy.stats.pearsonr(scores, dev_scores)\n",
        "print('Pearson correlation coefficient = {0}\\np-value = {1}'.format(\n",
        "    pearson_correlation[0], pearson_correlation[1]))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "RUymE2l9GZfO"
      ],
      "default_view": {},
      "name": "Semantic Similarity with TF-Hub Universal Encoder",
      "provenance": [],
      "version": "0.3.2",
      "views": {}
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}