ULMFit_Sentiment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ULMFit Imdb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load imdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lantonini/anaconda3/envs/fastai/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python -W ignore::DeprecationWarning\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "BOS = 'xbos'\n",
    "FLD = 'xfld'\n",
    "\n",
    "PATH = Path('data/aclImdb/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLAS_PATH = Path('data/imdb_clas/')\n",
    "CLAS_PATH.mkdir(exist_ok=True)\n",
    "\n",
    "LM_PATH = Path('data/imdb_lm/')\n",
    "LM_PATH.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLASSES = ['neg', 'pos', 'unsup']\n",
    "\n",
    "def get_texts(path, limit):\n",
    "    texts,labels = [],[]\n",
    "    for idx,label in enumerate(CLASSES):\n",
    "        for fname in (path/label).glob('*.*'):\n",
    "            if limit and idx == limit:\n",
    "                break\n",
    "            texts.append(fname.open('r').read())\n",
    "            labels.append(idx)\n",
    "    return np.array(texts),np.array(labels)\n",
    "\n",
    "trn_texts,trn_labels = get_texts(PATH/'train',750)\n",
    "val_texts,val_labels = get_texts(PATH/'test',250)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(trn_texts),len(val_texts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Sort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_names = ['labels','text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "trn_idx = np.random.permutation(len(trn_texts))\n",
    "val_idx = np.random.permutation(len(val_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts = trn_texts[trn_idx]\n",
    "val_texts = val_texts[val_idx]\n",
    "\n",
    "trn_labels = trn_labels[trn_idx]\n",
    "val_labels = val_labels[val_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setting the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn = pd.DataFrame({'text':trn_texts,'labels':trn_labels}, columns=col_names)\n",
    "df_val = pd.DataFrame({'text':val_texts,'labels':val_labels}, columns=col_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn[df_trn['labels']!=2].to_csv(CLAS_PATH/'train.csv', header=False, index=False)\n",
    "df_val.to_csv(CLAS_PATH/'test.csv', header=False, index=False)\n",
    "\n",
    "(CLAS_PATH/'classes.txt').open('w').writelines(f'{o}\\n' for o in CLASSES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(CLAS_PATH/'classes.txt').open().readlines()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split train and validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_texts,val_texts = sklearn.model_selection.train_test_split(np.concatenate([trn_texts,val_texts]),test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(trx_texts),len(val_texts)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}