12_linearization_my_solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A/B Testing Course"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lesson 12. Ratio Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Homework"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 580,
     "status": "ok",
     "timestamp": 1674542192205,
     "user": {
      "displayName": "Nikolay Nazarov",
      "userId": "18018785782997408667"
     },
     "user_tz": -420
    },
    "id": "agQEC38nE-72",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1674542192206,
     "user": {
      "displayName": "Nikolay Nazarov",
      "userId": "18018785782997408667"
     },
     "user_tz": -420
    },
    "id": "xxFeCfFuFKng",
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_exp_users = pd.read_csv('12_experiment_users.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 6399,
     "status": "ok",
     "timestamp": 1674542198600,
     "user": {
      "displayName": "Nikolay Nazarov",
      "userId": "18018785782997408667"
     },
     "user_tz": -420
    },
    "id": "nQGxlqByE-73",
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_web_logs = pd.read_csv('12_2022-04-13T12_df_web_logs.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Data Description"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df_exp_users - users split in the experiment:\n",
    "- user_id - user identifier;\n",
    "- pilot - test group flag."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nNRTGF6aBHrZ"
   },
   "source": [
    "df_web_logs - web logs information:\n",
    "- user_id - user identifier;\n",
    "- page - visited page;\n",
    "- date - date and time of a page visit;\n",
    "- load_time - page load time."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Checking Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "executionInfo": {
     "elapsed": 53,
     "status": "ok",
     "timestamp": 1674542198602,
     "user": {
      "displayName": "Nikolay Nazarov",
      "userId": "18018785782997408667"
     },
     "user_tz": -420
    },
    "id": "0dZkzMMTE-73",
    "outputId": "44f7566a-b32d-4479-b963-7c5977351149",
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>pilot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c36b2e</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20336e</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>034652</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e98e3b</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3f1105</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id  pilot\n",
       "0  c36b2e      0\n",
       "1  20336e      0\n",
       "2  034652      0\n",
       "3  e98e3b      0\n",
       "4  3f1105      0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_exp_users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5456 entries, 0 to 5455\n",
      "Data columns (total 2 columns):\n",
      " #   Column   Non-Null Count  Dtype \n",
      "---  ------   --------------  ----- \n",
      " 0   user_id  5456 non-null   object\n",
      " 1   pilot    5456 non-null   int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 85.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df_exp_users.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>page</th>\n",
       "      <th>date</th>\n",
       "      <th>load_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f25239</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:45:37</td>\n",
       "      <td>80.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>06d6df</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:49:56</td>\n",
       "      <td>70.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>06d6df</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:51:16</td>\n",
       "      <td>89.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f25239</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:51:43</td>\n",
       "      <td>74.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>697870</td>\n",
       "      <td>m</td>\n",
       "      <td>2022-02-03 23:53:12</td>\n",
       "      <td>66.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id page                 date  load_time\n",
       "0  f25239    m  2022-02-03 23:45:37       80.8\n",
       "1  06d6df    m  2022-02-03 23:49:56       70.5\n",
       "2  06d6df    m  2022-02-03 23:51:16       89.7\n",
       "3  f25239    m  2022-02-03 23:51:43       74.4\n",
       "4  697870    m  2022-02-03 23:53:12       66.8"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_web_logs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2401709 entries, 0 to 2401708\n",
      "Data columns (total 4 columns):\n",
      " #   Column     Dtype  \n",
      "---  ------     -----  \n",
      " 0   user_id    object \n",
      " 1   page       object \n",
      " 2   date       object \n",
      " 3   load_time  float64\n",
      "dtypes: float64(1), object(3)\n",
      "memory usage: 73.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df_web_logs.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_web_logs.load_time.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Task 1. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Previously, we evaluated the experiment assuming that the backend response time data was independent. Now, we can properly evaluate this experiment even if the data were dependent on users.\n",
    "\n",
    "Evaluate the experiment using linearization assuming user-dependent data.\n",
    "\n",
    "The experiment was conducted from 2022-04-05 to 2022-04-12. The metric is the average request processing time.\n",
    "\n",
    "Please enter the p-value as a response with an accuracy of up to 4 decimal places."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filt = df_web_logs[(df_web_logs['date'] >= '2022-04-05') & (df_web_logs['date'] < '2022-04-12')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_agg = df_filt.groupby('user_id', as_index=False).agg({'load_time': list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.merge(data_agg, df_exp_users, how='inner', on='user_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pilot\n",
       "1    2773\n",
       "0    2683\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.pilot.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def check_linearization(a, b):\n",
    "    \"\"\"Hypothesis testing using linearization.\n",
    "    \n",
    "    a: List[np.array], a list of sets of session length for users in the control group.\n",
    "    b: List[np.array], a list of sets of session length for users in the pilot group.\n",
    "    \n",
    "    return: p-value and point estimate.\n",
    "    \"\"\"\n",
    "    a_x = np.array([np.sum(row) for row in a])\n",
    "    a_y = np.array([len(row) for row in a])\n",
    "    b_x = np.array([np.sum(row) for row in b])\n",
    "    b_y = np.array([len(row) for row in b])\n",
    "    \n",
    "    coef = np.sum(a_x) / np.sum(a_y)\n",
    "    \n",
    "    a_lin = a_x - coef * a_y\n",
    "    b_lin = b_x - coef * b_y\n",
    "    \n",
    "    delta = np.mean(b_lin) - np.mean(a_lin)\n",
    "    _, pvalue = ttest_ind(a_lin, b_lin)\n",
    "\n",
    "    return pvalue, delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0442\n"
     ]
    }
   ],
   "source": [
    "control_data = data[data['pilot'] == 0].load_time.values\n",
    "test_data = data[data['pilot'] == 1].load_time.values\n",
    "\n",
    "pval, delta = check_linearization(control_data, test_data)\n",
    "\n",
    "print(f'p-value: {pval:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Task 2. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code class `MetricsService` method `calculate_linearized_metrics`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "simple test passed\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "class MetricsService:\n",
    "\n",
    "    def calculate_linearized_metrics(\n",
    "        self, control_metrics, pilot_metrics, control_user_ids=None, pilot_user_ids=None\n",
    "    ):\n",
    "        \"\"\"Calculates the ratio metric values.\n",
    "        \n",
    "        You need to compute the kappa parameter (coefficient in the linearization function) based on the data from control_metrics and use it to calculate the linearized metric.\n",
    "\n",
    "        :param control_metrics (pd.DataFrame): DataFrame with metric values for the control group.\n",
    "        The 'user_id' column may contain non-unique values.\n",
    "        Measurements for the same user_id are considered dependent, while different user_ids are considered independent.\n",
    "        columns=['user_id', 'metric']\n",
    "        :param pilot_metrics (pd.DataFrame): DataFrame with metric values for the pilot group.\n",
    "        The 'user_id' column may contain non-unique values.\n",
    "        Measurements for the same user_id are considered dependent, while different user_ids are considered independent.\n",
    "        columns=['user_id', 'metric']\n",
    "        :param control_user_ids (list): List of user IDs for the control group for which the metric should be calculated.\n",
    "        If None, use the users from control_metrics.\n",
    "        If a user has no records in the control_metrics table, their linearized metric will be zero.\n",
    "        :param pilot_user_ids (list): List of user IDs for the pilot group for which the metric should be calculated.\n",
    "        If None, use the users from pilot_metrics.\n",
    "        If a user has no records in the pilot_metrics table, their linearized metric will be zero.\n",
    "        \n",
    "        :return lin_control_metrics, lin_pilot_metrics: DataFrame with columns=['user_id', 'metric']\n",
    "        \"\"\"\n",
    "        # YOUR_CODE_HERE\n",
    "        def linearization(a, b):\n",
    "            a_x = np.array([np.sum(row) for row in a])\n",
    "            a_y = np.array([len(row) for row in a])\n",
    "            b_x = np.array([np.sum(row) for row in b])\n",
    "            b_y = np.array([len(row) for row in b])\n",
    "            coef = np.sum(a_x) / np.sum(a_y)\n",
    "            a_lin = a_x - coef * a_y\n",
    "            b_lin = b_x - coef * b_y\n",
    "            return a_lin, b_lin\n",
    "        \n",
    "        control_grp = control_metrics.groupby('user_id', as_index=False).agg({'metric': list})\n",
    "        test_grp = pilot_metrics.groupby('user_id', as_index=False).agg({'metric': list})\n",
    "        \n",
    "        control_lin, test_lin = linearization(control_grp.metric.values, test_grp.metric.values)\n",
    "        \n",
    "        control_grp['metric'] = control_lin\n",
    "        test_grp['metric'] = test_lin\n",
    "        \n",
    "        if control_user_ids != None:\n",
    "            control_users_df = pd.DataFrame({'user_id': control_user_ids, 'metric': [0] * len(control_user_ids)})\n",
    "            control_grp = pd.merge(control_grp, control_users_df, how='right', on='user_id')[['user_id', 'metric_x']] \\\n",
    "                .rename(columns={'metric_x': 'metric'}).fillna(0)\n",
    "        \n",
    "        if pilot_user_ids != None:\n",
    "            test_users_df = pd.DataFrame({'user_id': pilot_user_ids, 'metric': [0] * len(pilot_user_ids)})\n",
    "            test_grp = pd.merge(test_grp, test_users_df, how='right', on='user_id')[['user_id', 'metric_x']] \\\n",
    "                .rename(columns={'metric_x': 'metric'}).fillna(0)\n",
    "            \n",
    "        return control_grp, test_grp\n",
    "            \n",
    "\n",
    "\n",
    "def _chech_df(df, df_ideal, sort_by, reindex=False, set_dtypes=False, decimal=None):\n",
    "    assert isinstance(df, pd.DataFrame), 'The return of the function is not a pd.DataFrame.'\n",
    "    assert len(df) == len(df_ideal), 'Wrong number of lines.'\n",
    "    assert len(df.T) == len(df_ideal.T), 'Wrong number of columns.'\n",
    "    columns = df_ideal.columns\n",
    "    assert df.columns.isin(columns).sum() == len(df.columns), 'Wrong names of columns.'\n",
    "    df = df[columns].sort_values(sort_by)\n",
    "    df_ideal = df_ideal.sort_values(sort_by)\n",
    "    if reindex:\n",
    "        df_ideal.index = range(len(df_ideal))\n",
    "        df.index = range(len(df))\n",
    "    if set_dtypes:\n",
    "        for column, dtype in df_ideal.dtypes.to_dict().items():\n",
    "            df[column] = df[column].astype(dtype)\n",
    "    if decimal:\n",
    "        ideal_values = df_ideal.astype(float).values\n",
    "        values = df.astype(float).values\n",
    "        np.testing.assert_almost_equal(ideal_values, values, decimal=decimal)\n",
    "    else:\n",
    "        assert df_ideal.equals(df), 'The final dataframe does not match with an expected result.'\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    control_metrics = pd.DataFrame({'user_id': [1, 1, 2], 'metric': [3, 5, 7],})\n",
    "    pilot_metrics = pd.DataFrame({'user_id': [3, 3], 'metric': [3, 6], })\n",
    "    ideal_lin_control_metrics = pd.DataFrame({'user_id': [1, 2], 'metric': [-2, 2],})\n",
    "    ideal_lin_pilot_metrics = pd.DataFrame({'user_id': [3,], 'metric': [-1,],})\n",
    "\n",
    "    metrics_service = MetricsService()\n",
    "    lin_control_metrics, lin_pilot_metrics = metrics_service.calculate_linearized_metrics(\n",
    "        control_metrics, pilot_metrics\n",
    "    )\n",
    "    _chech_df(lin_control_metrics, ideal_lin_control_metrics, ['user_id', 'metric'], True, True, decimal=3)\n",
    "    _chech_df(lin_pilot_metrics, ideal_lin_pilot_metrics, ['user_id', 'metric'], True, True, decimal=3)\n",
    "    print('simple test passed')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "KRzEGsPUE-7k"
   ],
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}