code/notebooks/the-3-reasons-why-i-have-permanently-switched-from-pandas-to-polars.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cdddf358",
   "metadata": {},
   "source": [
    "# The 3 Reasons Why I Have Permanently Switched From Pandas To Polars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fd35bec5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:95% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:95% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b987d45a",
   "metadata": {},
   "source": [
    "## 1. The `.list` Namespace"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98309036",
   "metadata": {},
   "source": [
    "### Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "49f01f05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  last_name                    members city_of_residence\n",
      "0   Johnson           [John, Ron, Con]            Boston\n",
      "1   Jackson               [Jack, Rack]     New York City\n",
      "2  Smithson  [Smith, Pith, With, Lith]            Dallas\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame({\n",
    "    \"last_name\": [\"Johnson\", \"Jackson\", \"Smithson\"],\n",
    "    \"members\": [[\"John\", \"Ron\", \"Con\"], [\"Jack\", \"Rack\"], [\"Smith\", \"Pith\", \"With\", \"Lith\"]],\n",
    "    \"city_of_residence\": [\"Boston\", \"New York City\", \"Dallas\"]\n",
    "})\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c7769bc9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  last_name                    members city_of_residence family_leader\n",
      "0   Johnson           [John, Ron, Con]            Boston          John\n",
      "1   Jackson               [Jack, Rack]     New York City          Jack\n",
      "2  Smithson  [Smith, Pith, With, Lith]            Dallas         Smith\n"
     ]
    }
   ],
   "source": [
    "df[\"family_leader\"] = df[\"members\"].str[0]\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2debb15",
   "metadata": {},
   "source": [
    "### Polars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5eedb4f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape: (3, 4)\n",
      "┌───────────┬─────────────────────────────┬───────────────────┬───────────────┐\n",
      "│ last_name ┆ members                     ┆ city_of_residence ┆ family_leader │\n",
      "│ ---       ┆ ---                         ┆ ---               ┆ ---           │\n",
      "│ str       ┆ list[str]                   ┆ str               ┆ str           │\n",
      "╞═══════════╪═════════════════════════════╪═══════════════════╪═══════════════╡\n",
      "│ Johnson   ┆ [\"John\", \"Ron\", \"Con\"]      ┆ Boston            ┆ John          │\n",
      "│ Jackson   ┆ [\"Jack\", \"Rack\"]            ┆ New York City     ┆ Jack          │\n",
      "│ Smithson  ┆ [\"Smith\", \"Pith\", … \"Lith\"] ┆ Dallas            ┆ Smith         │\n",
      "└───────────┴─────────────────────────────┴───────────────────┴───────────────┘\n"
     ]
    }
   ],
   "source": [
    "import polars as pl\n",
    "df = pl.DataFrame({\n",
    "    \"last_name\": [\"Johnson\", \"Jackson\", \"Smithson\"],\n",
    "    \"members\": [[\"John\", \"Ron\", \"Con\"], [\"Jack\", \"Rack\"], [\"Smith\", \"Pith\", \"With\", \"Lith\"]],\n",
    "    \"city_of_residence\": [\"Boston\", \"New York City\", \"Dallas\"]\n",
    "})\n",
    "df = df.with_columns([\n",
    "    pl.col(\"members\").list.get(0).alias(\"family_leader\")])\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24c3b9d1",
   "metadata": {},
   "source": [
    "### Bloated API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "52ff02dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   a\n",
      "0  1\n",
      "1  1\n",
      "2  1\n",
      "   a  b\n",
      "1  1  5\n",
      "   a  b\n",
      "1  1  5\n",
      "2  1  6\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame({\n",
    "    \"a\": [1, 1, 1],\n",
    "    \"b\": [4, 5, 6]\n",
    "})\n",
    "\n",
    "column_name_indexer = [\"a\"]\n",
    "boolean_mask_indexer = df[\"b\"]==5\n",
    "slice_indexer = slice(1, 3)\n",
    "\n",
    "for o in [column_name_indexer, boolean_mask_indexer, slice_indexer]:\n",
    "    print(df[o])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d57287cc",
   "metadata": {},
   "source": [
    "## 2. `.scan_parquet()` and `.sink_parquet()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8baa6eac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import polars as pl\n",
    "eager_df = pl.DataFrame({\n",
    "    \"a\": [1, 2, 3],\n",
    "    \"b\": [4, 5, 6]\n",
    "})\n",
    "lazy_df = pl.LazyFrame({\n",
    "    \"a\": [1, 2, 3],\n",
    "    \"b\": [4, 5, 6]\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf61e86e",
   "metadata": {},
   "source": [
    "## 3. Data-Oriented Programming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "841786f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "decade\n",
      "0     2\n",
      "10    1\n",
      "40    1\n",
      "Name: name, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = (\n",
    "    pd.DataFrame({\n",
    "        \"name\": [\"George\", \"Polly\", \"Golly\", \"Dolly\"],\n",
    "        \"age\": [3, 4, 13, 44]\n",
    "    })\n",
    ")\n",
    "df[\"decade\"] = (df[\"age\"] / 10).astype(int) * 10\n",
    "decade_counts = (\n",
    "    df\n",
    "    .groupby(\"decade\")\n",
    "    [\"name\"]\n",
    "    .agg(\"count\")\n",
    ")\n",
    "print(decade_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1252a86b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape: (3, 2)\n",
      "┌────────┬───────┐\n",
      "│ decade ┆ count │\n",
      "│ ---    ┆ ---   │\n",
      "│ i32    ┆ u32   │\n",
      "╞════════╪═══════╡\n",
      "│ 0      ┆ 2     │\n",
      "│ 10     ┆ 1     │\n",
      "│ 40     ┆ 1     │\n",
      "└────────┴───────┘\n"
     ]
    }
   ],
   "source": [
    "import polars as pl\n",
    "decade_counts = (\n",
    "    pl.DataFrame({\n",
    "        \"name\": [\"George\", \"Polly\", \"Golly\", \"Dolly\"],\n",
    "        \"age\": [3, 4, 13, 44]\n",
    "    })\n",
    "    .with_columns([\n",
    "        ((pl.col(\"age\") / 10).cast(pl.Int32) * 10).alias(\"decade\")\n",
    "    ])\n",
    "    .groupby(\"decade\")\n",
    "    .agg(\n",
    "        pl.col(\"name\").count().alias(\"count\")\n",
    "    )\n",
    ")\n",
    "print(decade_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dda2e41",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}