nbs/validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d2e8f71-a152-4402-a024-058d89e4d18f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp validation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09160671-140a-4627-8702-f302b5f1a3cc",
   "metadata": {},
   "source": [
    "# Validation\n",
    "\n",
    "> Utilities to validate input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b750a71-5c9a-48df-9607-498c6fef23b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "import re\n",
    "from typing import Optional, Union\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from utilsforecast.compat import DFType, DataFrame, Series, pl_DataFrame, pl_Series, pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7638f2a6-0303-41a9-a103-d3055e7bc572",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "from fastcore.test import test_eq, test_fail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e001379f-a552-4fd9-ac82-3f4d5c76f555",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| polars\n",
    "import polars.testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70eb41a2-869c-451f-8608-27aae7d7ef73",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| exporti\n",
    "def _is_int_dtype(s: Union[pd.Index, Series]) -> bool:\n",
    "    if isinstance(s, (pd.Index, pd.Series)):\n",
    "        out = pd.api.types.is_integer_dtype(s.dtype)\n",
    "    else:\n",
    "        try:\n",
    "            out = s.dtype.is_integer()\n",
    "        except AttributeError:\n",
    "            out = s.is_integer()\n",
    "    return out\n",
    "\n",
    "def _is_dt_dtype(s: Union[pd.Index, Series]) -> bool:\n",
    "    if isinstance(s, (pd.Index, pd.Series)):\n",
    "        out = pd.api.types.is_datetime64_any_dtype(s.dtype)\n",
    "    else:\n",
    "        out = s.dtype in (pl.Date, pl.Datetime)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b73338a3-b935-483b-942b-84a439ce6aab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "assert _is_int_dtype(pd.Series([1, 2]))\n",
    "assert _is_int_dtype(pd.Index([1, 2], dtype='uint8'))\n",
    "assert not _is_int_dtype(pd.Series([1.0]))\n",
    "assert _is_dt_dtype(pd.to_datetime(['2000-01-01']))\n",
    "assert _is_dt_dtype(pd.to_datetime(['2000-01-01'], utc=True))\n",
    "assert _is_dt_dtype(pd.to_datetime(['2000-01-01']).astype('datetime64[s]'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80f090c9-c293-47df-a95a-af530737201e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "#| pyarrow\n",
    "assert _is_int_dtype(pd.Series([1, 2], dtype='int32[pyarrow]'))\n",
    "assert _is_dt_dtype(pd.to_datetime(['2000-01-01']).astype('timestamp[ns][pyarrow]'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "242c213e-f5c6-400b-b905-605a9fa44fe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "#| polars\n",
    "assert _is_int_dtype(pl.Series([1, 2]))\n",
    "assert _is_int_dtype(pl.Series([1, 2], dtype=pl.UInt8))\n",
    "assert not _is_int_dtype(pl.Series([1.0]))\n",
    "assert _is_dt_dtype(pl.Series([datetime.date(2000, 1, 1)]))\n",
    "assert _is_dt_dtype(pl.Series([datetime.datetime(2000, 1, 1)]))\n",
    "assert _is_dt_dtype(pl.Series([datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98e0f72a-3128-46d7-8d6c-cbd0c62ce68a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| exporti\n",
    "def _is_dt_or_int(s: Series) -> bool:\n",
    "    return _is_dt_dtype(s) or _is_int_dtype(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6679513a-4f2b-4eda-9970-e5c6725dd761",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def ensure_shallow_copy(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    from packaging.version import Version\n",
    "\n",
    "    if Version(pd.__version__) < Version(\"1.4\"):\n",
    "        # https://github.com/pandas-dev/pandas/pull/43406\n",
    "        df = df.copy()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccb15f20-56e4-4d0d-96bc-830c5effff19",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def ensure_time_dtype(df: DFType, time_col: str = 'ds') -> DFType:\n",
    "    \"\"\"Make sure that `time_col` contains timestamps or integers.\n",
    "    If it contains strings, try to cast them as timestamps.\"\"\"\n",
    "    times = df[time_col]\n",
    "    if _is_dt_or_int(times):\n",
    "        return df\n",
    "    parse_err_msg = (\n",
    "        f\"Failed to parse '{time_col}' from string to datetime. \"\n",
    "        'Please make sure that it contains valid timestamps or integers.'\n",
    "    )\n",
    "    if isinstance(times, pd.Series) and pd.api.types.is_object_dtype(times):\n",
    "        try:\n",
    "            times = pd.to_datetime(times)\n",
    "        except ValueError:\n",
    "            raise ValueError(parse_err_msg)\n",
    "        df = ensure_shallow_copy(df.copy(deep=False))\n",
    "        df[time_col] = times\n",
    "    elif isinstance(times, pl_Series) and times.dtype == pl.Utf8:\n",
    "        try:\n",
    "            times = times.str.to_datetime()\n",
    "        except pl.exceptions.ComputeError:\n",
    "            raise ValueError(parse_err_msg)\n",
    "        df = df.with_columns(times)\n",
    "    else:\n",
    "        raise ValueError(f\"'{time_col}' should have valid timestamps or integers.\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "604ec21b-314f-42cb-9ee0-5950fd611b97",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.testing.assert_frame_equal(\n",
    "    ensure_time_dtype(pd.DataFrame({'ds': ['2000-01-01']})),\n",
    "    pd.DataFrame({'ds': pd.to_datetime(['2000-01-01'])})\n",
    ")\n",
    "df = pd.DataFrame({'ds': [1, 2]})\n",
    "assert df is ensure_time_dtype(df)\n",
    "test_fail(\n",
    "    lambda: ensure_time_dtype(pd.DataFrame({'ds': ['2000-14-14']})),\n",
    "    contains='Please make sure that it contains valid timestamps',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5335d217-e240-46df-90a0-6aeeb07a0586",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| polars\n",
    "pl.testing.assert_frame_equal(\n",
    "    ensure_time_dtype(pl.DataFrame({'ds': ['2000-01-01']})),\n",
    "    pl.DataFrame().with_columns(ds=pl.datetime(2000, 1, 1))\n",
    ")\n",
    "df = pl.DataFrame({'ds': [1, 2]})\n",
    "assert df is ensure_time_dtype(df)\n",
    "test_fail(\n",
    "    lambda: ensure_time_dtype(pl.DataFrame({'ds': ['hello']})),\n",
    "    contains='Please make sure that it contains valid timestamps',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fd76fe2-9b47-470b-8e63-f9fe639b2e0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def validate_format(\n",
    "    df: DataFrame,\n",
    "    id_col: str = 'unique_id',\n",
    "    time_col: str = 'ds',\n",
    "    target_col: Optional[str] = 'y',\n",
    ") -> None:\n",
    "    \"\"\"Ensure DataFrame has expected format.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df : pandas or polars DataFrame\n",
    "        DataFrame with time series in long format.\n",
    "    id_col : str (default='unique_id')\n",
    "        Column that identifies each serie.\n",
    "    time_col : str (default='ds')\n",
    "        Column that identifies each timestamp.\n",
    "    target_col : str, optional (default='y')\n",
    "        Column that contains the target.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    None\n",
    "    \"\"\"\n",
    "    if not isinstance(df, (pd.DataFrame, pl_DataFrame)):\n",
    "        raise ValueError(\n",
    "            f'`df` must be either pandas or polars dataframe, got {type(df)}'\n",
    "        )\n",
    "\n",
    "    # required columns\n",
    "    expected_cols = {id_col, time_col}\n",
    "    if target_col is not None:\n",
    "        expected_cols.add(target_col)\n",
    "    missing_cols = sorted(expected_cols - set(df.columns))\n",
    "    if missing_cols:\n",
    "        raise ValueError(f\"The following columns are missing: {missing_cols}\")\n",
    "\n",
    "    # time col\n",
    "    if not _is_dt_or_int(df[time_col]):\n",
    "        times_dtype = df[time_col].dtype\n",
    "        raise ValueError(f\"The time column ('{time_col}') should have either timestamps or integers, got '{times_dtype}'.\")\n",
    "\n",
    "    # target col\n",
    "    if target_col is None:\n",
    "        return None\n",
    "    target = df[target_col]\n",
    "    if isinstance(target, pd.Series):\n",
    "        is_numeric = pd.api.types.is_numeric_dtype(target.dtype)\n",
    "    else:\n",
    "        try:\n",
    "            is_numeric = target.dtype.is_numeric()\n",
    "        except AttributeError:\n",
    "            is_numeric = target.is_numeric()\n",
    "    if not is_numeric:\n",
    "        raise ValueError(f\"The target column ('{target_col}') should have a numeric data type, got '{target.dtype}')\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9d90a45-f371-49ab-b590-35f6a36fb888",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "from nbdev import show_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d26edadb-2fa0-41cf-8550-a3b29a76f839",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/Nixtla/utilsforecast/blob/main/utilsforecast/validation.py#L75){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### validate_format\n",
       "\n",
       ">      validate_format\n",
       ">                       (df:Union[pandas.core.frame.DataFrame,polars.dataframe.f\n",
       ">                       rame.DataFrame], id_col:str='unique_id',\n",
       ">                       time_col:str='ds', target_col:Optional[str]='y')\n",
       "\n",
       "Ensure DataFrame has expected format.\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| df | Union |  | DataFrame with time series in long format. |\n",
       "| id_col | str | unique_id | Column that identifies each serie. |\n",
       "| time_col | str | ds | Column that identifies each timestamp. |\n",
       "| target_col | Optional | y | Column that contains the target. |\n",
       "| **Returns** | **None** |  |  |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/Nixtla/utilsforecast/blob/main/utilsforecast/validation.py#L75){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### validate_format\n",
       "\n",
       ">      validate_format\n",
       ">                       (df:Union[pandas.core.frame.DataFrame,polars.dataframe.f\n",
       ">                       rame.DataFrame], id_col:str='unique_id',\n",
       ">                       time_col:str='ds', target_col:Optional[str]='y')\n",
       "\n",
       "Ensure DataFrame has expected format.\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| df | Union |  | DataFrame with time series in long format. |\n",
       "| id_col | str | unique_id | Column that identifies each serie. |\n",
       "| time_col | str | ds | Column that identifies each timestamp. |\n",
       "| target_col | Optional | y | Column that contains the target. |\n",
       "| **Returns** | **None** |  |  |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(validate_format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0072de7-2cb2-469d-a0c8-59d6d60e7858",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "from utilsforecast.compat import POLARS_INSTALLED, pl\n",
    "from utilsforecast.data import generate_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c86a3919-73a9-4cfd-886a-557ba2feaf24",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fail(lambda: validate_format(1), contains=\"got <class 'int'>\")\n",
    "constructors = [pd.DataFrame]\n",
    "if POLARS_INSTALLED:\n",
    "    constructors.append(pl.DataFrame)\n",
    "for constructor in constructors:\n",
    "    df = constructor({'unique_id': [1]})\n",
    "    test_fail(lambda: validate_format(df), contains=\"missing: ['ds', 'y']\")\n",
    "    df = constructor({'unique_id': [1], 'time': ['x'], 'y': [1]})\n",
    "    test_fail(lambda: validate_format(df, time_col='time'), contains=\"('time') should have either timestamps or integers\")\n",
    "    for time in [1, datetime.datetime(2000, 1, 1)]:\n",
    "        df = constructor({'unique_id': [1], 'ds': [time], 'sales': ['x']})\n",
    "        test_fail(lambda: validate_format(df, target_col='sales'), contains=\"('sales') should have a numeric data type\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d7bd3cb-4c35-4a0c-bab1-02899618454c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def validate_freq(\n",
    "    times: Series,\n",
    "    freq: Union[str, int],\n",
    ") -> None:\n",
    "    if _is_int_dtype(times) and not isinstance(freq, int):\n",
    "        raise ValueError(\n",
    "            \"Time column contains integers but the specified frequency is not an integer. \"\n",
    "            \"Please provide a valid integer, e.g. `freq=1`\"\n",
    "        )\n",
    "    if _is_dt_dtype(times) and isinstance(freq, int):\n",
    "        raise ValueError(\n",
    "            \"Time column contains timestamps but the specified frequency is an integer. \"\n",
    "            \"Please provide a valid pandas or polars offset, e.g. `freq='D'` or `freq='1d'`.\"\n",
    "        )\n",
    "    # try to catch pandas frequency in polars dataframe\n",
    "    if isinstance(times, pl_Series) and isinstance(freq, str):\n",
    "        missing_n = re.search(r\"\\d+\", freq) is None\n",
    "        uppercase = re.sub(r\"\\d+\", \"\", freq).isupper()\n",
    "        if missing_n or uppercase:\n",
    "            raise ValueError(\n",
    "                \"You must specify a valid polars offset when using polars dataframes. \"\n",
    "                \"You can find the available offsets in \"\n",
    "                \"https://pola-rs.github.io/polars/py-polars/html/reference/expressions/api/polars.Expr.dt.offset_by.html\"\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acf3ab95-5629-45e6-921d-341f1dcf0141",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_fail(lambda: validate_freq(pd.Series([1, 2]), 'D'), contains='provide a valid integer')\n",
    "test_fail(lambda: validate_freq(pd.to_datetime(['2000-01-01']).to_series(), 1), contains='provide a valid pandas or polars offset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f6d354b-1393-49c5-bc2f-b1787fc26fe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| polars\n",
    "test_fail(lambda: validate_freq(pl.Series([1, 2]), '1d'), contains='provide a valid integer')\n",
    "test_fail(lambda: validate_freq(pl.Series([datetime.datetime(2000, 1, 1)]), 1), contains='provide a valid pandas or polars offset')\n",
    "test_fail(lambda: validate_freq(pl.Series([datetime.datetime(2000, 1, 1)]), 'D'), contains='valid polars offset')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}