Rename MSSE into RelMSE, add new implementation of MSSE

hierarchicalforecast/_modidx.py

@@ -37,6 +37,8 @@
                                                                                           'hierarchicalforecast/evaluation.py'),
                                                  'hierarchicalforecast.evaluation.msse': ( 'evaluation.html#msse',
                                                                                            'hierarchicalforecast/evaluation.py'),
+                                                 'hierarchicalforecast.evaluation.rel_mse': ( 'evaluation.html#rel_mse',
+                                                                                              'hierarchicalforecast/evaluation.py'),
                                                  'hierarchicalforecast.evaluation.scaled_crps': ( 'evaluation.html#scaled_crps',
                                                                                                   'hierarchicalforecast/evaluation.py')},
             'hierarchicalforecast.methods': { 'hierarchicalforecast.methods.BottomUp': ( 'methods.html#bottomup',

hierarchicalforecast/evaluation.py

@@ -1,7 +1,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/evaluation.ipynb.
 
 # %% auto 0
-__all__ = ['msse', 'scaled_crps', 'energy_score', 'log_score', 'HierarchicalEvaluation']
+__all__ = ['rel_mse', 'msse', 'scaled_crps', 'energy_score', 'log_score', 'HierarchicalEvaluation']
 
 # %% ../nbs/evaluation.ipynb 3
 from inspect import signature
@@ -108,13 +108,13 @@ def mqloss(y: np.ndarray, y_hat: np.ndarray,
     return mqloss
 
 # %% ../nbs/evaluation.ipynb 8
-def msse(y, y_hat, y_train, mask=None):
-    """Mean Squared Scaled Error
+def rel_mse(y, y_hat, y_train, mask=None):
+    """Relative Mean Squared Error
 
-    Computes Mean squared scaled error (MSSE), as proposed by Hyndman & Koehler (2006)
+    Computes Relative mean squared error (RelMSE), as proposed by Hyndman & Koehler (2006)
     as an alternative to percentage errors, to avoid measure unstability.
 
-    $$ \mathrm{MSSE}(\\mathbf{y}, \\mathbf{\hat{y}}, \\mathbf{\hat{y}}^{naive1}) =
+    $$ \mathrm{RelMSE}(\\mathbf{y}, \\mathbf{\hat{y}}, \\mathbf{\hat{y}}^{naive1}) =
     \\frac{\mathrm{MSE}(\\mathbf{y}, \\mathbf{\hat{y}})}{\mathrm{MSE}(\\mathbf{y}, \\mathbf{\hat{y}}^{naive1})} $$
 
     **Parameters:**<br>
@@ -145,6 +145,50 @@ def msse(y, y_hat, y_train, mask=None):
     return loss
 
 # %% ../nbs/evaluation.ipynb 11
+def msse(y, y_hat, y_train, mask=None):
+    """Mean Squared Scaled Error
+
+    Computes Mean squared scaled error (MSSE), as proposed by Hyndman & Koehler (2006)
+    as an alternative to percentage errors, to avoid measure unstability.
+
+    $$ \\mathrm{MSSE}(\\mathbf{y}, \\mathbf{\\hat{y}}, \\mathbf{y}^{in-sample}) =
+    \\frac{\\frac{1}{h} \\sum^{n+h}_{t=n+1} (y_t - \\hat{y}_t)^2}{\\frac{1}{n-1} \\sum^{n}_{t=2} (y_t - y_{t-1})^2},$$
+
+    where $n$ ($n=$`n`) is the size of the training data, and $h$ is the forecasting horizon ($h=$`horizon`).
+
+    **Parameters:**<br>
+    `y`: numpy array, Actual values of size (`n_series`, `horizon`).<br>
+    `y_hat`: numpy array, Predicted values (`n_series`, `horizon`).<br>
+    `y_train`: numpy array, Predicted values (`n_series`, `n`).<br>
+    `mask`: numpy array, Specifies date stamps per serie to consider in loss.<br>
+
+    **Returns:**<br>
+    `loss`: float.    
+
+    **References:**<br>
+    - [Hyndman, R. J and Koehler, A. B. (2006).
+       "Another look at measures of forecast accuracy",
+       International Journal of Forecasting, Volume 22, Issue 4.](https://www.sciencedirect.com/science/article/pii/S0169207006000239)<br>
+    - [Kin G. Olivares, O. Nganba Meetei, Ruijun Ma, Rohan Reddy, Mengfei Cao, Lee Dicker. 
+       "Probabilistic Hierarchical Forecasting with Deep Poisson Mixtures. 
+       Submitted to the International Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)<br>
+    - [Makridakis, S., Spiliotis E., and Assimakopoulos V. (2022). 
+       "M5 Accuracy Competition: Results, Findings, and Conclusions.",
+       International Journal of Forecasting, Volume 38, Issue 4.](https://www.sciencedirect.com/science/article/pii/S0169207021001874)<br>
+    """
+    if mask is None: 
+       mask = np.ones_like(y)
+    n_series, horizon = y.shape
+
+    eps = np.finfo(float).eps
+    y_in_sample_naive = y_train[:, :-1]
+    y_in_sample_true = y_train[:, 1:]
+    norm = mse(y=y_in_sample_true, y_hat=y_in_sample_naive)
+    loss = mse(y=y, y_hat=y_hat, weights=mask)
+    loss = loss / (norm + eps)
+    return loss
+
+# %% ../nbs/evaluation.ipynb 14
 def scaled_crps(y, y_hat, quantiles):
     """Scaled Continues Ranked Probability Score
 
@@ -185,7 +229,7 @@ def scaled_crps(y, y_hat, quantiles):
     loss  = 2 * loss * np.sum(np.ones(y.shape)) / (norm + eps)
     return loss
 
-# %% ../nbs/evaluation.ipynb 14
+# %% ../nbs/evaluation.ipynb 17
 def energy_score(y, y_sample1, y_sample2, beta=2):
     """Energy Score
 
@@ -229,7 +273,7 @@ def energy_score(y, y_sample1, y_sample2, beta=2):
     score = np.mean(term2 - 0.5 * term1)
     return score
 
-# %% ../nbs/evaluation.ipynb 16
+# %% ../nbs/evaluation.ipynb 19
 def log_score(y, y_hat, cov, allow_singular=True):
     """ Log Score.
 
@@ -267,7 +311,7 @@ def log_score(y, y_hat, cov, allow_singular=True):
     score = np.mean(scores)
     return score
 
-# %% ../nbs/evaluation.ipynb 20
+# %% ../nbs/evaluation.ipynb 23
 class HierarchicalEvaluation:
     """Hierarchical Evaluation Class.
 

nbs/evaluation.ipynb

@@ -165,10 +165,11 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Mean Squared Scaled Error"
+    "## Relative Mean Squared Error"
    ]
   },
   {
@@ -178,13 +179,13 @@
    "outputs": [],
    "source": [
     "#| export\n",
-    "def msse(y, y_hat, y_train, mask=None):\n",
-    "    \"\"\"Mean Squared Scaled Error\n",
+    "def rel_mse(y, y_hat, y_train, mask=None):\n",
+    "    \"\"\"Relative Mean Squared Error\n",
     "\n",
-    "    Computes Mean squared scaled error (MSSE), as proposed by Hyndman & Koehler (2006)\n",
+    "    Computes Relative mean squared error (RelMSE), as proposed by Hyndman & Koehler (2006)\n",
     "    as an alternative to percentage errors, to avoid measure unstability.\n",
     "\n",
-    "    $$ \\mathrm{MSSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}, \\\\mathbf{\\hat{y}}^{naive1}) =\n",
+    "    $$ \\mathrm{RelMSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}, \\\\mathbf{\\hat{y}}^{naive1}) =\n",
     "    \\\\frac{\\mathrm{MSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}})}{\\mathrm{MSE}(\\\\mathbf{y}, \\\\mathbf{\\hat{y}}^{naive1})} $$\n",
     "\n",
     "    **Parameters:**<br>\n",
@@ -215,6 +216,73 @@
     "    return loss"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "show_doc(rel_mse)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Mean Squared Scaled Error"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#| export\n",
+    "def msse(y, y_hat, y_train, mask=None):\n",
+    "    \"\"\"Mean Squared Scaled Error\n",
+    "\n",
+    "    Computes Mean squared scaled error (MSSE), as proposed by Hyndman & Koehler (2006)\n",
+    "    as an alternative to percentage errors, to avoid measure unstability.\n",
+    "\n",
+    "    $$ \\\\mathrm{MSSE}(\\\\mathbf{y}, \\\\mathbf{\\\\hat{y}}, \\\\mathbf{y}^{in-sample}) =\n",
+    "    \\\\frac{\\\\frac{1}{h} \\\\sum^{n+h}_{t=n+1} (y_t - \\\\hat{y}_t)^2}{\\\\frac{1}{n-1} \\\\sum^{n}_{t=2} (y_t - y_{t-1})^2},$$\n",
+    "\n",
+    "    where $n$ ($n=$`n`) is the size of the training data, and $h$ is the forecasting horizon ($h=$`horizon`).\n",
+    "\n",
+    "    **Parameters:**<br>\n",
+    "    `y`: numpy array, Actual values of size (`n_series`, `horizon`).<br>\n",
+    "    `y_hat`: numpy array, Predicted values (`n_series`, `horizon`).<br>\n",
+    "    `y_train`: numpy array, Predicted values (`n_series`, `n`).<br>\n",
+    "    `mask`: numpy array, Specifies date stamps per serie to consider in loss.<br>\n",
+    "\n",
+    "    **Returns:**<br>\n",
+    "    `loss`: float.    \n",
+    "\n",
+    "    **References:**<br>\n",
+    "    - [Hyndman, R. J and Koehler, A. B. (2006).\n",
+    "       \"Another look at measures of forecast accuracy\",\n",
+    "       International Journal of Forecasting, Volume 22, Issue 4.](https://www.sciencedirect.com/science/article/pii/S0169207006000239)<br>\n",
+    "    - [Kin G. Olivares, O. Nganba Meetei, Ruijun Ma, Rohan Reddy, Mengfei Cao, Lee Dicker. \n",
+    "       \"Probabilistic Hierarchical Forecasting with Deep Poisson Mixtures. \n",
+    "       Submitted to the International Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)<br>\n",
+    "    - [Makridakis, S., Spiliotis E., and Assimakopoulos V. (2022). \n",
+    "       \"M5 Accuracy Competition: Results, Findings, and Conclusions.\",\n",
+    "       International Journal of Forecasting, Volume 38, Issue 4.](https://www.sciencedirect.com/science/article/pii/S0169207021001874)<br>\n",
+    "    \"\"\"\n",
+    "    if mask is None: \n",
+    "       mask = np.ones_like(y)\n",
+    "    n_series, horizon = y.shape\n",
+    "\n",
+    "    eps = np.finfo(float).eps\n",
+    "    y_in_sample_naive = y_train[:, :-1]\n",
+    "    y_in_sample_true = y_train[:, 1:]\n",
+    "    norm = mse(y=y_in_sample_true, y_hat=y_in_sample_naive)\n",
+    "    loss = mse(y=y, y_hat=y_hat, weights=mask)\n",
+    "    loss = loss / (norm + eps)\n",
+    "    return loss"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -686,7 +754,10 @@
     "Proceedings of the 38th International Conference on Machine Learning (ICML).](https://proceedings.mlr.press/v139/rangapuram21a.html)\n",
     "- [Kin G. Olivares, O. Nganba Meetei, Ruijun Ma, Rohan Reddy, Mengfei Cao, Lee Dicker (2022). \n",
     "\"Probabilistic Hierarchical Forecasting with Deep Poisson Mixtures\". \n",
-    "Submitted to the International Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)"
+    "Submitted to the International Journal Forecasting, Working paper available at arxiv.](https://arxiv.org/pdf/2110.13179.pdf)\n",
+    "- [Makridakis, S., Spiliotis E., and Assimakopoulos V. (2022). \n",
+    "\"M5 Accuracy Competition: Results, Findings, and Conclusions.\",\n",
+    "International Journal of Forecasting, Volume 38, Issue 4.](https://www.sciencedirect.com/science/article/pii/S0169207021001874)"
    ]
   },
   {

nbs/probabilistic_methods.ipynb

@@ -577,6 +577,7 @@
    "source": [
     "#| hide\n",
     "from hierarchicalforecast.evaluation import (\n",
+    "        rel_mse,\n",
     "        msse, \n",
     "        energy_score, \n",
     "        scaled_crps, \n",
@@ -675,6 +676,9 @@
    "outputs": [],
    "source": [
     "#| hide\n",
+    "# test RelMSE's execution\n",
+    "rel_mse(y=y_test, y_hat=y_hat_base, y_train=y_base)\n",
+    "\n",
     "# test MSSE's execution\n",
     "msse(y=y_test, y_hat=y_hat_base, y_train=y_base)\n",
     "\n",