From b18218fdc1b0673f8f93bbeda5c498533c18324c Mon Sep 17 00:00:00 2001 From: Shion Date: Sun, 17 Sep 2023 21:18:40 +0900 Subject: [PATCH] New metric: Davies bouldin score (#2071) * implementation * fix error in other metric * links + init + utils * add tests * changelog * fix inf * changelog * docs --- CHANGELOG.md | 2 + .../clustering/davies_bouldin_score.rst | 21 +++ docs/source/links.rst | 1 + src/torchmetrics/clustering/__init__.py | 2 + .../clustering/calinski_harabasz_score.py | 12 +- .../clustering/davies_bouldin_score.py | 136 ++++++++++++++++++ .../functional/clustering/__init__.py | 2 + .../clustering/calinski_harabasz_score.py | 23 +-- .../clustering/davies_bouldin_score.py | 67 +++++++++ .../functional/clustering/utils.py | 19 +++ .../clustering/test_davies_bouldin_score.py | 56 ++++++++ 11 files changed, 318 insertions(+), 23 deletions(-) create mode 100644 docs/source/clustering/davies_bouldin_score.rst create mode 100644 src/torchmetrics/clustering/davies_bouldin_score.py create mode 100644 src/torchmetrics/functional/clustering/davies_bouldin_score.py create mode 100644 tests/unittests/clustering/test_davies_bouldin_score.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fe89119136b..7c38909ede3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `AdjustedMutualInfoScore` ([#2058](https://github.com/Lightning-AI/torchmetrics/pull/2058)) + - `DaviesBouldinScore` ([#2071](https://github.com/Lightning-AI/torchmetrics/pull/2071)) + ### Changed diff --git a/docs/source/clustering/davies_bouldin_score.rst b/docs/source/clustering/davies_bouldin_score.rst new file mode 100644 index 00000000000..c3d87a3fa29 --- /dev/null +++ b/docs/source/clustering/davies_bouldin_score.rst @@ -0,0 +1,21 @@ +.. customcarditem:: + :header: Davies Bouldin Score + :image: https://pl-flash-data.s3.amazonaws.com/assets/thumbnails/default.svg + :tags: Clustering + +.. include:: ../links.rst + +#################### +Davies Bouldin Score +#################### + +Module Interface +________________ + +.. autoclass:: torchmetrics.clustering.DaviesBouldinScore + :exclude-members: update, compute + +Functional Interface +____________________ + +.. autofunction:: torchmetrics.functional.clustering.davies_bouldin_score diff --git a/docs/source/links.rst b/docs/source/links.rst index e998c8896be..7a670b3cd86 100644 --- a/docs/source/links.rst +++ b/docs/source/links.rst @@ -159,4 +159,5 @@ .. _fork of pycocotools: https://github.com/ppwwyyxx/cocoapi .. _Adjusted Rand Score: https://en.wikipedia.org/wiki/Rand_index#Adjusted_Rand_index .. _Dunn Index: https://en.wikipedia.org/wiki/Dunn_index +.. _Davies-Bouldin Score: https://en.wikipedia.org/wiki/Davies%E2%80%93Bouldin_index .. _Fowlkes-Mallows Index: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.fowlkes_mallows_score.html#sklearn.metrics.fowlkes_mallows_score diff --git a/src/torchmetrics/clustering/__init__.py b/src/torchmetrics/clustering/__init__.py index 14d13dce39a..61d5ee3c422 100644 --- a/src/torchmetrics/clustering/__init__.py +++ b/src/torchmetrics/clustering/__init__.py @@ -14,6 +14,7 @@ from torchmetrics.clustering.adjusted_mutual_info_score import AdjustedMutualInfoScore from torchmetrics.clustering.adjusted_rand_score import AdjustedRandScore from torchmetrics.clustering.calinski_harabasz_score import CalinskiHarabaszScore +from torchmetrics.clustering.davies_bouldin_score import DaviesBouldinScore from torchmetrics.clustering.dunn_index import DunnIndex from torchmetrics.clustering.fowlkes_mallows_index import FowlkesMallowsIndex from torchmetrics.clustering.mutual_info_score import MutualInfoScore @@ -24,6 +25,7 @@ "AdjustedMutualInfoScore", "AdjustedRandScore", "CalinskiHarabaszScore", + "DaviesBouldinScore", "DunnIndex", "FowlkesMallowsIndex", "MutualInfoScore", diff --git a/src/torchmetrics/clustering/calinski_harabasz_score.py b/src/torchmetrics/clustering/calinski_harabasz_score.py index f7c78af990d..5d038f5010b 100644 --- a/src/torchmetrics/clustering/calinski_harabasz_score.py +++ b/src/torchmetrics/clustering/calinski_harabasz_score.py @@ -106,9 +106,9 @@ def plot(self, val: Union[Tensor, Sequence[Tensor], None] = None, ax: Optional[_ >>> # Example plotting a single value >>> import torch - >>> from torchmetrics.clustering import RandScore - >>> metric = RandScore() - >>> metric.update(torch.randint(0, 4, (10,)), torch.randint(0, 4, (10,))) + >>> from torchmetrics.clustering import CalinskiHarabaszScore + >>> metric = CalinskiHarabaszScore() + >>> metric.update(torch.randn(10, 3), torch.randint(0, 2, (10,))) >>> fig_, ax_ = metric.plot(metric.compute()) .. plot:: @@ -116,10 +116,10 @@ def plot(self, val: Union[Tensor, Sequence[Tensor], None] = None, ax: Optional[_ >>> # Example plotting multiple values >>> import torch - >>> from torchmetrics.clustering import RandScore - >>> metric = RandScore() + >>> from torchmetrics.clustering import CalinskiHarabaszScore + >>> metric = CalinskiHarabaszScore() >>> for _ in range(10): - ... metric.update(torch.randint(0, 4, (10,)), torch.randint(0, 4, (10,))) + ... metric.update(torch.randn(10, 3), torch.randint(0, 2, (10,))) >>> fig_, ax_ = metric.plot(metric.compute()) """ diff --git a/src/torchmetrics/clustering/davies_bouldin_score.py b/src/torchmetrics/clustering/davies_bouldin_score.py new file mode 100644 index 00000000000..3c65d08cb8d --- /dev/null +++ b/src/torchmetrics/clustering/davies_bouldin_score.py @@ -0,0 +1,136 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, List, Optional, Sequence, Union + +from torch import Tensor + +from torchmetrics.functional.clustering.davies_bouldin_score import davies_bouldin_score +from torchmetrics.metric import Metric +from torchmetrics.utilities.data import dim_zero_cat +from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE +from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE + +if not _MATPLOTLIB_AVAILABLE: + __doctest_skip__ = ["DaviesBouldinScore.plot"] + + +class DaviesBouldinScore(Metric): + r"""Compute `Davies-Bouldin Score`_ for clustering algorithms. + + Given the following quantities: + + ..math:: + S_i = \left( \frac{1}{T_i} \sum_{j=1}^{T_i} ||X_j - A_i||^2_2 \right)^{1/2} + + where :math:`T_i` is the number of samples in cluster :math:`i`, :math:`X_j` is the :math:`j`-th sample in cluster + :math:`i`, and :math:`A_i` is the centroid of cluster :math:`i`. This quantity is the average distance between all + the samples in cluster :math:`i` and its centroid. Let + + ..math:: + M_{i,j} = ||A_i - A_j||_2 + + e.g. the distance between the centroids of cluster :math:`i` and cluster :math:`j`. Then the Davies-Bouldin score + is defined as: + + ..math:: + DB = \frac{1}{n_{clusters}} \sum_{i=1}^{n_{clusters}} \max_{j \neq i} \left( \frac{S_i + S_j}{M_{i,j}} \right) + + This clustering metric is an intrinsic measure, because it does not rely on ground truth labels for the evaluation. + Instead it examines how well the clusters are separated from each other. The score is higher when clusters are dense + and well separated, which relates to a standard concept of a cluster. + + As input to ``forward`` and ``update`` the metric accepts the following input: + + - ``data`` (:class:`~torch.Tensor`): float tensor with shape ``(N,d)`` with the embedded data. ``d`` is the + dimensionality of the embedding space. + - ``labels`` (:class:`~torch.Tensor`): single integer tensor with shape ``(N,)`` with cluster labels + + As output of ``forward`` and ``compute`` the metric returns the following output: + + - ``chs`` (:class:`~torch.Tensor`): A tensor with the Calinski Harabasz Score + + Args: + kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info. + + Example: + >>> import torch + >>> from torchmetrics.clustering import DaviesBouldinScore + >>> _ = torch.manual_seed(42) + >>> data = torch.randn(10, 3) + >>> labels = torch.randint(3, (10,)) + >>> metric = DaviesBouldinScore() + >>> metric(data, labels) + tensor(1.2540) + + """ + is_differentiable: bool = True + higher_is_better: bool = True + full_state_update: bool = False + plot_lower_bound: float = 0.0 + data: List[Tensor] + labels: List[Tensor] + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + + self.add_state("data", default=[], dist_reduce_fx="cat") + self.add_state("labels", default=[], dist_reduce_fx="cat") + + def update(self, data: Tensor, labels: Tensor) -> None: + """Update metric state with new data and labels.""" + self.data.append(data) + self.labels.append(labels) + + def compute(self) -> Tensor: + """Compute the Davies Bouldin Score over all data and labels.""" + return davies_bouldin_score(dim_zero_cat(self.data), dim_zero_cat(self.labels)) + + def plot(self, val: Union[Tensor, Sequence[Tensor], None] = None, ax: Optional[_AX_TYPE] = None) -> _PLOT_OUT_TYPE: + """Plot a single or multiple values from the metric. + + Args: + val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results. + If no value is provided, will automatically call `metric.compute` and plot that result. + ax: An matplotlib axis object. If provided will add plot to that axis + + Returns: + Figure and Axes object + + Raises: + ModuleNotFoundError: + If `matplotlib` is not installed + + .. plot:: + :scale: 75 + + >>> # Example plotting a single value + >>> import torch + >>> from torchmetrics.clustering import DaviesBouldinScore + >>> metric = DaviesBouldinScore() + >>> metric.update(torch.randn(10, 3), torch.randint(0, 2, (10,))) + >>> fig_, ax_ = metric.plot(metric.compute()) + + .. plot:: + :scale: 75 + + >>> # Example plotting multiple values + >>> import torch + >>> from torchmetrics.clustering import DaviesBouldinScore + >>> metric = DaviesBouldinScore() + >>> for _ in range(10): + ... metric.update(torch.randn(10, 3), torch.randint(0, 2, (10,))) + >>> fig_, ax_ = metric.plot(metric.compute()) + + """ + return self._plot(val, ax) diff --git a/src/torchmetrics/functional/clustering/__init__.py b/src/torchmetrics/functional/clustering/__init__.py index 1a84ceda310..9f3426fc71a 100644 --- a/src/torchmetrics/functional/clustering/__init__.py +++ b/src/torchmetrics/functional/clustering/__init__.py @@ -14,6 +14,7 @@ from torchmetrics.functional.clustering.adjusted_mutual_info_score import adjusted_mutual_info_score from torchmetrics.functional.clustering.adjusted_rand_score import adjusted_rand_score from torchmetrics.functional.clustering.calinski_harabasz_score import calinski_harabasz_score +from torchmetrics.functional.clustering.davies_bouldin_score import davies_bouldin_score from torchmetrics.functional.clustering.dunn_index import dunn_index from torchmetrics.functional.clustering.fowlkes_mallows_index import fowlkes_mallows_index from torchmetrics.functional.clustering.mutual_info_score import mutual_info_score @@ -24,6 +25,7 @@ "adjusted_mutual_info_score", "adjusted_rand_score", "calinski_harabasz_score", + "davies_bouldin_score", "dunn_index", "fowlkes_mallows_index", "mutual_info_score", diff --git a/src/torchmetrics/functional/clustering/calinski_harabasz_score.py b/src/torchmetrics/functional/clustering/calinski_harabasz_score.py index 244a657f42e..5634dbb2fab 100644 --- a/src/torchmetrics/functional/clustering/calinski_harabasz_score.py +++ b/src/torchmetrics/functional/clustering/calinski_harabasz_score.py @@ -14,15 +14,10 @@ import torch from torch import Tensor - -def _calinski_harabasz_score_validate_input(data: Tensor, labels: Tensor) -> None: - """Validate that the input data and labels have correct shape and type.""" - if data.ndim != 2: - raise ValueError(f"Expected 2D data, got {data.ndim}D data instead") - if not data.is_floating_point(): - raise ValueError(f"Expected floating point data, got {data.dtype} data instead") - if labels.ndim != 1: - raise ValueError(f"Expected 1D labels, got {labels.ndim}D labels instead") +from torchmetrics.functional.clustering.utils import ( + _validate_intrinsic_cluster_data, + _validate_intrinsic_labels_to_samples, +) def calinski_harabasz_score(data: Tensor, labels: Tensor) -> Tensor: @@ -45,19 +40,13 @@ def calinski_harabasz_score(data: Tensor, labels: Tensor) -> Tensor: tensor(3.4998) """ - _calinski_harabasz_score_validate_input(data, labels) + _validate_intrinsic_cluster_data(data, labels) # convert to zero indexed labels unique_labels, labels = torch.unique(labels, return_inverse=True) n_labels = len(unique_labels) - n_samples = data.shape[0] - - if not 1 < n_labels < n_samples: - raise ValueError( - "Number of detected clusters must be greater than one and less than the number of samples." - f"Got {n_labels} clusters and {n_samples} samples." - ) + _validate_intrinsic_labels_to_samples(n_labels, n_samples) mean = data.mean(dim=0) between_cluster_dispersion = torch.tensor(0.0, device=data.device) diff --git a/src/torchmetrics/functional/clustering/davies_bouldin_score.py b/src/torchmetrics/functional/clustering/davies_bouldin_score.py new file mode 100644 index 00000000000..d4854ac3ef4 --- /dev/null +++ b/src/torchmetrics/functional/clustering/davies_bouldin_score.py @@ -0,0 +1,67 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +from torch import Tensor + +from torchmetrics.functional.clustering.utils import ( + _validate_intrinsic_cluster_data, + _validate_intrinsic_labels_to_samples, +) + + +def davies_bouldin_score(data: Tensor, labels: Tensor) -> Tensor: + """Compute the Davies bouldin score for clustering algorithms. + + Args: + data: float tensor with shape ``(N,d)`` with the embedded data. + labels: single integer tensor with shape ``(N,)`` with cluster labels + + Returns: + Scalar tensor with the Davies bouldin score + + Example: + >>> import torch + >>> from torchmetrics.functional.clustering import davies_bouldin_score + >>> _ = torch.manual_seed(42) + >>> data = torch.randn(10, 3) + >>> labels = torch.randint(0, 2, (10,)) + >>> davies_bouldin_score(data, labels) + tensor(1.3249) + + """ + _validate_intrinsic_cluster_data(data, labels) + + # convert to zero indexed labels + unique_labels, labels = torch.unique(labels, return_inverse=True) + n_labels = len(unique_labels) + n_samples, dim = data.shape + _validate_intrinsic_labels_to_samples(n_labels, n_samples) + + intra_dists = torch.zeros(n_labels, device=data.device) + centroids = torch.zeros((n_labels, dim), device=data.device) + for k in range(n_labels): + cluster_k = data[labels == k, :] + centroids[k] = cluster_k.mean(dim=0) + intra_dists[k] = (cluster_k - centroids[k]).pow(2.0).sum(dim=1).sqrt().mean() + centroid_distances = torch.cdist(centroids, centroids) + + cond1 = torch.allclose(intra_dists, torch.zeros_like(intra_dists)) + cond2 = torch.allclose(centroid_distances, torch.zeros_like(centroid_distances)) + if cond1 or cond2: + return torch.tensor(0.0, device=data.device, dtype=torch.float32) + + centroid_distances[centroid_distances == 0] = float("inf") + combined_intra_dists = intra_dists.unsqueeze(0) + intra_dists.unsqueeze(1) + scores = (combined_intra_dists / centroid_distances).max(dim=1).values + return scores.mean() diff --git a/src/torchmetrics/functional/clustering/utils.py b/src/torchmetrics/functional/clustering/utils.py index 5762233a15b..ba5057a1291 100644 --- a/src/torchmetrics/functional/clustering/utils.py +++ b/src/torchmetrics/functional/clustering/utils.py @@ -181,6 +181,25 @@ def check_cluster_labels(preds: Tensor, target: Tensor) -> None: raise ValueError(f"Expected real, discrete values for x but received {preds.dtype} and {target.dtype}.") +def _validate_intrinsic_cluster_data(data: Tensor, labels: Tensor) -> None: + """Validate that the input data and labels have correct shape and type.""" + if data.ndim != 2: + raise ValueError(f"Expected 2D data, got {data.ndim}D data instead") + if not data.is_floating_point(): + raise ValueError(f"Expected floating point data, got {data.dtype} data instead") + if labels.ndim != 1: + raise ValueError(f"Expected 1D labels, got {labels.ndim}D labels instead") + + +def _validate_intrinsic_labels_to_samples(n_labels: int, n_samples: int) -> None: + """Validate that the number of labels are in the correct range.""" + if not 1 < n_labels < n_samples: + raise ValueError( + "Number of detected clusters must be greater than one and less than the number of samples." + f"Got {n_labels} clusters and {n_samples} samples." + ) + + def calcualte_pair_cluster_confusion_matrix( preds: Optional[Tensor] = None, target: Optional[Tensor] = None, diff --git a/tests/unittests/clustering/test_davies_bouldin_score.py b/tests/unittests/clustering/test_davies_bouldin_score.py new file mode 100644 index 00000000000..9ae8bbd4a0d --- /dev/null +++ b/tests/unittests/clustering/test_davies_bouldin_score.py @@ -0,0 +1,56 @@ +# Copyright The Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from sklearn.metrics import davies_bouldin_score as sklearn_davies_bouldin_score +from torchmetrics.clustering.davies_bouldin_score import DaviesBouldinScore +from torchmetrics.functional.clustering.davies_bouldin_score import davies_bouldin_score + +from unittests.clustering.inputs import _single_target_intrinsic1, _single_target_intrinsic2 +from unittests.helpers import seed_all +from unittests.helpers.testers import MetricTester + +seed_all(42) + + +@pytest.mark.parametrize( + "data, labels", + [ + (_single_target_intrinsic1.data, _single_target_intrinsic1.labels), + (_single_target_intrinsic2.data, _single_target_intrinsic2.labels), + ], +) +class TestDaviesBouldinScore(MetricTester): + """Test class for `DaviesBouldinScore` metric.""" + + atol = 1e-5 + + @pytest.mark.parametrize("ddp", [True, False]) + def test_davies_bouldin_score(self, data, labels, ddp): + """Test class implementation of metric.""" + self.run_class_metric_test( + ddp=ddp, + preds=data, + target=labels, + metric_class=DaviesBouldinScore, + reference_metric=sklearn_davies_bouldin_score, + ) + + def test_davies_bouldin_score_functional(self, data, labels): + """Test functional implementation of metric.""" + self.run_functional_metric_test( + preds=data, + target=labels, + metric_functional=davies_bouldin_score, + reference_metric=sklearn_davies_bouldin_score, + )