Add Holographic Reduced Representations VSA model (#109)

* Add HRR vsa model

* [github-action] formatting fixes

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: mikeheddes

docs/torchhd.rst

@@ -80,7 +80,7 @@ VSA Models
     VSA_Model
     BSC
     MAP
-    .. HRR
+    HRR
     FHRR
 
 

torchhd/functional.py

@@ -7,6 +7,7 @@
 from torchhd.base import VSA_Model
 from torchhd.bsc import BSC
 from torchhd.map import MAP
+from torchhd.hrr import HRR
 from torchhd.fhrr import FHRR
 
 
@@ -484,6 +485,11 @@ def circular_hv(
                 [-0.887-0.460j, -0.906+0.421j, -0.727-0.686j, -0.271+0.962j, -0.705-0.709j,  0.562-0.827j]])
 
     """
+    if model == HRR:
+        raise ValueError(
+            "The circular hypervectors don't currently work with the HRR model. We are not sure why, if you have any insight that could help please share it at: https://github.com/hyperdimensional-computing/torchhd/issues/108."
+        )
+
     # convert from normalized "randomness" variable r to
     # number of levels between orthogonal pairs or "span"
     levels_per_span = ((1 - randomness) * (num_vectors / 2) + randomness * 1) * 2

torchhd/hrr.py

@@ -8,6 +8,11 @@
 
 
 class HRR(VSA_Model):
+    """Holographic Reduced Representation
+
+    Proposed in `Holographic reduced representations <https://ieeexplore.ieee.org/document/377968>`_, this model uses real valued hypervectors.
+    """
+
     supported_dtypes: Set[torch.dtype] = {torch.float32, torch.float64}
 
     @classmethod
@@ -20,6 +25,30 @@ def empty_hv(
         device=None,
         requires_grad=False,
     ) -> "HRR":
+        """Creates a set of hypervectors representing empty sets.
+
+        When bundled with a random-hypervector :math:`x`, the result is :math:`x`.
+        The empty vector of the HRR model is simply a set of 0 values.
+
+        Args:
+            num_vectors (int): the number of hypervectors to generate.
+            dimensions (int): the dimensionality of the hypervectors.
+            dtype (``torch.dtype``, optional): the desired data type of returned tensor. Default: if ``None`` is ``torch.get_default_dtype()``.
+            device (``torch.device``, optional):  the desired device of returned tensor. Default: if ``None``, uses the current device for the default tensor type (see torch.set_default_tensor_type()). ``device`` will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types.
+            requires_grad (bool, optional): If autograd should record operations on the returned tensor. Default: ``False``.
+
+        Examples::
+
+            >>> torchhd.HRR.empty_hv(3, 6)
+            HRR([[0., 0., 0., 0., 0., 0.],
+                [0., 0., 0., 0., 0., 0.],
+                [0., 0., 0., 0., 0., 0.]])
+            >>> torchhd.HRR.empty_hv(3, 6, dtype=torch.float64)
+            HRR([[0., 0., 0., 0., 0., 0.],
+                [0., 0., 0., 0., 0., 0.],
+                [0., 0., 0., 0., 0., 0.]], dtype=torch.float64)
+
+        """
 
         if dtype is None:
             dtype = torch.get_default_dtype()
@@ -48,6 +77,29 @@ def identity_hv(
         device=None,
         requires_grad=False,
     ) -> "HRR":
+        """Creates a set of identity hypervectors.
+
+        When bound with a random-hypervector :math:`x`, the result is :math:`x`.
+
+        Args:
+            num_vectors (int): the number of hypervectors to generate.
+            dimensions (int): the dimensionality of the hypervectors.
+            dtype (``torch.dtype``, optional): the desired data type of returned tensor. Default: if ``None`` is ``torch.get_default_dtype()``.
+            device (``torch.device``, optional):  the desired device of returned tensor. Default: if ``None``, uses the current device for the default tensor type (see torch.set_default_tensor_type()). ``device`` will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types.
+            requires_grad (bool, optional): If autograd should record operations on the returned tensor. Default: ``False``.
+
+        Examples::
+
+            >>> torchhd.HRR.identity_hv(3, 6)
+            HRR([[1., 0., 0., 0., 0., 0.],
+                 [1., 0., 0., 0., 0., 0.],
+                 [1., 0., 0., 0., 0., 0.]])
+            >>> torchhd.HRR.identity_hv(3, 6, dtype=torch.float64)
+            HRR([[1., 0., 0., 0., 0., 0.],
+                 [1., 0., 0., 0., 0., 0.],
+                 [1., 0., 0., 0., 0., 0.]], dtype=torch.float64)
+
+        """
 
         if dtype is None:
             dtype = torch.get_default_dtype()
@@ -57,13 +109,13 @@ def identity_hv(
             options = ", ".join([str(x) for x in cls.supported_dtypes])
             raise ValueError(f"{name} vectors must be one of dtype {options}.")
 
-        result = torch.ones(
+        result = torch.zeros(
             num_vectors,
             dimensions,
             dtype=dtype,
             device=device,
         )
-        result = torch.real(ifft(result))
+        result[:, 0] = 1
         result.requires_grad = requires_grad
         return result.as_subclass(cls)
 
@@ -78,6 +130,29 @@ def random_hv(
         device=None,
         requires_grad=False,
     ) -> "HRR":
+        """Creates a set of random independent hypervectors.
+
+        The resulting hypervectors are sampled at random from a normal with mean 0 and standard deviation 1/dimensions.
+
+        Args:
+            num_vectors (int): the number of hypervectors to generate.
+            dimensions (int): the dimensionality of the hypervectors.
+            generator (``torch.Generator``, optional): a pseudorandom number generator for sampling.
+            dtype (``torch.dtype``, optional): the desired data type of returned tensor. Default: if ``None`` is ``torch.get_default_dtype()``.
+            device (``torch.device``, optional):  the desired device of returned tensor. Default: if ``None``, uses the current device for the default tensor type (see torch.set_default_tensor_type()). ``device`` will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types.
+            requires_grad (bool, optional): If autograd should record operations on the returned tensor. Default: ``False``.
+
+        Examples::
+
+            >>> torchhd.HRR.random_hv(3, 6)
+            HRR([[ 0.2520, -0.0048, -0.0351,  0.2067,  0.0638, -0.0729],
+                 [-0.2695,  0.0815,  0.0103,  0.2211, -0.1202,  0.2134],
+                 [ 0.0086, -0.1748, -0.1715,  0.3215, -0.1353,  0.0044]])
+            >>> torchhd.HRR.random_hv(3, 6, dtype=torch.float64)
+            HRR([[-0.1327, -0.0396, -0.0065,  0.0886, -0.4665,  0.2656],
+                 [-0.2879, -0.1070, -0.0851, -0.4366, -0.1311,  0.3976],
+                 [-0.0472,  0.2987, -0.1567,  0.1496, -0.0098,  0.0344]], dtype=torch.float64)
+        """
 
         if dtype is None:
             dtype = torch.get_default_dtype()
@@ -91,40 +166,183 @@ def random_hv(
         result = torch.empty(size, dtype=dtype, device=device)
         result.normal_(0, 1.0 / dimensions, generator=generator)
 
-        # projection
-        # f = torch.abs(fft(result))
-        # p = ifft(fft(result) / f).real
-        # result = torch.nan_to_num(p)
         result.requires_grad = requires_grad
         return result.as_subclass(cls)
 
     def bundle(self, other: "HRR") -> "HRR":
+        r"""Bundle the hypervector with other using element-wise sum.
+
+        This produces a hypervector maximally similar to both.
+
+        The bundling operation is used to aggregate information into a single hypervector.
+
+        Args:
+            other (HRR): other input hypervector
+
+        Shapes:
+            - Self: :math:`(*)`
+            - Other: :math:`(*)`
+            - Output: :math:`(*)`
+
+        Examples::
+
+            >>> a, b = torchhd.HRR.random_hv(2, 6)
+            >>> a
+            HRR([ 0.1916, -0.1451, -0.0678,  0.0829,  0.3816, -0.0906])
+            >>> b
+            HRR([-0.2825,  0.3788,  0.0885, -0.1269, -0.0481, -0.3029])
+            >>> a.bundle(b)
+            HRR([-0.0909,  0.2336,  0.0207, -0.0440,  0.3336, -0.3935])
+
+            >>> a, b = torchhd.HRR.random_hv(2, 6, dtype=torch.float64)
+            >>> a
+            HRR([ 0.3879, -0.0452, -0.0082, -0.2262, -0.2764,  0.0166], dtype=torch.float64)
+            >>> b
+            HRR([ 0.0738, -0.0306,  0.4948,  0.1209,  0.1482,  0.1268], dtype=torch.float64)
+            >>> a.bundle(b)
+            HRR([ 0.4618, -0.0758,  0.4866, -0.1053, -0.1281,  0.1434], dtype=torch.float64)
+
+        """
         return self.add(other)
 
     def multibundle(self) -> "HRR":
+        """Bundle multiple hypervectors"""
         return self.sum(dim=-2, dtype=self.dtype)
 
     def bind(self, other: "HRR") -> "HRR":
+        r"""Bind the hypervector with other using circular convolution.
+
+        This produces a hypervector dissimilar to both.
+
+        Binding is used to associate information, for instance, to assign values to variables.
+
+        Args:
+            other (HRR): other input hypervector
+
+        Shapes:
+            - Self: :math:`(*)`
+            - Other: :math:`(*)`
+            - Output: :math:`(*)`
+
+        Examples::
+
+            >>> a, b = torchhd.HRR.random_hv(2, 6)
+            >>> a
+            HRR([ 0.0101, -0.2474, -0.0097, -0.0788,  0.1541, -0.1766])
+            >>> b
+            HRR([-0.0338,  0.0340,  0.0289, -0.1498,  0.1178, -0.2822])
+            >>> a.bind(b)
+            HRR([ 0.0786, -0.0260,  0.0591, -0.0706,  0.0799, -0.0216])
+
+            >>> a, b = torchhd.HRR.random_hv(2, 6, dtype=torch.float64)
+            >>> a
+            HRR([ 0.0354, -0.0818,  0.0216,  0.0384,  0.2961,  0.1976], dtype=torch.float64)
+            >>> b
+            HRR([ 0.3640, -0.0640, -0.1033, -0.1454,  0.0999,  0.0299], dtype=torch.float64)
+            >>> a.bind(b)
+            HRR([-0.0362, -0.0910,  0.0114,  0.0445,  0.1244,  0.0388], dtype=torch.float64)
+
+        """
         result = ifft(torch.mul(fft(self), fft(other)))
-        return result.real
+        return torch.real(result)
 
     def multibind(self) -> "HRR":
+        """Bind multiple hypervectors"""
         result = ifft(torch.prod(fft(self), dim=-2, dtype=self.dtype))
-        return result.real
+        return torch.real(result)
+
+    def exact_inverse(self) -> "HRR":
+        """Unstable, but exact, inverse"""
+        result = ifft(1.0 / fft(self).conj())
+        result = torch.real(result)
+        return torch.nan_to_num(result)
 
     def inverse(self) -> "HRR":
-        return self.flip(dims=(-1,)).roll(1, dims=-1)
+        r"""Stable inversion of the hypervector for binding.
+
+        For HRR the stable inverse of hypervector is its conjugate in the frequency domain, this returns the conjugate of the hypervector.
+
+        Shapes:
+            - Self: :math:`(*)`
+            - Output: :math:`(*)`
+
+        Examples::
+
+            >>> a = torchhd.HRR.random_hv(1, 6)
+            >>> a
+            HRR([[ 0.1406,  0.0014, -0.0502,  0.2888,  0.2969, -0.2637]])
+            >>> a.inverse()
+            HRR([[ 0.1406, -0.2637,  0.2969,  0.2888, -0.0502,  0.0014]])
+
+            >>> a = torchhd.HRR.random_hv(1, 6, dtype=torch.float64)
+            >>> a
+            HRR([[ 0.0090,  0.2620,  0.0836,  0.0441, -0.2351, -0.1744]], dtype=torch.float64)
+            >>> a.inverse()
+            HRR([[ 0.0090, -0.1744, -0.2351,  0.0441,  0.0836,  0.2620]], dtype=torch.float64)
+
+        """
+        result = ifft(fft(self).conj())
+        return torch.real(result)
 
     def negative(self) -> "HRR":
+        r"""Negate the hypervector for the bundling inverse.
+
+        Shapes:
+            - Self: :math:`(*)`
+            - Output: :math:`(*)`
+
+        Examples::
+
+            >>> a = torchhd.HRR.random_hv(1, 6)
+            >>> a
+            HRR([[ 0.2658, -0.2808,  0.1436,  0.1131,  0.1567, -0.1426]])
+            >>> a.negative()
+            HRR([[-0.2658,  0.2808, -0.1436, -0.1131, -0.1567,  0.1426]])
+
+            >>> a = torchhd.HRR.random_hv(1, 6, dtype=torch.float64)
+            >>> a
+            HRR([[ 0.0318,  0.1944,  0.1229,  0.0193,  0.0135, -0.2521]], dtype=torch.float64)
+            >>> a.negative()
+            HRR([[-0.0318, -0.1944, -0.1229, -0.0193, -0.0135,  0.2521]], dtype=torch.float64)
+
+        """
         return torch.negative(self)
 
     def permute(self, shifts: int = 1) -> "HRR":
+        r"""Permute the hypervector.
+
+        The permutation operator is used to assign an order to hypervectors.
+
+        Args:
+            shifts (int, optional): The number of places by which the elements of the tensor are shifted.
+
+        Shapes:
+            - Self: :math:`(*)`
+            - Output: :math:`(*)`
+
+        Examples::
+
+            >>> a = torchhd.HRR.random_hv(1, 6)
+            >>> a
+            HRR([[-0.2521,  0.1140, -0.1647, -0.1490, -0.2091, -0.0618]])
+            >>> a.permute()
+            HRR([[-0.0618, -0.2521,  0.1140, -0.1647, -0.1490, -0.2091]])
+
+            >>> a = torchhd.HRR.random_hv(1, 6, dtype=torch.float64)
+            >>> a
+            HRR([[-0.0495, -0.0318,  0.3923, -0.3205,  0.1587,  0.1926]], dtype=torch.float64)
+            >>> a.permute()
+            HRR([[ 0.1926, -0.0495, -0.0318,  0.3923, -0.3205,  0.1587]], dtype=torch.float64)
+
+        """
         return self.roll(shifts=shifts, dims=-1)
 
     def dot_similarity(self, others: "HRR") -> Tensor:
+        """Inner product with other hypervectors"""
         return F.linear(self, others)
 
     def cos_similarity(self, others: "HRR", *, eps=1e-08) -> Tensor:
+        """Cosine similarity with other hypervectors"""
         self_dot = torch.sum(self * self, dim=-1)
         self_mag = self_dot.sqrt()