Add sinusoid embedding and fix projection embedding (#88)

* Fix projection embedding and implement cosine-based

* WIP

* Add sinusoid encoding and fix projection encoding

Co-authored-by: Didanny <daa50@mail.aub.edu>

Author: mikeheddes

docs/embeddings.rst

@@ -13,4 +13,5 @@ torchhd.embeddings
     Random
     Level
     Circular
-    Projection
+    Projection
+    Sinusoid

sandbox.py

@@ -0,0 +1,31 @@
+import pytest
+import torch
+
+from torchhd import functional
+from torchhd import embeddings
+
+# torch.float32,
+# torch.float64,
+# torch.complex64,
+# torch.complex128,
+# torch.float16,
+# torch.bfloat16,
+# torch.uint8,
+# torch.int8,
+# torch.int16,
+# torch.int32,
+# torch.int64,
+# torch.bool,
+
+# from .utils import (
+#     torch_dtypes,
+#     torch_complex_dtypes,
+#     supported_dtype,
+# )
+
+for i in range(5, 20):
+    emb = embeddings.Identity(i, 3)
+    idx = torch.LongTensor([0, 1, 4])
+    res = emb(idx)
+
+    print("{0},{1}".format(res.size(dim=0),res.size(dim=1)))

torchhd/embeddings.py

@@ -11,6 +11,7 @@
     "Level",
     "Circular",
     "Projection",
+    "Sinusoid",
 ]
 
 
@@ -230,7 +231,7 @@ class Projection(nn.Module):
     r"""Embedding using a random projection matrix.
 
     Implemented based on `A Theoretical Perspective on Hyperdimensional Computing <https://arxiv.org/abs/2010.07426>`_.
-    :math:`\Phi x` where :math:`\Phi \in \mathbb{R}^{d \times m}` is a matrix whose rows are uniformly sampled at random from the surface of an :math:`m`-dimensional unit sphere.
+    It computes :math:`x \Phi^{\mathsf{T}}` where :math:`\Phi \in \mathbb{R}^{d \times m}` is a matrix whose rows are uniformly sampled at random from the surface of an :math:`d`-dimensional unit sphere.
     This encoding ensures that similarities in the input space are preserved in the hyperspace.
 
     Args:
@@ -242,11 +243,16 @@ class Projection(nn.Module):
 
     Examples::
 
-        >>> emb = embeddings.Projection(5, 3)
-        >>> x = torch.rand(2, 5)
-        >>> emb(x)
-        tensor([[ 0.2747, -0.8804, -0.6810],
-                [ 0.5610, -0.9227,  0.1671]])
+        >>> embed = embeddings.Projection(6, 5)
+        >>> x = torch.randn(3, 6)
+        >>> x
+        tensor([[ 0.4119, -0.4284,  1.8022,  0.3715, -1.4563, -0.2842],
+                [-0.3772, -1.2664, -1.5173,  1.3317,  0.4707, -1.3362],
+                [-1.8142,  0.0274, -1.0989,  0.8193,  0.7619,  0.9181]])
+        >>> embed(x).sign()
+        tensor([[-1.,  1.,  1.,  1.,  1.],
+                [ 1.,  1.,  1.,  1.,  1.],
+                [ 1., -1., -1., -1., -1.]])
 
     """
 
@@ -270,8 +276,70 @@ def __init__(
         self.reset_parameters()
 
     def reset_parameters(self) -> None:
-        nn.init.uniform_(self.weight, -1, 1)
-        self.weight.data[:] = F.normalize(self.weight.data)
+        nn.init.normal_(self.weight, 0, 1)
+        self.weight.data.copy_(F.normalize(self.weight.data))
 
     def forward(self, input: torch.Tensor) -> torch.Tensor:
         return F.linear(input, self.weight)
+
+
+class Sinusoid(nn.Module):
+    r"""Embedding using a nonlinear random projection
+
+    Implemented based on `Scalable Edge-Based Hyperdimensional Learning System with Brain-Like Neural Adaptation <https://dl.acm.org/doi/abs/10.1145/3458817.3480958>`_.
+    It computes :math:`\cos(x \Phi^{\mathsf{T}} + b) \odot \sin(x \Phi^{\mathsf{T}})` where :math:`\Phi \in \mathbb{R}^{d \times m}` is a matrix whose elements are sampled at random from a standard normal distribution and :math:`b \in \mathbb{R}^{d}` is a vectors whose elements are sampled uniformly at random between 0 and :math:`2\pi`.
+
+    Args:
+        in_features (int): the dimensionality of the input feature vector.
+        out_features (int): the dimensionality of the hypervectors.
+        requires_grad (bool, optional): If autograd should record operations on the returned tensor. Default: ``False``.
+        dtype (``torch.dtype``, optional): the desired data type of returned tensor. Default: if ``None``, uses a global default (see ``torch.set_default_tensor_type()``).
+        device (``torch.device``, optional):  the desired device of returned tensor. Default: if ``None``, uses the current device for the default tensor type (see torch.set_default_tensor_type()). ``device`` will be the CPU for CPU tensor types and the current CUDA device for CUDA tensor types.
+
+    Examples::
+
+        >>> embed = embeddings.Sinusoid(6, 5)
+        >>> x = torch.randn(3, 6)
+        >>> x
+        tensor([[ 0.5043,  0.3161, -0.0938,  0.6134, -0.1280,  0.3647],
+                [-0.1907,  1.6468, -0.3242,  0.8614,  0.3332, -0.2055],
+                [-0.8662, -1.3861, -0.1577,  0.1321, -0.1157, -2.8928]])
+        >>> embed(x)
+        tensor([[-0.0555,  0.2292, -0.1833,  0.0301, -0.2416],
+                [-0.0725,  0.7042, -0.5644,  0.2235,  0.3603],
+                [-0.9021,  0.8899, -0.9802,  0.3565,  0.2367]])
+
+    """
+
+    __constants__ = ["in_features", "out_features"]
+    in_features: int
+    out_features: int
+    weight: torch.Tensor
+    bias: torch.Tensor
+
+    def __init__(
+        self, in_features, out_features, requires_grad=False, device=None, dtype=None
+    ):
+        factory_kwargs = {"device": device, "dtype": dtype}
+        super(Sinusoid, self).__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+
+        self.weight = nn.parameter.Parameter(
+            torch.empty((out_features, in_features), **factory_kwargs),
+            requires_grad=requires_grad,
+        )
+
+        self.bias = nn.parameter.Parameter(
+            torch.empty((1, out_features), **factory_kwargs),
+            requires_grad=requires_grad,
+        )
+        self.reset_parameters()
+
+    def reset_parameters(self) -> None:
+        nn.init.normal_(self.weight, 0, 1)
+        nn.init.uniform_(self.bias, 0, 2*math.pi)
+
+    def forward(self, input: torch.Tensor) -> torch.Tensor:
+        projected = F.linear(input, self.weight)
+        return torch.cos(projected + self.bias) * torch.sin(projected)

torchhd/tests/test_embeddings.py

@@ -0,0 +1,41 @@
+import pytest
+import torch
+
+from torchhd import functional
+from torchhd import embeddings
+
+from .utils import (
+    torch_dtypes,
+    torch_complex_dtypes,
+    supported_dtype,
+)
+
+# class TestIdentity:
+#     def test_num_embeddings(self):
+#         for i in range(1, 10):
+#             emb = embeddings.Identity(i, 3)
+#             idx = torch.LongTensor([0, 1, 4])
+#             res = emb(idx)
+
+#             assert res.size != i
+#         assert True
+
+#     def test_embedding_dim(self):
+#         assert True
+
+#     def test_value(self):
+#         assert True
+
+# class TestRandom:
+#     @pytest.mark.parametrize("dtype", torch_dtypes)
+#     def test_num_embeddings(self, dtype):
+#         assert True
+
+#     @pytest.mark.parametrize("dtype", torch_dtypes)
+#     def test_embedding_dim(self, dtype):
+#         assert True
+
+#     @pytest.mark.parametrize("dtype", torch_dtypes)
+#     def test_value(self, dtype):
+#         assert True
+