add glu op

docs/source/functional.rst

@@ -22,6 +22,7 @@ Functional operations for neural networks
 .. autofunction:: prelu
 .. autofunction:: log_sigmoid 
 .. autofunction:: gelu
+.. autofunction:: glu
 .. autofunction:: softsign
 .. autofunction:: softmax 
 .. autofunction:: softplus

docs/source/nn.rst

@@ -33,6 +33,7 @@ Operators for neural networks
         Embedding,
         Flatten,
         GELU,
+        GLU,
         GroupNorm,
         Hardsigmoid,
         Hardswish,

oneflow/core/functional/functional_api.yaml

@@ -251,6 +251,10 @@
   signature: "Tensor (Tensor dy, Tensor x) => GeluGrad"
   bind_python: False
 
+- name: "glu"
+  signature: "Tensor (Tensor input, Int64 dim=-1) => Glu"
+  bind_python: True
+
 - name: "sigmoid"
   signature: "Tensor (Tensor x) => Sigmoid"
   bind_python: True

oneflow/core/functional/impl/activation_functor.cpp

@@ -26,6 +26,7 @@ limitations under the License.
 #include "oneflow/core/functional/function_library.h"
 #include "oneflow/core/functional/scalar.h"
 #include "oneflow/core/autograd/autograd_mode.h"
+#include "oneflow/core/functional/functional.h"
 
 namespace oneflow {
 namespace one {
@@ -164,6 +165,27 @@ class GeluGradFunctor : public BinaryFunctor {
   }
 };
 
+class GluFunctor {
+ public:
+  GluFunctor() {}
+  Maybe<Tensor> operator()(const std::shared_ptr<one::Tensor>& input, int64_t dim) const {
+    auto ndim = input->ndim();
+    CHECK_GT_OR_RETURN(ndim, 0) << "glu does not support 0-dimensional tensors";
+    CHECK_OR_RETURN(dim >= -ndim && dim < ndim)
+        << ", Dimension out of range (expected to be in range of [" << -ndim << ", " << ndim - 1
+        << "], but got " << dim << ")";
+    if (dim < 0) { dim += ndim; }
+    int64_t nc = input->dim(dim);
+    CHECK_EQ_OR_RETURN(nc % 2, 0) << "Halving dimension must be even, but dimension " << dim
+                                  << " is size " << nc;
+    nc = nc / 2;
+    std::vector<int64_t> split_sizes(2, nc);
+    auto split_x = JUST(SplitWithSize(input, split_sizes, dim));
+    auto sgmd_x1 = JUST(Sigmoid(split_x->at(1)));
+    return Mul(split_x->at(0), sgmd_x1);
+  }
+};
+
 class HardSigmoidFunctor {
  public:
   HardSigmoidFunctor() {
@@ -322,6 +344,7 @@ ONEFLOW_FUNCTION_LIBRARY(m) {
   m.add_functor<impl::EluGradFunctor>("EluGrad");
   m.add_functor<impl::GeluFunctor>("Gelu");
   m.add_functor<impl::GeluGradFunctor>("GeluGrad");
+  m.add_functor<impl::GluFunctor>("Glu");
   m.add_functor<impl::HardSigmoidFunctor>("HardSigmoid");
   m.add_functor<impl::HardSigmoidGradFunctor>("HardSigmoidGrad");
   m.add_functor<impl::SoftmaxFunctor>("Softmax");

python/oneflow/framework/docstr/activation.py

@@ -387,3 +387,33 @@
         tensor([1.0507, 2.1014, 3.1521], dtype=oneflow.float32)
     """,
 )
+add_docstr(
+    oneflow._C.glu,
+    """
+    glu(input: Tensor, dim: int) -> Tensor 
+
+    The equation is:
+
+    .. math::
+         GLU(input) = GLU(a, b) = a \otimes sigmoid(b)
+
+    .. note::
+        where input is split in half along dim to form a and b, ⊗ is the element-wise product between matrices.
+
+    For example:
+
+    .. code-block:: python
+
+        >>> import oneflow as flow
+        >>> import oneflow.nn as nn
+        >>> x = flow.tensor([[1, 2, 3, 4], [5, 6, 7, 8]], dtype=flow.float32)
+        >>> y = nn.functional.glu(x)
+        >>> y
+        tensor([[0.9526, 1.9640],
+                [4.9954, 5.9980]], dtype=oneflow.float32)
+
+    See    
+    :class:`~oneflow.nn.GLU` for more details.
+
+    """,
+)

python/oneflow/nn/__init__.py

@@ -18,6 +18,7 @@
 from oneflow.nn.modules.activation import (
     ELU,
     GELU,
+    GLU,
     Hardsigmoid,
     Hardswish,
     Hardtanh,

python/oneflow/nn/functional/__init__.py

@@ -39,6 +39,7 @@
 from oneflow._C import sigmoid
 from oneflow._C import prelu
 from oneflow._C import gelu
+from oneflow._C import glu
 from oneflow._C import log_sigmoid as logsigmoid
 from oneflow._C import log_sigmoid
 from oneflow._C import softsign

python/oneflow/nn/modules/activation.py

@@ -962,6 +962,50 @@ def forward(self, x):
         return flow._C.softsign(x)
 
 
+class GLU(Module):
+    r"""The GLU activation.
+
+    Args:
+        input (Tensor, float): input tensor. 
+        dim (int, optional): dimension on which to split the input. Default: -1
+
+    Shape:
+        - Input: :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional
+          dimensions
+        - Output: :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`
+
+    The formula is: 
+
+    .. math::  
+
+        GLU(input) = GLU(a, b) = a \otimes sigmoid(b)
+
+    .. note::
+        where input is split in half along dim to form a and b, ⊗ is the element-wise product between matrices.
+
+    For example:
+
+    .. code-block:: python
+
+        >>> import oneflow as flow
+        >>> import oneflow.nn as nn
+        >>> m = nn.GLU()
+        >>> x = flow.tensor([[1, 2, 3, 4], [5, 6, 7, 8]], dtype=flow.float32)
+        >>> y = m(x)
+        >>> y
+        tensor([[0.9526, 1.9640],
+                [4.9954, 5.9980]], dtype=oneflow.float32)
+
+    """
+
+    def __init__(self, dim: Optional[int] = -1):
+        super().__init__()
+        self.dim = dim
+
+    def forward(self, input):
+        return flow._C.glu(input, self.dim)
+
+
 if __name__ == "__main__":
     import doctest
 

python/oneflow/test/modules/test_glu.py

@@ -0,0 +1,47 @@
+"""
+Copyright 2020 The OneFlow Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import unittest
+from automated_test_util import *
+
+import oneflow as flow
+import oneflow.unittest
+
+
+@flow.unittest.skip_unless_1n1d()
+class TestGluModule(flow.unittest.TestCase):
+    @autotest(n=5)
+    def test_glu_module_with_random_data(test_case):
+        device = random_device()
+        dim = random(-3, 3).to(int)
+        m = torch.nn.functional.glu
+        x = random_pytorch_tensor(ndim=3, dim0=2, dim1=4, dim2=6).to(device)
+        y = m(x, dim)
+        return y
+
+    @autotest(n=5)
+    def test_GLU_module_with_random_data(test_case):
+        device = random_device()
+        m = torch.nn.GLU()
+        m.train(random())
+        m.to(device)
+        x = random_pytorch_tensor(ndim=3, dim0=2, dim1=4, dim2=6).to(device)
+        y = m(x)
+        return y
+
+
+if __name__ == "__main__":
+    unittest.main()