Added stack FP32 FWD oneDNN kernel

paddle/fluid/operators/mkldnn/stack_mkldnn_op.cc

@@ -0,0 +1,143 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/operators/utils.h"
+#include "paddle/fluid/platform/mkldnn_reuse.h"
+namespace paddle {
+namespace operators {
+
+using framework::DataLayout;
+using framework::Tensor;
+using framework::LoDTensor;
+using mkldnn::memory;
+using mkldnn::primitive;
+using mkldnn::concat;
+using mkldnn::stream;
+using platform::to_void_cast;
+
+template <typename T>
+class StackMKLDNNHandler
+    : public platform::MKLDNNHandlerNoCachingT<T, dnnl::concat> {
+ public:
+  StackMKLDNNHandler(const framework::ExecutionContext& ctx,
+                     const mkldnn::engine mkldnn_engine,
+                     const std::vector<const Tensor*>& inputs, Tensor* output)
+      : platform::MKLDNNHandlerNoCachingT<T, dnnl::concat>(mkldnn_engine,
+                                                           ctx.GetPlace()) {
+    int stack_axis = ctx.Attr<int>("axis");
+
+    int ndims = inputs[0]->dims().size();
+
+    if (stack_axis < 0) {
+      stack_axis = ndims + 1 + stack_axis;  // +1 to match output's ndims
+    }
+
+    // in stack op all inputs must have same dims
+    auto input_dims = framework::vectorize<int64_t>(inputs[0]->dims());
+
+    memory::data_type dt = framework::ToMKLDNNDataType(inputs[0]->type());
+    std::vector<memory::desc> srcs_md;
+    memory::desc dst_md;
+    MKLDNNMemoryFormat dst_fmt;
+
+    srcs_md.reserve(inputs.size());
+
+    // if stack is not done on last(non existing) axis, then we can optimize
+    // concat primitive by not adding additional dimension, since it causes
+    // wrong output format deduction and suboptimal performance as a result
+    if (stack_axis != ndims) {
+      for (size_t i = 0; i < inputs.size(); ++i) {
+        srcs_md.emplace_back(memory::desc(input_dims, dt, inputs[i]->format()));
+      }
+
+      input_dims[stack_axis] *= inputs.size();
+      dst_md = memory::desc(input_dims, dt, MKLDNNMemoryFormat::any);
+    } else {
+      auto extended_input_dims = framework::vectorize<int64_t>(output->dims());
+      extended_input_dims[stack_axis] = 1;
+
+      for (size_t i = 0; i < inputs.size(); ++i) {
+        srcs_md.emplace_back(memory::desc(input_dims, dt, inputs[i]->format())
+                                 .reshape(extended_input_dims));
+      }
+
+      // concat primitive choses suboptimal format tag because it cannot
+      // distinguish between f.e. abcd and abdc if last dim is equal to 1 so
+      // enforcing is needed for better performance
+      dst_fmt = platform::GetPlainMKLDNNFormat(extended_input_dims.size());
+      dst_md = memory::desc(framework::vectorize(output->dims()), dt, dst_fmt);
+    }
+
+    this->AcquireForwardPrimitiveDescriptor(dst_md, stack_axis, srcs_md);
+  }
+
+  // concat oneDNN prim is not having .desc attribute so we cannot use default
+  // AcquireForwardPrimitiveDescriptor
+  void AcquireForwardPrimitiveDescriptor(
+      const memory::desc& dst_md, const int stack_axis,
+      const std::vector<memory::desc>& srcs_md) {
+    this->fwd_pd_.reset(new dnnl::concat::primitive_desc(
+        dst_md, stack_axis, srcs_md, this->engine_));
+  }
+
+  std::shared_ptr<mkldnn::memory> AcquireSrcMemory(const Tensor& input, int i) {
+    const T* input_data = input.data<T>();
+    return this->AcquireMemoryFromPrimitive(this->fwd_pd_->src_desc(i),
+                                            to_void_cast<T>(input_data));
+  }
+};
+
+template <typename T>
+class StackMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
+ public:
+  void Compute(const paddle::framework::ExecutionContext& ctx) const override {
+    auto& dev_ctx =
+        ctx.template device_context<platform::MKLDNNDeviceContext>();
+    const auto& mkldnn_engine = dev_ctx.GetEngine();
+
+    auto multi_input = ctx.MultiInput<Tensor>("X");
+
+    Tensor* output = ctx.Output<Tensor>("Y");
+
+    StackMKLDNNHandler<T> handler(ctx, mkldnn_engine, multi_input, output);
+
+    std::vector<std::shared_ptr<memory>> srcs;
+    srcs.reserve(multi_input.size());
+
+    auto dst_mem = handler.AcquireDstMemory(output);
+    auto concat_p = handler.AcquireForwardPrimitive();
+
+    auto& astream = platform::MKLDNNDeviceContext::tls().get_stream();
+    std::unordered_map<int, memory> args;
+    for (size_t i = 0; i < multi_input.size(); ++i) {
+      srcs.push_back(handler.AcquireSrcMemory(*(multi_input[i]), i));
+      args.insert({MKLDNN_ARG_MULTIPLE_SRC + i, *(srcs.at(i))});
+    }
+    args.insert({MKLDNN_ARG_DST, *dst_mem});
+
+    concat_p->execute(astream, args);
+    astream.wait();
+
+    output->set_layout(DataLayout::kMKLDNN);
+    output->set_format(platform::GetMKLDNNFormat(
+        dst_mem->get_desc().reshape(framework::vectorize(output->dims()))));
+  }
+};
+}  // namespace operators
+}  // namespace paddle
+
+namespace ops = paddle::operators;
+
+REGISTER_OP_KERNEL(stack, MKLDNN, ::paddle::platform::CPUPlace,
+                   ops::StackMKLDNNOpKernel<float>);

paddle/fluid/operators/stack_op.cc

@@ -71,6 +71,21 @@ class StackOp : public framework::OperatorWithKernel {
     vec.insert(vec.begin() + axis, input_dims.size());
     ctx->SetOutputDim("Y", framework::make_ddim(vec));
   }
+
+  framework::OpKernelType GetExpectedKernelType(
+      const framework::ExecutionContext &ctx) const override {
+    auto input_data_type =
+        framework::OperatorWithKernel::IndicateVarDataType(ctx, "X");
+
+#ifdef PADDLE_WITH_MKLDNN
+    if (this->CanMKLDNNBeUsed(ctx, input_data_type)) {
+      return framework::OpKernelType(input_data_type, ctx.GetPlace(),
+                                     framework::DataLayout::kMKLDNN,
+                                     framework::LibraryType::kMKLDNN);
+    }
+#endif
+    return framework::OpKernelType(input_data_type, ctx.GetPlace());
+  }
 };
 
 class StackOpMaker : public framework::OpProtoAndCheckerMaker {
@@ -81,6 +96,11 @@ class StackOpMaker : public framework::OpProtoAndCheckerMaker {
     AddAttr<int>("axis",
                  "The axis along which all of the Inputs(X) should be stacked.")
         .SetDefault(0);
+    AddAttr<bool>(
+        "use_mkldnn",
+        "(bool, default false) Indicates if MKL-DNN kernel will be used")
+        .SetDefault(false)
+        .AsExtra();
     AddComment(R"DOC(
 Stack Operator.
 Stack all of the Inputs(X) into one tensor along Attr(axis). The dims of all Inputs(X) must be the same.

paddle/fluid/platform/mkldnn_helper.h

@@ -333,6 +333,43 @@ inline mkldnn::memory::format_tag GetMKLDNNFormat(const mkldnn::memory memory) {
   return GetMKLDNNFormat(mem_desc);
 }
 
+inline mkldnn::memory::format_tag GetPlainMKLDNNFormat(int tensor_rank) {
+  switch (tensor_rank) {
+    case 1:
+      return mkldnn::memory::format_tag::a;
+      break;
+    case 2:
+      return mkldnn::memory::format_tag::ab;
+      break;
+    case 3:
+      return mkldnn::memory::format_tag::abc;
+      break;
+    case 4:
+      return mkldnn::memory::format_tag::abcd;
+      break;
+    case 5:
+      return mkldnn::memory::format_tag::abcde;
+      break;
+    case 6:
+      return mkldnn::memory::format_tag::abcdef;
+      break;
+    case 7:
+      return mkldnn::memory::format_tag::abcdefg;
+      break;
+    case 8:
+      return mkldnn::memory::format_tag::abcdefgh;
+      break;
+    case 9:
+      return mkldnn::memory::format_tag::abcdefghi;
+      break;
+    default:
+      PADDLE_THROW(platform::errors::Unimplemented(
+          "Paddle support tensors with rank in range <1, 9>, but received "
+          "tensor with rank: %d",
+          tensor_rank));
+  }
+}
+
 inline MKLDNNMemoryFormat MKLDNNFormatForSize(size_t dims_size,
                                               MKLDNNMemoryFormat data_format) {
   if (dims_size == 1) {

python/paddle/fluid/tests/unittests/mkldnn/test_stack_mkldnn_op.py

@@ -0,0 +1,115 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import numpy as np
+from paddle.fluid.tests.unittests.op_test import OpTest, OpTestTool, skip_check_grad_ci
+import paddle
+import paddle.fluid as fluid
+import paddle.fluid.core as core
+
+
+@OpTestTool.skip_if_not_cpu()
+class TestStack2DOneDNNOp(OpTest):
+    def initDefaultParameters(self):
+        self.num_inputs = 4
+        self.input_dim = (2, 2)
+        self.axis = 1
+        self.dtype = np.float32
+
+    def initParameters(self):
+        pass
+
+    def getInputNames(self):
+        input_names = []
+        for i in range(self.num_inputs):
+            input_names.append('x{}'.format(i))
+        return input_names
+
+    def setUp(self):
+        self.initDefaultParameters()
+        self.initParameters()
+        self.op_type = 'stack'
+        self.op_inputs = []
+
+        for i in range(self.num_inputs):
+            self.op_inputs.append(
+                np.random.random(size=self.input_dim).astype(np.float32))
+
+        input_list = []
+        input_names = self.getInputNames()
+        for i in range(self.num_inputs):
+            input_list.append((input_names[i], self.op_inputs[i]))
+
+        self.inputs = {'X': input_list}
+        self.outputs = {'Y': np.stack(self.op_inputs, axis=self.axis)}
+        self.attrs = {'axis': self.axis, 'use_mkldnn': True}
+
+    def test_check_output(self):
+        self.check_output_with_place(core.CPUPlace())
+
+    # JUST FOR CI TO PASS, GRAD IS NOT IMPLEMENTED YET
+    def test_check_grad(self):
+        pass
+
+
+class TestStack1DOneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (100)
+        self.axis = 0
+
+
+class TestStack1DAxis1OneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (100)
+        self.axis = 1
+
+
+class TestStack2DAxisLastOneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (13, 24)
+        self.num_inputs = 5
+        self.axis = -1
+
+
+class TestStack3DAxisNegativeOneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (10, 128, 128)
+        self.axis = -2
+
+
+class TestStack3DOneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (10, 128, 128)
+        self.num_inputs = 3
+        self.axis = 1
+
+
+class TestStack4DOneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (2, 2, 2, 2)
+        self.num_inputs = 3
+        self.axis = 4
+
+
+class TestStack5DOneDNNOp(TestStack2DOneDNNOp):
+    def initParameters(self):
+        self.input_dim = (2, 3, 4, 5, 6)
+        self.num_inputs = 6
+        self.axis = 0
+
+
+if __name__ == "__main__":
+    paddle.enable_static()
+    unittest.main()

python/paddle/fluid/tests/unittests/op_test.py

@@ -1832,3 +1832,9 @@ def skip_if_not_cpu_bf16(cls):
             not (isinstance(_current_expected_place(), core.CPUPlace) and
                  core.supports_bfloat16()),
             "Place does not support BF16 evaluation")
+
+    @classmethod
+    def skip_if_not_cpu(cls):
+        return OpTestTool.skip_if(
+            not isinstance(_current_expected_place(), core.CPUPlace),
+            "OneDNN supports only CPU for now")