PaddlePaddle · tonyyang-svail · Jun 22, 2018 · Jun 22, 2018
diff --git a/paddle b/paddle
diff --git a/src/function.h b/src/function.h
@@ -40,14 +40,41 @@ class Fill {
       : initializer_(initializer), attrs_(attrs) {}
 
   void operator()(VariableHandle var) {
-    get_global_tape().AddOp(initializer_, {}, {{"Out", {var}}}, attrs_);
+    if (initializer_ == "fill_constant") {
+      // fill_constant is not OperatorWithKernel, so we can't add it to the tape
+      framework::OpDesc op_desc =
+          CreateOpDesc(initializer_, {}, {{"Out", {var}}}, attrs_);
+      ScopeWrapper scope({}, {{"Out", {var}}});
+      framework::OpRegistry::CreateOp(op_desc)->Run(scope,
+                                                    platform::CPUPlace());
+    } else {
+      get_global_tape().AddOp(initializer_, {}, {{"Out", {var}}}, attrs_);
+    }
   }
 
  private:
   const std::string initializer_;
   const framework::AttributeMap attrs_;
 };
 
+void init_params(VariableHandle v,
+                 const std::string &initializer,
+                 const framework::AttributeMap &attrs) {
+  if (initializer == "fill_constant") {
+    // fill_constant is not OperatorWithKernel, so we can't add it to the tape
+    framework::OpDesc op_desc =
+        CreateOpDesc(initializer, {}, {{"Out", {v}}}, attrs);
+    ScopeWrapper scope({}, {{"Out", {v}}});
+    framework::OpRegistry::CreateOp(op_desc)->Run(scope, platform::CPUPlace());
+  } else {
+    Tape init_tape;
+    init_tape.AddOp(initializer, {}, {{"Out", {v}}}, attrs);
+    init_tape.Forward();
+  }
+}
+
+// TODO(tonyyang-svail): change this to a function
+// https://github.com/PaddlePaddle/tape/issues/23
 class Mean {
  public:
   VariableHandle operator()(VariableHandle var) {
@@ -82,8 +109,6 @@ class Linear {
       : w_(new Variable("LinearWeight")),
         b_(new Variable("LinearBias")),
         act_(act) {
-    Tape init_tape;
-
     // Use Xavier to initialize Weight
     float limit = sqrt(6.0 / static_cast<float>(in_dim + out_dim));
     framework::AttributeMap attrs;
@@ -92,15 +117,13 @@ class Linear {
     attrs["min"] = -limit;
     attrs["max"] = limit;
     attrs["seed"] = RandomSeed::GetRandomSeed();
-    init_tape.AddOp("uniform_random", {}, {{"Out", {w_}}}, attrs);
+    init_params(w_, "uniform_random", attrs);
 
     // Use fill zero to initialize Bias
     attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
     attrs["shape"] = std::vector<int>{out_dim};
     attrs["value"] = 0.0f;
-    init_tape.AddOp("fill_constant", {}, {{"Out", {b_}}}, attrs);
-
-    init_tape.Forward();
+    init_params(b_, "fill_constant", attrs);
   }
 
   VariableHandle operator()(VariableHandle input) {
@@ -134,8 +157,6 @@ class Convolution2D {
       : w_(new Variable("ConvolutionWeight")),
         b_(new Variable("ConvolutionBias")),
         act_(act) {
-    Tape init_tape;
-
     // Use Xavier to initialize Weight
     float fan_in = c_in * f * f, fan_out = c_out * f * f;
     float limit = sqrt(6.0 / (fan_in + fan_out));
@@ -145,15 +166,13 @@ class Convolution2D {
     attrs["min"] = -limit;
     attrs["max"] = limit;
     attrs["seed"] = RandomSeed::GetRandomSeed();
-    init_tape.AddOp("uniform_random", {}, {{"Out", {w_}}}, attrs);
+    init_params(w_, "uniform_random", attrs);
 
     // Use fill zero to initialize Bias
     attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
     attrs["shape"] = std::vector<int>{c_out};
     attrs["value"] = 0.0f;
-    init_tape.AddOp("fill_constant", {}, {{"Out", {b_}}}, attrs);
-
-    init_tape.Forward();
+    init_params(b_, "fill_constant", attrs);
   }
 
   VariableHandle operator()(VariableHandle input) {
@@ -190,16 +209,12 @@ class Convolution2D {
 class SGD {
  public:
   explicit SGD(float learning_rate) : learning_rate_(new Variable("sgd")) {
-    Tape init_tape;
-
     std::string initializer = "fill_constant";
     framework::AttributeMap attrs;
     attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
     attrs["shape"] = std::vector<int>{1};
     attrs["value"] = learning_rate;
-    init_tape.AddOp(initializer, {}, {{"Out", {learning_rate_}}}, attrs);
-
-    init_tape.Forward();
+    init_params(learning_rate_, initializer, attrs);
   }
 
   void Update(VariableHandle input) {
@@ -224,7 +239,6 @@ VariableHandle CreateRecordioFileReader(std::string filename,
                                         std::vector<int> ranks,
                                         std::vector<int> lod_levels) {
   VariableHandle reader(new paddle::tape::Variable("reader"));
-  reader->MutableDesc()->SetType(paddle::framework::proto::VarType::READER);
 
   framework::OpDesc op_desc = CreateOpDesc("create_recordio_file_reader",
                                            {},
@@ -240,10 +254,7 @@ VariableHandle CreateRecordioFileReader(std::string filename,
 }
 
 void ReadNext(VariableHandle reader, VariableHandle data_holder) {
-  PADDLE_ENFORCE_EQ(reader->Desc().GetType(),
-                    paddle::framework::proto::VarType::READER);
-  PADDLE_ENFORCE_EQ(data_holder->Desc().GetType(),
-                    paddle::framework::proto::VarType::LOD_TENSOR_ARRAY);
+  PADDLE_ENFORCE(reader->Var().IsType<framework::ReaderHolder>());
 
   reader->GetMutable<paddle::framework::ReaderHolder>()->ReadNext(
       data_holder->GetMutable<paddle::framework::LoDTensorArray>());

diff --git a/src/tape.cc b/src/tape.cc
@@ -58,12 +58,12 @@ std::string to_string(const std::string &type,
   ss << type << " ";
   for (auto &param_name : in_vars) {
     for (auto &var : param_name.second) {
-      ss << param_name.first << ":(" << var->Desc() << ") ";
+      ss << param_name.first << ":(" << var << ") ";
     }
   }
   for (auto &param_name : out_vars) {
     for (auto &var : param_name.second) {
-      ss << param_name.first << ":(" << var->Desc() << ") ";
+      ss << param_name.first << ":(" << var << ") ";
     }
   }
   return ss.str();
@@ -85,41 +85,33 @@ framework::OpDesc CreateOpDesc(const std::string &type,
       outputs[param_name.first].emplace_back(var->Name());
     }
   }
-  return framework::OpDesc(type, inputs, outputs, attrs);
+  framework::OpDesc op_desc(type, inputs, outputs, attrs);
+  op_desc.CheckAttrs();
+  return op_desc;
 }
 
 void InferShapeAndVarType(const std::string &type,
                           const VariableHandleMap &in_vars,
                           VariableHandleMap *out_vars,
                           const framework::AttributeMap &attrs) {
-  framework::OpDesc op_desc = CreateOpDesc(type, in_vars, *out_vars, attrs);
-  op_desc.CheckAttrs();
-
-  // Create a temporary block for compile-time
-  framework::ProgramDesc program_desc;
-  framework::BlockDesc *block_desc = program_desc.MutableBlock(0);
-  PADDLE_ENFORCE(block_desc);
-
-  for (auto &param_name : in_vars) {
-    for (auto &var : param_name.second) {
-      *block_desc->Var(var->Name())->Proto() = *var->MutableDesc()->Proto();
-    }
-  }
-  for (auto &param_name : *out_vars) {
-    for (auto &var : param_name.second) {
-      *block_desc->Var(var->Name())->Proto() = *var->MutableDesc()->Proto();
+  // Tape only supports LoDTensor
+  for (auto &param2var : *out_vars) {
+    for (auto &var : param2var.second) {
+      var->GetMutable<framework::LoDTensor>();
     }
   }
 
-  LOG(INFO) << "- " << to_string(type, in_vars, *out_vars, attrs);
-  op_desc.InferShape(*block_desc);
-  op_desc.InferVarType(block_desc);
-  for (auto &param_name : *out_vars) {
-    for (auto &var : param_name.second) {
-      *var->MutableDesc()->Proto() = *block_desc->Var(var->Name())->Proto();
-    }
-  }
-  LOG(INFO) << "+ " << to_string(type, in_vars, *out_vars, attrs);
+  framework::OpDesc op_desc = CreateOpDesc(type, in_vars, *out_vars, attrs);
+  ScopeWrapper scope(in_vars, *out_vars);
+
+  // Tape only supports OperatorWithKernel
+  auto op = framework::OpRegistry::CreateOp(op_desc);
+  auto *op_with_kernel =
+      dynamic_cast<framework::OperatorWithKernel *>(op.get());
+  PADDLE_ENFORCE_NOT_NULL(op_with_kernel, "%s doesn't have kernel", type);
+  paddle::framework::RuntimeInferShapeContext infer_shape_ctx(*op_with_kernel,
+                                                              scope);
+  op_with_kernel->InferShape(&infer_shape_ctx);
 }
 
 void Tape::AddOp(const std::string &type,
@@ -135,14 +127,6 @@ void Tape::Forward() {
   PADDLE_ENFORCE(!has_been_backwarded_);
   while (current_position_ < tape_.size()) {
     OpHandle &op = tape_[current_position_];
-
-    // Create Output Tensor, this is only necessary for OpWithKernel
-    for (auto &param2var : op.outputs_) {
-      for (auto &var : param2var.second) {
-        var->InitializeVariable();
-      }
-    }
-
     framework::OpDesc op_desc =
         CreateOpDesc(op.type_, op.inputs_, op.outputs_, op.attrs_);
     ScopeWrapper scope(op.inputs_, op.outputs_);
@@ -161,14 +145,9 @@ void Tape::Backward(VariableHandle target) {
   // TODO(tonyyang-svail): check output of last op is target
   backward_tape_.reset(new Tape());
 
-  framework::AttributeMap attrs;
-
   // FIXME(tonyyang-svail): Need to infer_data_type
-  attrs["dtype"] = framework::proto::VarType::Type::VarType_Type_FP32;
-  attrs["shape"] = std::vector<int>{1};
-  attrs["value"] = 1.0f;
   backward_tape_->AddOp(
-      "fill_constant", {}, {{"Out", {target->Grad()}}}, attrs);
+      "fill_ones_like", {{"X", {target}}}, {{"Out", {target->Grad()}}}, {});
 
   for (auto it = tape_.rbegin(); it != tape_.rend(); ++it) {
     framework::OpDesc op_desc =

diff --git a/src/test_tape.cc b/src/test_tape.cc
@@ -31,46 +31,12 @@ using paddle::tape::ReadNext;
 
 TEST(Tape, TestReader) {
   VariableHandle data_label(new paddle::tape::Variable("data_label"));
-  data_label->MutableDesc()->SetType(
-      paddle::framework::proto::VarType::LOD_TENSOR_ARRAY);
-
   VariableHandle reader = CreateRecordioFileReader(
       "/tape/src/data/mnist.recordio", {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0});
-
   ReadNext(reader, data_label);
   LOG(INFO) << *data_label;
 }
 
-TEST(Tape, TestSoftmax) {
-  std::string data_initializer = "uniform_random";
-  paddle::framework::AttributeMap data_attrs;
-  data_attrs["min"] = -1.0f;
-  data_attrs["max"] = 1.0f;
-  data_attrs["dtype"] =
-      paddle::framework::proto::VarType::Type::VarType_Type_FP32;
-  data_attrs["shape"] = std::vector<int>{10, 10};
-  data_attrs["seed"] = 123;
-  Fill data_filler(data_initializer, data_attrs);
-
-  std::string label_initializer = "fill_constant";
-  paddle::framework::AttributeMap label_attrs;
-  label_attrs["dtype"] =
-      paddle::framework::proto::VarType::Type::VarType_Type_INT64;
-  label_attrs["shape"] = std::vector<int>{10, 1};
-  label_attrs["value"] = 1.0f;
-  Fill label_filler(label_initializer, label_attrs);
-
-  VariableHandle input(new Variable("input"));
-  data_filler(input);
-  VariableHandle label(new Variable("input"));
-  label_filler(label);
-
-  auto loss = cross_entropy(softmax(input), label);
-
-  LOG(INFO) << input->Value();
-  LOG(INFO) << loss->Value();
-}
-
 TEST(Tape, TestRelu) {
   std::string initializer = "uniform_random";
   paddle::framework::AttributeMap attrs;

diff --git a/src/variable.cc b/src/variable.cc
@@ -22,10 +22,9 @@ namespace tape {
 
 std::ostream& operator<<(std::ostream& os, const Variable& var) {
   LOG(INFO) << "Printing " << var.Name();
-  framework::proto::VarType::Type var_type = var.Desc().GetType();
-  if (var_type == framework::proto::VarType::LOD_TENSOR) {
+  if (var.Var().IsType<framework::LoDTensor>()) {
     os << var.Var().Get<framework::LoDTensor>();
-  } else if (var_type = framework::proto::VarType::LOD_TENSOR_ARRAY) {
+  } else if (var.Var().IsType<framework::LoDTensorArray>()) {
     framework::LoDTensorArray array =
         var.Var().Get<framework::LoDTensorArray>();
     for (size_t i = 0; i < array.size(); ++i) {
@@ -39,19 +38,6 @@ std::ostream& operator<<(std::ostream& os, const Variable& var) {
   return os;
 }
 
-void Variable::InitializeVariable() {
-  LOG(INFO) << "Initialzing " << desc_.Name() << " as " << desc_.GetType();
-  framework::proto::VarType::Type var_type = desc_.GetType();
-  if (var_type == framework::proto::VarType::LOD_TENSOR) {
-    var_.GetMutable<framework::LoDTensor>();
-  } else if (var_type == framework::proto::VarType::SELECTED_ROWS) {
-    var_.GetMutable<framework::SelectedRows>();
-  } else {
-    PADDLE_THROW("Variable type %d is not in [LOD_TENSOR, SELECTED_ROWS]",
-                 var_type);
-  }
-}
-
 const Variable& Variable::Value() {
   get_global_tape().Forward();
   return *this;

diff --git a/src/variable.h b/src/variable.h
@@ -36,20 +36,17 @@ std::ostream& operator<<(std::ostream&, const Variable&);
 class Variable {
  public:
   explicit Variable(const std::string pre_fix)
-      : desc_(pre_fix + std::to_string(count())) {}
+      : name_(pre_fix + std::to_string(count())) {}
 
   Variable(const std::string pre_fix, bool is_grad)
-      : desc_(pre_fix + (is_grad ? framework::kGradVarSuffix
+      : name_(pre_fix + (is_grad ? framework::kGradVarSuffix
                                  : std::to_string(count()))) {}
 
   ~Variable() { LOG(INFO) << "Deleting " << Name(); }
 
-  // Instantiate LoDTensor/SelectedRow
-  void InitializeVariable();
-
   VariableHandle Grad() {
     if (grad_.expired()) {
-      VariableHandle new_grad(new Variable(desc_.Name(), true));
+      VariableHandle new_grad(new Variable(name_, true));
       grad_ = new_grad;
       return new_grad;
     } else {
@@ -66,11 +63,8 @@ class Variable {
   // Evaluate a variable by running Forward() on the global tape
   const Variable& Value();
 
-  const framework::VarDesc& Desc() const { return desc_; }
-  framework::VarDesc* MutableDesc() { return &desc_; }
-
   // TODO(tonyyang-svail): No need to expose name
-  std::string Name() const { return desc_.Name(); }
+  std::string Name() const { return name_; }
 
   const framework::Variable& Var() const { return var_; }
   framework::Variable* MutableVar() { return &var_; }
@@ -91,7 +85,7 @@ class Variable {
     return counter++;
   }
 
-  framework::VarDesc desc_;
+  std::string name_;
   framework::Variable var_;
 
   // Not own
+7 −180		paddle/fluid/framework/operator.cc
+179 −0		paddle/fluid/framework/operator.h
+60 −0		paddle/fluid/operators/fill_ones_like_op.cc
+25 −0		paddle/fluid/operators/fill_ones_like_op.cu.cc
+36 −0		paddle/fluid/operators/fill_ones_like_op.h