backward pass all tests

Author: Yang Yang

src/function.h

@@ -378,6 +378,13 @@ VariableHandle cross_entropy(VariableHandle x, VariableHandle label) {
   return out;
 }
 
+VariableHandle add(VariableHandle x, VariableHandle y) {
+  VariableHandle out(new Variable("add"));
+  get_global_tape().AddOp(
+      "elementwise_add", {{"X", {x}}, {"Y", {y}}}, {{"Out", {out}}}, {});
+  return out;
+}
+
 VariableHandle CreateRecordioFileReader(std::string filename,
                                         std::vector<int> shape_concat,
                                         std::vector<int> ranks,

src/tape.cc

@@ -147,35 +147,41 @@ void Tape::Forward() {
   VLOG(3) << "Finishing forward -------------------------";
 }
 
-void DescMapToVarMap(
+void Tape::DescMapToVarMap(
     const unordered_map<string, VariableHandle> &name2var,
-    const framework::VariableNameMap &vmp,
+    const framework::VariableNameMap &variable_name_map,
     VariableHandleMap *vhm,
     vector<pair<VariableHandle, VariableHandle>> *duplicated_grad,
     bool is_output) {
-  for (auto &p2a : vmp) {
-    for (auto &argu : p2a.second) {
-      if (name2var.count(argu)) {
-        (*vhm)[p2a.first].push_back(name2var.at(argu));
+  for (auto &p2a : variable_name_map) {
+    for (auto &arg : p2a.second) {
+      auto &param = p2a.first;
+      if (name2var.count(arg)) {
+        (*vhm)[param].push_back(name2var.at(arg));
       } else {
-        PADDLE_ENFORCE(ends_with(argu, framework::kGradVarSuffix),
+        PADDLE_ENFORCE(ends_with(arg, framework::kGradVarSuffix),
                        "%s not end with %s",
-                       argu,
+                       arg,
                        framework::kGradVarSuffix);
         string name =
-            argu.substr(0, argu.size() - strlen(framework::kGradVarSuffix));
+            arg.substr(0, arg.size() - strlen(framework::kGradVarSuffix));
         PADDLE_ENFORCE(name2var.count(name), "%s not found", name);
         if (is_output && name2var.at(name)->GradExist()) {
+          // Sum duplicated grad
           VariableHandle temp_grad(new Variable(
               name + framework::kGradVarSuffix + framework::kTempVarName));
-          (*vhm)[p2a.first].emplace_back(temp_grad);
-          duplicated_grad->emplace_back(name2var.at(name)->Grad(),
-                                        temp_grad);  // name2var[name]->Grad has
-                                                     // to be the first element
-                                                     // since sum_op use X[0] ==
-                                                     // Out to determine inplace
+          // name2var[name]->Grad has to be the first element since sum_op use
+          // X[0] == Out to determine inplace
+          duplicated_grad->emplace_back(name2var.at(name)->Grad(), temp_grad);
+          (*vhm)[param].emplace_back(temp_grad);
+        } else if (!is_output && !name2var.at(name)->GradExist()) {
+          // zero initialize empty grad
+          auto var = name2var.at(name);
+          backward_tape_->AddOp(
+              "fill_zeros_like", {{"X", {var}}}, {{"Out", {var->Grad()}}}, {});
+          (*vhm)[param].push_back(var->Grad());
         } else {
-          (*vhm)[p2a.first].push_back(name2var.at(name)->Grad());
+          (*vhm)[param].push_back(name2var.at(name)->Grad());
         }
       }
     }
@@ -232,8 +238,6 @@ void Tape::Backward(VariableHandle target) {
                               {});
       }
     }
-
-    // TODO(tonyyang-svail): how to fill empty grad?
   }
 
   backward_tape_->Forward();

src/tape.h

@@ -16,6 +16,7 @@
 #include <map>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "src/variable.h"
@@ -83,18 +84,18 @@ class Tape {
   bool HasBeenBackwarded() { return has_been_backwarded_; }
 
  private:
+  void Tape::DescMapToVarMap(
+      const std::unordered_map<std::string, VariableHandle> &name2var,
+      const framework::VariableNameMap &variable_name_map,
+      VariableHandleMap *vhm,
+      std::vector<std::pair<VariableHandle, VariableHandle>> *duplicated_grad,
+      bool is_output);
+
   bool has_been_backwarded_ = false;
   size_t current_position_ = 0;
 
   std::vector<OpHandle> tape_;
   std::shared_ptr<Tape> backward_tape_;
-
-  //  void DescMapToVarMap(const std::unordered_map<std::string, VariableHandle>
-  //  &name2var,
-  //                       const std::vector<std::pair<VariableHandle,
-  //                       VariableHandle>> &duplicated_grad,
-  //                       const std::pair<const framework::VariableNameMap
-  //                       *const, VariableHandleMap *> &each) const;
 };
 
 Tape &get_global_tape();

src/test_backward.cc

@@ -12,26 +12,85 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <vector>
+
 #include "gtest/gtest.h"
 #include "src/function.h"
 
+using paddle::tape::reset_global_tape;
+using paddle::tape::get_global_tape;
+
+using paddle::framework::LoDTensor;
+
+using paddle::tape::add;
+using paddle::tape::mean;
+using paddle::tape::Variable;
+using paddle::tape::VariableHandle;
+
+VariableHandle fill_with_vector(std::vector<float> data) {
+  VariableHandle var(new Variable("fill"));
+  auto* tensor = var->GetMutable<LoDTensor>();
+  tensor->Resize({static_cast<int64_t>(data.size())});
+  LOG(INFO) << tensor->dims();
+  auto* ptr = tensor->mutable_data<float>(paddle::platform::CPUPlace());
+  for (size_t i = 0; i < data.size(); ++i) {
+    ptr[i] = data[i];
+  }
+  return var;
+}
+
 /*
  * y = op(x)
  * z = op(x)
  * loss = y + z
  */
-TEST(Backward, TestMultipleAssignment) {}
+TEST(Backward, TestMultipleAssignment) {
+  reset_global_tape();
+
+  auto x = fill_with_vector({42});
+  auto y = mean(x);
+  auto z = mean(x);
+  auto loss = add(y, z);
+
+  get_global_tape().Backward(loss);
+
+  LOG(INFO) << x->Value();
+  LOG(INFO) << x->Grad()->Value();
+  PADDLE_ENFORCE_EQ(x->Grad()->Get<LoDTensor>().data<float>()[0], 2.0);
+}
 
 /*
  * loss = x + x
  */
-TEST(Backward, TestInplaceSum) {}
+TEST(Backward, TestInplaceSum) {
+  reset_global_tape();
+
+  auto x = fill_with_vector({42});
+  auto loss = add(x, x);
+
+  get_global_tape().Backward(loss);
+
+  PADDLE_ENFORCE_EQ(x->Grad()->Get<LoDTensor>().data<float>()[0], 2.0);
+}
 
 /*
  * y = op(x)  // y@grad is not initialized
  * loss = op(z)
  */
-TEST(Backward, TestEmptyGrad) {}
+TEST(Backward, TestEmptyGrad) {
+  reset_global_tape();
+  auto x = fill_with_vector({42});
+  auto y = mean(x);
+
+  auto z = fill_with_vector({42});
+  auto loss = mean(z);
+
+  get_global_tape().Backward(loss);
+
+  PADDLE_ENFORCE_EQ(x->Grad()->Get<LoDTensor>().data<float>()[0], 0.0);
+  PADDLE_ENFORCE_EQ(y->Grad()->Get<LoDTensor>().data<float>()[0], 0.0);
+  PADDLE_ENFORCE_EQ(z->Grad()->Get<LoDTensor>().data<float>()[0], 1.0);
+}
 
 /*
  * vector<> v
@@ -41,7 +100,7 @@ TEST(Backward, TestEmptyGrad) {}
  *   v.push_back(out)
  * loss = v.back()
  */
-TEST(Backward, TestForLoop) {}
+TEST(Backward, TestForLoop) { reset_global_tape(); }
 
 int main(int argc, char** argv) {
   std::vector<paddle::platform::Place> places;

src/variable.h

@@ -45,6 +45,8 @@ class Variable {
 
   ~Variable() { VLOG(10) << "Deleting " << Name(); }
 
+  bool GradExist() { return !grad_.expired(); }
+
   VariableHandle Grad() {
     if (grad_.expired()) {
       VariableHandle new_grad(new Variable(name_, true));
@@ -55,8 +57,6 @@ class Variable {
     }
   }
 
-  bool GradExist() { return !grad_.expired(); }
-
   // Evaluate a variable by running Forward() on the global tape
   const Variable& Value();
 
@@ -85,8 +85,8 @@ class Variable {
   }
 
  private:
-  int count() {
-    static int counter = 0;
+  int64_t count() {
+    static int64_t counter = 0;
     return counter++;
   }