Support 0-size tensor in parameter creating, forward and backward

paddle/fluid/eager/accumulation/accumulation_node.cc

@@ -65,8 +65,12 @@ static void CopyOrAddTensor(paddle::Tensor* tensor,
                 *reinterpret_cast<phi::DenseTensor*>(tensor->impl().get()),
                 *reinterpret_cast<phi::DenseTensor*>(t.impl().get()),
                 reinterpret_cast<phi::DenseTensor*>(tensor->impl().get()));
-          } else {
+          } else if (t.initialized() && tensor->initialized()) {
             paddle::imperative::TensorAdd<paddle::Tensor>(t, tensor);
+          } else {
+            PADDLE_THROW(common::errors::PreconditionNotMet(
+                "DenseTensor 't' and 'tensor' should be "
+                "both initialized when accumulating gradient."));
           }
         } else {
           // TODO(jiabin): Support Other TensorBase later
@@ -179,7 +183,7 @@ GradNodeAccumulation::operator()(
   if (!weak_grad_.expired() && !is_new_grad) {
     auto grad = weak_grad_.lock();
     if (grad_out.defined() &&
-        (grad_out.is_dist_tensor() || grad_out.initialized())) {
+        (grad_out.is_dist_tensor() || grad_out.has_allocation())) {
       CopyOrAddTensor(grad.get(), grad_out, is_fake_empty_);
     }
     // else { do nothing since there is no valid value in grad out tensor }

paddle/fluid/eager/auto_code_generator/generator/eager_gen.py

@@ -694,7 +694,7 @@ class {} : public egr::GradNodeBase {{
 """
 CREATE_PLAIN_OPTIONAL_TENSOR_TEMPLATE = """
   paddle::optional<paddle::Tensor> {name}_optional;
-  if({name}.initialized() ||
+  if(({name}.has_allocation()) ||
      ({name}.defined() && {name}.is_dist_tensor() &&
       phi::distributed::NeedComputationClipForPP({name}.impl()))) {name}_optional = paddle::make_optional<paddle::Tensor>({name});
 """
@@ -731,7 +731,7 @@ class {} : public egr::GradNodeBase {{
 
 CHECK_BACKWARD_INPLACE_TEMPLATE = """
   bool can_be_inplaced = false;
-  if ({}.initialized()) {{
+  if ({}.has_allocation()) {{
     VLOG(10) << {}.name() << "({}) use_count: " << {}.impl().use_count();
     if ({}.impl().use_count() == 1 || ({}.impl().use_count() == 2 && {}.impl().get() == {}.impl().get())) {{
       if ({}.is_dense_tensor() && !std::dynamic_pointer_cast<phi::DenseTensor>({}.impl())->meta().is_contiguous()) {{
@@ -2949,7 +2949,7 @@ def _gen_api_call_code_block(
             if IsPlainTensorType(rtype):
                 output_autograd_meta = f"""
   auto& {transformed_tensor_name} = returns[{pos}][0];
-  egr::AutogradMeta* {output_autograd_meta_name} = returns[{pos}][0].initialized() ? egr::EagerUtils::autograd_meta(&{transformed_tensor_name}) : nullptr;
+  egr::AutogradMeta* {output_autograd_meta_name} = returns[{pos}][0].has_allocation() ? egr::EagerUtils::autograd_meta(&{transformed_tensor_name}) : nullptr;
   if ({output_autograd_meta_name}) {output_autograd_meta_name}->SetStopGradient(false);
 """
 

paddle/fluid/eager/backward.cc

@@ -180,7 +180,8 @@ std::vector<paddle::Tensor> RunBackward(
 
     // copy grad tensor since we should totally run grad without affect forward
     // value
-    if (!grad_tensors.empty() && grad_tensors[i].initialized()) {
+    if (!grad_tensors.empty() &&
+        (grad_tensors[i].defined() && grad_tensors[i].has_allocation())) {
       PADDLE_ENFORCE(
           grad_tensors.size() == tensors.size(),
           common::errors::Fatal(
@@ -354,9 +355,10 @@ std::vector<paddle::Tensor> RunBackward(
         paddle::Tensor& grad_output_tensor = grad_output_tensors[i][j];
 
         if ((!grad_output_tensor.defined() ||
-             !grad_output_tensor.initialized())) {
+             !grad_output_tensor.has_allocation())) {
           VLOG(7) << "We get grad_output_tensor with slot: " << i
-                  << ", rank: " << j << " as uninitialized or undefined tensor";
+                  << ", rank: " << j
+                  << " as undefined tensor or without allocation.";
         }
 
         VLOG(7) << "Get Edge and grad_output_tensor with slot: " << i

paddle/fluid/eager/custom_operator/custom_operator_utils.cc

@@ -810,7 +810,7 @@ void run_custom_op_impl(const paddle::OpMetaInfo& op_info,
   std::vector<Tensor>* all_inputs = ctx.AllMutableInput();
   for (size_t i = 0; i < all_inputs->size(); ++i) {
     auto& tensor = all_inputs->at(i);
-    if (tensor.initialized() && tensor.is_dense_tensor() &&
+    if (tensor.has_allocation() && tensor.is_dense_tensor() &&
         !std::dynamic_pointer_cast<phi::DenseTensor>(tensor.impl())
              ->meta()
              .is_contiguous()) {

paddle/fluid/eager/grad_node_info.cc

@@ -37,11 +37,11 @@
 namespace egr {
 
 static void CheckTensor(const paddle::Tensor& pre, const paddle::Tensor& post) {
-  if (!pre.initialized() && post.initialized()) {
+  if (!pre.has_allocation() && post.has_allocation()) {
     PADDLE_THROW(common::errors::PermissionDenied(
         "The tensor in before and after hook are not consistent"));
   }
-  if (pre.initialized() && post.initialized()) {
+  if (pre.has_allocation() && post.has_allocation()) {
     VLOG(7) << phi::DataTypeToString(pre.dtype()) << " "
             << phi::DataTypeToString(post.dtype());
     PADDLE_ENFORCE_EQ(
@@ -105,7 +105,7 @@ void GradNodeBase::SetGradInMeta(const paddle::Tensor& fwd_out,
     meta.SetStopGradient(fwd_out_meta->StopGradient());
   }
 
-  if (!fwd_out.initialized()) {
+  if (!fwd_out.has_allocation()) {
     if (fwd_out.defined() && fwd_out.is_dist_tensor() &&
         phi::distributed::NeedComputationClipForPP(fwd_out.impl())) {
       VLOG(3) << "Tensor " << fwd_out.name() << " is DistTensor,"
@@ -195,7 +195,7 @@ void GradNodeBase::SetGradInMeta(const std::vector<paddle::Tensor>& fwd_out,
       meta.SetStopGradient(fwd_out_meta->StopGradient());
     }
 
-    if (!fwd_out_tensor.initialized()) {
+    if (!fwd_out_tensor.has_allocation()) {
       if (fwd_out_tensor.defined() && fwd_out_tensor.is_dist_tensor() &&
           phi::distributed::NeedComputationClipForPP(fwd_out_tensor.impl())) {
         VLOG(3) << "Tensor " << fwd_out_tensor.name() << " is DistTensor,"
@@ -295,7 +295,7 @@ void GradNodeBase::SetGradInMeta(const std::vector<paddle::Tensor*>& fwd_out,
       meta.SetStopGradient(fwd_out_meta->StopGradient());
     }
 
-    if (!fwd_out_tensor.initialized()) {
+    if (!fwd_out_tensor.has_allocation()) {
       if (fwd_out_tensor.defined() && fwd_out_tensor.is_dist_tensor() &&
           phi::distributed::NeedComputationClipForPP(fwd_out_tensor.impl())) {
         VLOG(3) << "Tensor " << fwd_out_tensor.name() << " is DistTensor,"
@@ -762,7 +762,7 @@ GradNodeBase::ApplyGradientHooks(
     std::vector<paddle::Tensor>& slot_out = outs[slot_id];
     slot_out.resize(tensors[slot_id].size());
     paddle::Tensor& out = slot_out[rank];
-    if (!out.defined() || !out.initialized()) {
+    if (!out.defined() || !out.has_allocation()) {
       out = (*hook)(tensors[slot_id][rank]);
     } else {
       // If more than one hook is registered, the input to the next hook func

paddle/fluid/eager/grad_tensor_holder.cc

@@ -58,7 +58,7 @@ void GradTensorHolder::CopyValueFromTensor(size_t slot_id,
           rank));
   if (!fill_one) {
     paddle::Tensor& buffer_tensor = buffer_[slot_id][rank];
-    if ((!buffer_tensor.defined() || !buffer_tensor.initialized())) {
+    if ((!buffer_tensor.defined() || !buffer_tensor.has_allocation())) {
       // Perform deep copy here
       buffer_tensor.copy_(t, t.place(), false);
       auto* meta = egr::EagerUtils::autograd_meta(&buffer_tensor);
@@ -112,7 +112,7 @@ void GradTensorHolder::add(size_t slot_id,
                            size_t rank,
                            const paddle::Tensor& t,
                            bool create_graph) {
-  if (!t.initialized()) {
+  if (!t.has_allocation()) {
     if (t.defined() && t.is_dist_tensor() &&
         phi::distributed::NeedComputationClipForPP(t.impl())) {
       // Pipeline parallel still needs to construct GradNode graph
@@ -152,7 +152,7 @@ void GradTensorHolder::add(size_t slot_id,
   // related code later.
   // This if statement is trying to test neither phi::Tensor nor
   // framework::Variable is initialized.
-  if ((!buffer_tensor.defined() || !buffer_tensor.initialized())) {
+  if ((!buffer_tensor.defined() || !buffer_tensor.has_allocation())) {
     // Simply copy tensor->impl
     VLOG(6) << "Move Tensor for buffer_ slot: " << slot_id
             << ", size: " << buffer_[slot_id].size();
@@ -161,13 +161,14 @@ void GradTensorHolder::add(size_t slot_id,
     VLOG(6) << "Add Tensor for buffer_ slot: " << slot_id
             << ", size: " << buffer_[slot_id].size();
     // Accumulation
-    PADDLE_ENFORCE_EQ(t.initialized(),
-                      true,
-                      common::errors::Fatal(
-                          "We can only accumulate initialized tensor, but we "
-                          "got tensor: %s is empty please check you network "
-                          "and make sure it creates grads.",
-                          t.name()));
+    PADDLE_ENFORCE_EQ(
+        t.has_allocation(),
+        true,
+        common::errors::Fatal(
+            "We can only accumulate tensor having allocation, but we "
+            "got tensor: %s without allocation, please check you network "
+            "and make sure it creates grads.",
+            t.name()));
 
     if (t.is_dense_tensor()) {
       if (buffer_tensor.is_dense_tensor()) {

paddle/fluid/eager/tensor_wrapper.h

@@ -43,12 +43,12 @@ class TensorWrapper {
                          bool no_need_buffer = false) {
     // set inplace_version_snapshot_ according to tensor's current inplace
     // version.
-    if (tensor.initialized() && tensor.is_dense_tensor()) {
+    if (tensor.has_allocation() && tensor.is_dense_tensor()) {
       phi::DenseTensor* dense_tensor =
           static_cast<phi::DenseTensor*>(tensor.impl().get());
       auto& inplace_version_counter = dense_tensor->InplaceVersionCounter();
       inplace_version_snapshot_ = inplace_version_counter.CurrentVersion();
-    } else if (tensor.initialized() && tensor.is_dist_tensor()) {
+    } else if (tensor.has_allocation() && tensor.is_dist_tensor()) {
       phi::DenseTensor* dense_tensor =
           static_cast<phi::distributed::DistTensor*>(tensor.impl().get())
               ->unsafe_mutable_value();
@@ -92,7 +92,7 @@ class TensorWrapper {
     } else {
 #ifndef PADDLE_NO_PYTHON
       if (egr::SavedTensorsHooks::GetInstance().IsEnable() &&
-          tensor.is_dense_tensor() && tensor.initialized()) {
+          tensor.has_allocation() && tensor.is_dense_tensor()) {
         phi::DenseTensor* dense_tensor =
             static_cast<phi::DenseTensor*>(tensor.impl().get());
         intermediate_tensor_.set_impl(std::make_shared<phi::DenseTensor>(
@@ -102,7 +102,7 @@ class TensorWrapper {
         unpack_hook_ = egr::SavedTensorsHooks::GetInstance().GetUnPackHook();
         packed_value_ = (*pack_hook)(tensor);
       } else if (egr::SavedTensorsHooks::GetInstance().IsEnable() &&
-                 tensor.is_dist_tensor() && tensor.initialized()) {
+                 tensor.initialized() && tensor.is_dist_tensor()) {
         intermediate_tensor_.set_impl(
             std::make_shared<phi::distributed::DistTensor>(
                 tensor.dims(),

paddle/fluid/eager/utils.cc

@@ -354,7 +354,7 @@ void EagerUtils::HandleViewBetweenInputAndOutput(
 void EagerUtils::HandleViewBetweenInputAndOutput(
     const paddle::Tensor& input_tensor, paddle::Tensor* view_output_tensor) {
   PADDLE_ENFORCE_EQ(
-      input_tensor.initialized(),
+      input_tensor.has_allocation(),
       true,
       common::errors::InvalidArgument("Tensor %s has not been initialized!",
                                       input_tensor.name()));
@@ -814,7 +814,7 @@ std::string EagerUtils::TensorStr(const paddle::Tensor& t) {
                                                    dist_t->dist_attr());
       }
     } else {
-      if (t.initialized()) {
+      if (t.has_allocation()) {
         tensor_info_str += paddle::string::Sprintf(TENSOR_INFO_TEMPLATE,
                                                    t.impl()->type_info().name(),
                                                    t.dtype(),
@@ -847,37 +847,37 @@ std::string EagerUtils::TensorStr(const paddle::Tensor& t) {
                                              GradNodeStr(t),
                                              ad_meta->StopGradient());
       auto* data_ptr = dynamic_cast<phi::DenseTensor*>(t.impl().get());
-      if (t.initialized() && data_ptr) {
+      if (t.has_allocation() && data_ptr) {
         return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                        tensor_name_str,
-                                       t.initialized(),
+                                       t.has_allocation(),
                                        t.impl(),
                                        tensor_info_str,
                                        *data_ptr,
                                        ad_info_str);
       } else {
         return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                        tensor_name_str,
-                                       t.initialized(),
+                                       t.has_allocation(),
                                        t.impl(),
                                        tensor_info_str,
                                        "None",
                                        ad_info_str);
       }
     } else {
       auto* data_ptr = dynamic_cast<phi::DenseTensor*>(t.impl().get());
-      if (t.initialized() && data_ptr) {
+      if (t.has_allocation() && data_ptr) {
         return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                        tensor_name_str,
-                                       t.initialized(),
+                                       t.has_allocation(),
                                        t.impl(),
                                        tensor_info_str,
                                        *data_ptr,
                                        "None");
       } else {
         return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                        tensor_name_str,
-                                       t.initialized(),
+                                       t.has_allocation(),
                                        t.impl(),
                                        tensor_info_str,
                                        "None",
@@ -899,14 +899,14 @@ std::string EagerUtils::TensorStr(const paddle::Tensor& t) {
                                              ad_meta->StopGradient());
       return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                      tensor_name_str,
-                                     t.initialized(),
+                                     t.has_allocation(),
                                      t.impl(),
                                      tensor_info_str,
                                      ad_info_str);
     } else {
       return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                      tensor_name_str,
-                                     t.initialized(),
+                                     t.has_allocation(),
                                      t.impl(),
                                      tensor_info_str,
                                      "None");
@@ -917,14 +917,14 @@ std::string EagerUtils::TensorStr(const paddle::Tensor& t) {
         "TensorInfo: [ %s ]}";
     return paddle::string::Sprintf(TENSOR_PRINT_TEMPLATE,
                                    tensor_name_str,
-                                   t.initialized(),
+                                   t.has_allocation(),
                                    t.impl(),
                                    tensor_info_str);
   } else if (VLOG_IS_ON(4)) {
     const char* TENSOR_PRINT_TEMPLATE =
         "{ Name: %s, Initialized: %d, Ptr: %d }";
     return paddle::string::Sprintf(
-        TENSOR_PRINT_TEMPLATE, tensor_name_str, t.initialized(), t.impl());
+        TENSOR_PRINT_TEMPLATE, tensor_name_str, t.has_allocation(), t.impl());
   } else {
     return "[ Not specified tensor log level ]";
   }

paddle/fluid/framework/tensor_util.cc

@@ -1121,7 +1121,7 @@ TEST_API std::ostream& operator<<(std::ostream& os, const phi::DenseTensor& t) {
   os << "  - shape: [" << t.dims() << "]\n";
   os << "  - layout: " << common::DataLayoutToString(t.layout()) << "\n";
 
-  if (!t.initialized()) {
+  if (!t.has_allocation()) {
     os << "uninited\n";
     return os;
   }

paddle/fluid/imperative/gradient_accumulator.cc

@@ -134,7 +134,7 @@ const TType& GetInnerTensor(const framework::Variable& src) {
 template <typename TType>
 TType& GetInnerTensor(const paddle::Tensor& src) {
   PADDLE_ENFORCE_EQ(
-      src.initialized(),
+      (src.has_allocation()),
       true,
       common::errors::Fatal("We only add tensor with value if a tensor is "
                             "NOT INITIALIZED, it should just move instead of "

paddle/fluid/pybind/eager_functions.cc

@@ -305,7 +305,7 @@ PyObject* eager_api_get_grads_types(PyObject* self,
     }
 
     auto& grad = meta->Grad();
-    if (meta && grad.initialized()) {
+    if (meta && grad.has_allocation()) {
       if ((grad.is_dense_tensor() || grad.is_dist_tensor()) &&
           (tensor.dtype() == phi::DataType::FLOAT32 ||
            tensor.dtype() == phi::DataType::FLOAT16 ||
@@ -690,7 +690,7 @@ PyObject* eager_api_run_custom_op(PyObject* self,
         const auto& input_tensor = ctx.InputAt(input_range.first);
         // inplace optional [Tensor or vector<Tensor>], un-initialized tensor.
         if (paddle::framework::detail::IsOptionalVar(output) &&
-            !input_tensor.initialized()) {
+            !input_tensor.has_allocation()) {
           VLOG(7) << "Custom operator add output " << output
                   << " to CustomOpKernelContext. Add un-initialized tensor "
                      "because the inplace optional input is None";

paddle/fluid/pybind/eager_generator.cc

@@ -1600,22 +1600,23 @@ static std::pair<std::string, std::string> GenerateForwardFunctionContents(
                                     LegalizeVarName(input_name));
       } else {
         const char* FWD_INS_CONTENT_TEMPLATE =
-            "  if(%s.initialized()) "
+            "  if(%s.has_allocation()) "
             "ins[\"%s\"] = egr::EagerUtils::TrySyncToVars(%s);\n";
         dispensable_ins_contents_str +=
             paddle::string::Sprintf(FWD_INS_CONTENT_TEMPLATE,
                                     LegalizeVarName(input_name),
                                     input_name,
                                     LegalizeVarName(input_name));
         const char* FWD_AMP_TENSORS_VECTOR_TEMPLATE =
-            "    if(%s.initialized()) "
+            "    if(%s.has_allocation()) "
             "amp_tensors_vector.push_back({ %s });\n";
         dispensable_amp_tensors_vector_str +=
             paddle::string::Sprintf(FWD_AMP_TENSORS_VECTOR_TEMPLATE,
                                     LegalizeVarName(input_name),
                                     LegalizeVarName(input_name));
         const char* DISPENSABLE_AMP_AUTO_CAST_TEMPLATE =
-            "    auto NEW_%s = ((%s.initialized()) ? egr::AmpAutoCast(\"%s\", "
+            "    auto NEW_%s = ((%s.has_allocation()) ? "
+            "egr::AmpAutoCast(\"%s\", "
             "%s, amp_dst_dtype, \"%s\") : %s);\n";
         dispensable_amp_auto_cast_str +=
             paddle::string::Sprintf(DISPENSABLE_AMP_AUTO_CAST_TEMPLATE,
@@ -2189,7 +2190,7 @@ static std::string GenerateSingleOpBase(
       "  // Check backward inplace info\n"
       "  bool %s = false;\n"
       "  %s\n"
-      "  if (%s.initialized()) {\n"
+      "  if (%s.has_allocation()) {\n"
       "    VLOG(10) << %s.name() << \"(%s) use_count: \" << "
       "%s.impl().use_count();\n"
       "    if (%s.impl().use_count() == 1 || (%s.impl().use_count() == 2 && "