[dygraph] refine dygraph backward error info (#70709)

wanghuancoder · web-flow · commit 8d120573b105 · 2025-01-09T18:17:50.000+08:00
* refine dygraph backward error info
diff --git a/paddle/fluid/eager/accumulation/accumulation_node.h b/paddle/fluid/eager/accumulation/accumulation_node.h
@@ -19,6 +19,8 @@
 #include "paddle/fluid/eager/hooks.h"
 #include "paddle/utils/test_macros.h"
 
+COMMON_DECLARE_int32(call_stack_level);
+
 namespace egr {
 
 class TEST_API GradNodeAccumulation : public GradNodeBase {
@@ -30,6 +32,10 @@ class TEST_API GradNodeAccumulation : public GradNodeBase {
       weak_grad_ = meta->WeakGrad();
     }
 
+    if (FLAGS_call_stack_level == 3) {
+      this->SetForwardTrace(egr::Controller::Instance().GetPythonStack());
+    }
+
     SetDefaultGradInOutMeta();
   }
 
diff --git a/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py b/paddle/fluid/eager/auto_code_generator/generator/eager_gen.py
@@ -485,7 +485,7 @@ class {} : public egr::GradNodeBase {{
     // Node Construction
 {}
     // Set for forward trace
-  if (FLAGS_check_nan_inf) {{
+  if (FLAGS_check_nan_inf || FLAGS_call_stack_level == 3) {{
     grad_node->SetForwardTrace(egr::Controller::Instance().GetPythonStack());
   }}
     // SetAttributes if needed
@@ -590,6 +590,7 @@ class {} : public egr::GradNodeBase {{
 #include "paddle/fluid/imperative/amp_utils.h"
 
 COMMON_DECLARE_bool(check_nan_inf);
+COMMON_DECLARE_int32(call_stack_level);
 COMMON_DECLARE_string(tensor_operants_mode);
 COMMON_DECLARE_bool(use_stride_kernel);
 {}
diff --git a/paddle/fluid/eager/backward.cc b/paddle/fluid/eager/backward.cc
diff --git a/paddle/fluid/framework/op_call_stack.cc b/paddle/fluid/framework/op_call_stack.cc
@@ -74,6 +74,40 @@ void InsertCallStackInfo(const std::string &type,
   exception->set_error_str(sout.str());
 }
 
+void InsertCallStackInfoDygraph(
+    const std::string &node_name,
+    const std::vector<std::string> &forward_callstack_str,
+    platform::EnforceNotMet *exception) {
+  const std::vector<std::string> *callstack = &forward_callstack_str;
+  std::ostringstream sout;
+  // Step 1. Construct python call stack string
+  if (callstack) {
+    if (FLAGS_call_stack_level > 1) {
+      sout << "\n\n  Forward Traceback (most recent call last):";
+    } else {
+      sout << "In user code:\n";
+    }
+    for (auto &line : *callstack) {
+      sout << "\n  " << line;
+    }
+  }
+  VLOG(1) << exception->error_str();
+  // Step 2. Construct final call stack & append error op name
+  if (FLAGS_call_stack_level > 1) {
+    sout << exception->what();
+  } else {
+    // If callstack exists, use err_str_ instead sub_err_str_
+    if (callstack) {
+      sout << "\n\n";
+      sout << InsertIndentationIntoEachLine(exception->error_str());
+    } else {
+      sout << exception->simple_error_str();
+    }
+  }
+  sout << "  [GradNode < " << node_name << " > error]";
+  exception->set_error_str(sout.str());
+}
+
 void InsertCallStackInfo(const std::string &type,
                          const std::vector<std::string> &callstack_attr_str,
                          platform::EnforceNotMet *exception) {
diff --git a/paddle/fluid/framework/op_call_stack.h b/paddle/fluid/framework/op_call_stack.h
@@ -31,6 +31,11 @@ void InsertCallStackInfo(const std::string &type,
                          const std::vector<std::string> &callstack_attr_str,
                          platform::EnforceNotMet *exception);
 
+void InsertCallStackInfoDygraph(
+    const std::string &type,
+    const std::vector<std::string> &callstack_attr_str,
+    platform::EnforceNotMet *exception);
+
 // only append error op for exception message
 void AppendErrorOpHint(const std::string &type,
                        platform::EnforceNotMet *exception);
diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc
@@ -827,6 +827,7 @@ Tensor is the basic data structure in PaddlePaddle. There are some ways to creat
  *  **/
 int TensorInit(PyObject* self, PyObject* args, PyObject* kwargs) {
   EAGER_TRY
+  SetPythonStack();
   // set a flag to record use kwargs or not
   bool flag_kwargs = false;
   if (kwargs) flag_kwargs = true;
diff --git a/paddle/fluid/pybind/eager_math_op_patch.cc b/paddle/fluid/pybind/eager_math_op_patch.cc
@@ -180,6 +180,7 @@ paddle::Tensor CallScalarFunction(const paddle::Tensor& self_tensor,
                                   double other,
                                   std::string op_type) {
   paddle::Tensor ret;
+  SetPythonStack();
   // scale_ad_func need sclar and bias with float type.
   if (op_type == "add" || op_type == "radd") {
     ret = scale_ad_func(self_tensor, phi::Scalar(1.0), other, true);
@@ -223,6 +224,7 @@ void TypePromotionForZeroDimTensor(std::string func,
         promote_type = self_tensor.dtype();
       }
     }
+    SetPythonStack();
     if (self_tensor.dtype() != promote_type) {
       eager_gil_scoped_release guard;
       self_tensor = cast_ad_func(self_tensor, promote_type);
@@ -243,6 +245,9 @@ static PyObject* tensor__add__method(TensorObject* self,
 
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__add__method";
+
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -338,6 +343,8 @@ static PyObject* tensor__sub__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__sub__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -432,6 +439,8 @@ static PyObject* tensor__rsub__method(TensorObject* self,
   EAGER_TRY
   VLOG(4) << "Running Eager tensor__rsub__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -525,6 +534,8 @@ static PyObject* tensor__mul__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__mul__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -628,6 +639,8 @@ static PyObject* tensor__div__method(TensorObject* self,
 
   VLOG(6) << "Running Eager tensor__div__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -720,6 +733,8 @@ static PyObject* tensor__rdiv__method(TensorObject* self,
 
   VLOG(6) << "Running Eager tensor__rdiv__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -806,6 +821,8 @@ static PyObject* tensor__gt__method(TensorObject* self,
   EAGER_TRY
   VLOG(4) << "Running Eager tensor__gt__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -897,6 +914,8 @@ static PyObject* tensor__ge__method(TensorObject* self,
   EAGER_TRY
   VLOG(4) << "Running Eager tensor__ge__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -988,6 +1007,8 @@ static PyObject* tensor__mod__method(TensorObject* self,
 
   VLOG(6) << "Running Eager tensor__mod__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1079,6 +1100,8 @@ static PyObject* tensor__rmod__method(TensorObject* self,
 
   VLOG(6) << "Running Eager tensor__rmod__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1170,6 +1193,8 @@ static PyObject* tensor__matmul__method(TensorObject* self,
 
   VLOG(6) << "Running Eager tensor__matmul__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1295,6 +1320,8 @@ static PyObject* tensor__rmatmul__method(TensorObject* self,
 
   VLOG(6) << "Running Eager tensor__rmatmul__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1420,6 +1447,8 @@ static PyObject* tensor__lt__method(TensorObject* self,
   EAGER_TRY
   VLOG(4) << "Running Eager tensor__lt__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1511,6 +1540,8 @@ static PyObject* tensor__le__method(TensorObject* self,
   EAGER_TRY
   VLOG(4) << "Running Eager tensor__le__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1601,6 +1632,8 @@ static PyObject* tensor__floordiv__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__floordiv__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1691,6 +1724,8 @@ static PyObject* tensor__rfloordiv__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__rfloordiv__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1782,6 +1817,8 @@ static PyObject* tensor__pow__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__pow__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1877,6 +1914,8 @@ static PyObject* tensor__rpow__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__rpow__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -1970,6 +2009,8 @@ static PyObject* tensor__ne__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__ne__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
@@ -2061,6 +2102,8 @@ static PyObject* tensor__eq__method(TensorObject* self,
   EAGER_TRY
   VLOG(6) << "Running Eager tensor__eq__method";
 
+  SetPythonStack();
+
   // Set Device ID
   auto place = egr::Controller::Instance().GetExpectedPlace();
   SetDevice(place);
diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc
@@ -1421,6 +1421,7 @@ static PyObject* tensor__getitem_dygraph(TensorObject* self,
                                          PyObject* args,
                                          PyObject* kwargs) {
   EAGER_TRY
+  SetPythonStack();
   PyObject* _index = PyTuple_GET_ITEM(args, 0);
   VLOG(4) << "Call new indexing strategy _getitem_dygraph";
 
@@ -1691,6 +1692,7 @@ static PyObject* tensor__setitem_dygraph(TensorObject* self,
                                          PyObject* args,
                                          PyObject* kwargs) {
   EAGER_TRY
+  SetPythonStack();
   VLOG(4) << "Call new indexing strategy _setitem_dygraph";
 
   PyObject* _index = PyTuple_GET_ITEM(args, 0);
@@ -2006,6 +2008,7 @@ static PyObject* tensor_register_grad_hook(TensorObject* self,
                                            PyObject* args,
                                            PyObject* kwargs) {
   EAGER_TRY
+  SetPythonStack();
   int64_t hook_id = 0;
   if (egr::EagerUtils::IsLeafTensor(self->tensor)) {
     VLOG(6) << "Register hook for leaf tensor: " << self->tensor.name();
diff --git a/paddle/fluid/pybind/eager_py_layer.cc b/paddle/fluid/pybind/eager_py_layer.cc
@@ -134,6 +134,7 @@ PyObject* pylayer_method_apply(PyObject* cls,
                                PyObject* args,
                                PyObject* kwargs) {
   EAGER_TRY
+  SetPythonStack();
   VLOG(6) << "Begin run PyLayer apply...";
   PyObject* backward_function =
       PyObject_GetAttrString(cls, "_backward_function");
diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc
@@ -51,6 +51,7 @@ limitations under the License. */
 
 COMMON_DECLARE_bool(check_nan_inf);
 COMMON_DECLARE_int32(check_nan_inf_level);
+COMMON_DECLARE_int32(call_stack_level);
 
 using egr::ConvertToDistTensor;
 
@@ -269,6 +270,19 @@ void SetPythonStack() {
     std::string last = str + egr::Controller::Instance().GetPythonStack();
     egr::Controller::Instance().SetPythonStack(last);
   }
+
+  if (FLAGS_call_stack_level == 3) {
+    VLOG(4) << "this is SetPythonStack";
+    pybind11::gil_scoped_acquire gil;
+    PyObject* mod = PyImport_ImportModule("traceback");
+    PyObject* traceback_list = PyObject_CallMethod(mod, "format_stack", "");
+    std::string str = "";
+    for (Py_ssize_t i = 0; i < PyList_Size(traceback_list); i++) {
+      PyObject* line = PyList_GetItem(traceback_list, i);
+      str += py::str(PyUnicode_AsUTF8(line));
+    }
+    egr::Controller::Instance().SetPythonStack(str);
+  }
 }
 
 std::shared_ptr<jit::Function> CastPyArg2JitFunction(PyObject* obj,
