PaddlePaddle
diff --git a/‎paddle/fluid/framework/new_executor/instruction/phi_kernel_instruction.cc
Copy file name to clipboard
+3-2Lines changed: 3 additions & 2 deletions b/‎paddle/fluid/framework/new_executor/instruction/phi_kernel_instruction.cc
Copy file name to clipboard
+3-2Lines changed: 3 additions & 2 deletions
diff --git a/‎paddle/fluid/framework/new_executor/instruction/phi_kernel_instruction.h
Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions b/‎paddle/fluid/framework/new_executor/instruction/phi_kernel_instruction.h
Copy file name to clipboard
+2Lines changed: 2 additions & 0 deletions
diff --git a/‎paddle/fluid/pybind/pybind.cc
Copy file name to clipboard
+49-25Lines changed: 49 additions & 25 deletions b/‎paddle/fluid/pybind/pybind.cc
Copy file name to clipboard
+49-25Lines changed: 49 additions & 25 deletions
diff --git a/‎paddle/phi/api/generator/api_base.py
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion b/‎paddle/phi/api/generator/api_base.py
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion
diff --git a/‎paddle/phi/api/generator/dist_api_gen.py
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion b/‎paddle/phi/api/generator/dist_api_gen.py
Copy file name to clipboard
+1-1Lines changed: 1 addition & 1 deletion
diff --git a/‎paddle/phi/api/profiler/trace_event.h
Copy file name to clipboard
+54-42Lines changed: 54 additions & 42 deletions b/‎paddle/phi/api/profiler/trace_event.h
Copy file name to clipboard
+54-42Lines changed: 54 additions & 42 deletions
diff --git a/‎paddle/phi/core/platform/profiler/utils.cc
Copy file name to clipboard
+18-22Lines changed: 18 additions & 22 deletions b/‎paddle/phi/core/platform/profiler/utils.cc
Copy file name to clipboard
+18-22Lines changed: 18 additions & 22 deletions
diff --git a/‎python/paddle/base/dygraph/base.py
Copy file name to clipboard
+22-1Lines changed: 22 additions & 1 deletion b/‎python/paddle/base/dygraph/base.py
Copy file name to clipboard
+22-1Lines changed: 22 additions & 1 deletion
diff --git a/‎python/paddle/base/framework.py
Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions b/‎python/paddle/base/framework.py
Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions
diff --git a/‎python/paddle/jit/dy2static/program_translator.py
Copy file name to clipboard
+3-3Lines changed: 3 additions & 3 deletions b/‎python/paddle/jit/dy2static/program_translator.py
Copy file name to clipboard
+3-3Lines changed: 3 additions & 3 deletions
@@ -135,6 +135,7 @@ PhiKernelInstruction::PhiKernelInstruction(
                         .data();
   auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError(
       kernel_name, kernel_key);
+  kernel_name_ = kernel_name;
   phi_kernel_ = new phi::Kernel(kernel_result.kernel);
   PADDLE_ENFORCE_EQ(
       phi_kernel_->IsValid(), true, "not found kernel for [%s]", kernel_name);
@@ -192,8 +193,8 @@ void PhiKernelInstruction::Run() {
   }
   VLOG(6) << "Begin run op " << phi_op_name_ << " kernel.";
   {
-    phi::RecordEvent record_event("PhiKernelInstruction::kernel launch",
-                                  phi::TracerEventType::UserDefined,
+    phi::RecordEvent record_event(kernel_name_ + " kernel launch",
+                                  phi::TracerEventType::StaticKernelLaunch,
                                   1);
     (*(phi_kernel_))(&(kernel_context_));
   }
 
@@ -64,6 +64,8 @@ class PhiKernelInstruction : public InstructionBase {
 
   std::string phi_op_name_;
 
+  std::string kernel_name_;
+
   ::pir::Operation* op_{nullptr};  // not owned
 
   const ValueExecutionInfo* value_exec_info_;  // not owned
 
@@ -2828,7 +2828,16 @@ All parameter, weight, gradient are variables in Paddle.
       .def_readwrite("peak_allocated",
                      &paddle::platform::MemPythonNode::peak_allocated)
       .def_readwrite("peak_reserved",
-                     &paddle::platform::MemPythonNode::peak_reserved);
+                     &paddle::platform::MemPythonNode::peak_reserved)
+      .def("__repr__", [](paddle::platform::MemPythonNode &event_node) {
+        std::stringstream ostr;
+        ostr << "MemPythonNode(timestamp_ns=" << event_node.timestamp_ns
+             << ", addr=" << event_node.addr << ", type='"
+             << paddle::platform::StringTracerMemEventType(event_node.type)
+             << "', process_id=" << event_node.process_id
+             << ", thread_id=" << event_node.thread_id << ")";
+        return ostr.str();
+      });
 
   py::class_<paddle::platform::DevicePythonNode>(m, "DevicePythonNode")
       .def(py::init<>())
@@ -2862,7 +2871,18 @@ All parameter, weight, gradient are variables in Paddle.
                      &paddle::platform::DevicePythonNode::occupancy)
       .def_readwrite("num_bytes",
                      &paddle::platform::DevicePythonNode::num_bytes)
-      .def_readwrite("value", &paddle::platform::DevicePythonNode::value);
+      .def_readwrite("value", &paddle::platform::DevicePythonNode::value)
+      .def("__repr__", [](paddle::platform::DevicePythonNode &event_node) {
+        std::stringstream ostr;
+        ostr << "DevicePythonNode(name='" << event_node.name << "', type='"
+             << paddle::platform::StringTracerEventType(event_node.type)
+             << "', start_ns=" << event_node.start_ns
+             << ", end_ns=" << event_node.end_ns
+             << ", device_id=" << event_node.device_id
+             << ", context_id=" << event_node.context_id
+             << ", stream_id=" << event_node.stream_id << ")";
+        return ostr.str();
+      });
 
   py::class_<paddle::platform::HostPythonNode>(m, "HostPythonNode")
       .def(py::init<>())
@@ -2889,7 +2909,17 @@ All parameter, weight, gradient are variables in Paddle.
       .def_readwrite("device_node",
                      &paddle::platform::HostPythonNode::device_node_ptrs)
       .def_readwrite("mem_node",
-                     &paddle::platform::HostPythonNode::mem_node_ptrs);
+                     &paddle::platform::HostPythonNode::mem_node_ptrs)
+      .def("__repr__", [](paddle::platform::HostPythonNode &event_node) {
+        std::stringstream ostr;
+        ostr << "HostPythonNode(name='" << event_node.name << "', type='"
+             << paddle::platform::StringTracerEventType(event_node.type)
+             << "', start_ns=" << event_node.start_ns
+             << ", end_ns=" << event_node.end_ns
+             << ", process_id=" << event_node.process_id
+             << ", thread_id=" << event_node.thread_id << ")";
+        return ostr.str();
+      });
 
   py::class_<paddle::platform::Profiler>(m, "_Profiler")
       .def("create",
@@ -2928,28 +2958,22 @@ All parameter, weight, gradient are variables in Paddle.
       }))
       .def("end", [](phi::RecordEvent *event) { event->End(); });
 
-  py::enum_<phi::TracerMemEventType>(m, "TracerMemEventType")
-      .value("Allocate", phi::TracerMemEventType::Allocate)
-      .value("Free", phi::TracerMemEventType::Free)
-      .value("ReservedAllocate", phi::TracerMemEventType::ReservedAllocate)
-      .value("ReservedFree", phi::TracerMemEventType::ReservedFree);
-
-  py::enum_<phi::TracerEventType>(m, "TracerEventType")
-      .value("Operator", phi::TracerEventType::Operator)
-      .value("Dataloader", phi::TracerEventType::Dataloader)
-      .value("ProfileStep", phi::TracerEventType::ProfileStep)
-      .value("CudaRuntime", phi::TracerEventType::CudaRuntime)
-      .value("Kernel", phi::TracerEventType::Kernel)
-      .value("Memcpy", phi::TracerEventType::Memcpy)
-      .value("Memset", phi::TracerEventType::Memset)
-      .value("UserDefined", phi::TracerEventType::UserDefined)
-      .value("OperatorInner", phi::TracerEventType::OperatorInner)
-      .value("Forward", phi::TracerEventType::Forward)
-      .value("Backward", phi::TracerEventType::Backward)
-      .value("Optimization", phi::TracerEventType::Optimization)
-      .value("Communication", phi::TracerEventType::Communication)
-      .value("PythonOp", phi::TracerEventType::PythonOp)
-      .value("PythonUserDefined", phi::TracerEventType::PythonUserDefined);
+  py::enum_<paddle::platform::TracerMemEventType>(m, "TracerMemEventType")
+#define BIND_ENUM_ITEM(name) .value(#name, phi::TracerMemEventType::name)
+      FOR_EACH_TRACER_MEM_EVENT_TYPES(BIND_ENUM_ITEM)
+#undef BIND_ENUM_ITEM
+          ;  // NOLINT
+
+  py::enum_<paddle::platform::TracerEventType>(m, "TracerEventType")
+#define BIND_ENUM_ITEM(name) .value(#name, phi::TracerEventType::name)
+      FOR_EACH_TRACER_EVENT_TYPES(BIND_ENUM_ITEM)
+#undef BIND_ENUM_ITEM
+          ;  // NOLINT
+
+  m.def("tracer_event_type_to_string",
+        &paddle::platform::StringTracerEventType);
+  m.def("tracer_mem_event_type_to_string",
+        &paddle::platform::StringTracerMemEventType);
   m.def("load_profiler_result", &paddle::platform::LoadProfilerResult);
   m.def("enable_memory_recorder", &paddle::platform::EnableMemoryRecorder);
   m.def("disable_memory_recorder", &paddle::platform::DisableMemoryRecorder);
 
@@ -1349,7 +1349,7 @@ def gen_kernel_code(self, kernel_name, code_indent, inplace_flag=False):
 {code_indent}  auto* kernel_fn = kernel.GetVariadicKernelFn<kernel_signature>();
 {code_indent}  phi::RecordEvent* kernel_record_event = nullptr;
 {code_indent}  if(phi::RecordEvent::IsEnabled()){{
-{code_indent}    kernel_record_event = new phi::RecordEvent(\"{self.api} compute\", phi::TracerEventType::OperatorInner, 1);
+{code_indent}    kernel_record_event = new phi::RecordEvent(\"{kernel_name} kernel launch\", phi::TracerEventType::DygraphKernelLaunch, 1);
 {code_indent}  }}
 {code_indent}    (*kernel_fn)({kernel_args}, {", ".join(outputs_args)});
 {code_indent}  if(kernel_record_event != nullptr){{
 
@@ -418,7 +418,7 @@
 KERNEL_CALL_TEMPLATE = """
       phi::RecordEvent* kernel_record_event = nullptr;
       if(phi::RecordEvent::IsEnabled()){{
-        kernel_record_event = new phi::RecordEvent(\"{} dist compute\", phi::TracerEventType::OperatorInner, 1);
+        kernel_record_event = new phi::RecordEvent(\"{} dist compute\", phi::TracerEventType::DygraphKernelLaunch, 1);
       }}
       using kernel_signature = {};
       auto* kernel_fn = kernel.GetVariadicKernelFn<kernel_signature>();
 
@@ -20,52 +20,64 @@ limitations under the License. */
 
 namespace phi {
 
+#define FOR_EACH_TRACER_EVENT_TYPES(_)                                      \
+  /* Used to mark operator record */                                        \
+  _(Operator)                                                               \
+  /* Used to mark dataloader record */                                      \
+  _(Dataloader)                                                             \
+  /* Used to mark profile step record */                                    \
+  _(ProfileStep)                                                            \
+  /* Used to mark cuda runtime record returned by cupti */                  \
+  _(CudaRuntime)                                                            \
+  /* Used to mark kernel computation record returned by cupti */            \
+  _(Kernel)                                                                 \
+  /* Used to mark memcpy record returned by cupti */                        \
+  _(Memcpy)                                                                 \
+  /* Used to mark memset record returned by cupti */                        \
+  _(Memset)                                                                 \
+  /* Used to mark record defined by user */                                 \
+  _(UserDefined)                                                            \
+  /* Used to mark operator detail, (such as infer shape, compute) */        \
+  _(OperatorInner)                                                          \
+  /* Used to mark model training or testing perspective, forward process */ \
+  _(Forward)                                                                \
+  /* Used to mark model training perspective, backward process */           \
+  _(Backward)                                                               \
+  /* Used to mark model training perspective, optimization process */       \
+  _(Optimization)                                                           \
+  /* Used to mark distributed training perspective */                       \
+  _(Communication)                                                          \
+  /* Used to mark python api */                                             \
+  _(PythonOp)                                                               \
+  /* Used to mark python level user-defined */                              \
+  _(PythonUserDefined)                                                      \
+  /* Used to mark kernel call in dynamic graph mode */                      \
+  _(DygraphKernelLaunch)                                                    \
+  /* Used to mark kernel call in static graph mode */                       \
+  _(StaticKernelLaunch)
+
 enum class TracerEventType {
-  // Used to mark operator record
-  Operator = 0,
-  // Used to mark dataloader record
-  Dataloader = 1,
-  // Used to mark profile step record
-  ProfileStep = 2,
-  // Used to mark cuda runtime record returned by cupti
-  CudaRuntime = 3,
-  // Used to mark kernel computation record returned by cupti
-  Kernel = 4,
-  // Used to mark memcpy record returned by cupti
-  Memcpy = 5,
-  // Used to mark memset record returned by cupti
-  Memset = 6,
-  // Used to mark record defined by user
-  UserDefined = 7,
-  // Used to mark operator detail, (such as infer shape, compute)
-  OperatorInner = 8,
-  // Used to mark model training or testing perspective, forward process
-  Forward = 9,
-  // Used to mark model training perspective, backward process
-  Backward = 10,
-  // Used to mark model training perspective, optimization process
-  Optimization = 11,
-  // Used to mark distributed training perspective
-  Communication = 12,
-  // Used to mark python api
-  PythonOp = 13,
-  // Used to mark python level user-defined
-  PythonUserDefined = 14,
-  // A flag to denote the number of current types
-  NumTypes
+#define DEFINE_ENUM_ITEM(name) name,
+  FOR_EACH_TRACER_EVENT_TYPES(DEFINE_ENUM_ITEM)
+#undef DEFINE_ENUM_ITEM
+      NumTypes
 };
 
+#define FOR_EACH_TRACER_MEM_EVENT_TYPES(_)                                    \
+  /* Used to mark memory allocation which is managed by paddle */             \
+  _(Allocate)                                                                 \
+  /* Used to mark memory free which is managed by paddle */                   \
+  _(Free)                                                                     \
+  /* Used to mark reserved memory allocation which is applied from device. */ \
+  _(ReservedAllocate)                                                         \
+  /* Used to mark reserved memory free which is released to device. */        \
+  _(ReservedFree)
+
 enum class TracerMemEventType {
-  // Used to mark memory allocation which is managed by paddle
-  Allocate = 0,
-  // Used to mark memory free which is managed by paddle
-  Free = 1,
-  // Used to mark reserved memory allocation which is applied from device.
-  ReservedAllocate = 2,
-  // Used to mark reserved memory free which is released to device.
-  ReservedFree = 3,
-  // A flag to denote the number of current types
-  NumTypes
+#define DEFINE_ENUM_ITEM(name) name,
+  FOR_EACH_TRACER_MEM_EVENT_TYPES(DEFINE_ENUM_ITEM)
+#undef DEFINE_ENUM_ITEM
+      NumTypes
 };
 
 struct KernelEventInfo {
 
@@ -148,31 +148,27 @@ float CalculateEstOccupancy(uint32_t DeviceId,
 #endif  // PADDLE_WITH_CUPTI
 
 const char* StringTracerMemEventType(phi::TracerMemEventType type) {
-  static const char* category_name_[] = {// NOLINT
-                                         "Allocate",
-                                         "Free",
-                                         "ReservedAllocate",
-                                         "ReservedFree"};
-  return category_name_[static_cast<int>(type)];
+  switch (type) {
+#define CASE_NAME(name)               \
+  case phi::TracerMemEventType::name: \
+    return #name;
+    FOR_EACH_TRACER_MEM_EVENT_TYPES(CASE_NAME)
+#undef CASE_NAME
+    default:
+      return "Unknown";
+  }
 }
 
 const char* StringTracerEventType(phi::TracerEventType type) {
-  static const char* category_name_[] = {"Operator",  // NOLINT
-                                         "Dataloader",
-                                         "ProfileStep",
-                                         "CudaRuntime",
-                                         "Kernel",
-                                         "Memcpy",
-                                         "Memset",
-                                         "UserDefined",
-                                         "OperatorInner",
-                                         "Forward",
-                                         "Backward",
-                                         "Optimization",
-                                         "Communication",
-                                         "PythonOp",
-                                         "PythonUserDefined"};
-  return category_name_[static_cast<int>(type)];
+  switch (type) {
+#define CASE_NAME(name)            \
+  case phi::TracerEventType::name: \
+    return #name;
+    FOR_EACH_TRACER_EVENT_TYPES(CASE_NAME)
+#undef CASE_NAME
+    default:
+      return "Unknown";
+  }
 }
 
 }  // namespace platform
 
@@ -64,6 +64,14 @@ def in_to_static_mode() -> bool:
     return global_var._in_to_static_mode_
 
 
+def in_sot_simulation_mode() -> bool:
+    """
+    Return a bool value that indicates whether running code under SOT simulation context.
+
+    """
+    return global_var._in_sot_simulation_mode_
+
+
 # TODO(Aurelius84): Need to remove this alias after clean usage in PaddleX
 in_declarative_mode = in_to_static_mode
 
@@ -99,7 +107,7 @@ def __impl__(*args: _InputT.args, **kwargs: _InputT.kwargs) -> _RetT:
 
 
 @signature_safe_contextmanager
-def _to_static_mode_guard_(
+def to_static_mode_guard(
     is_to_static: bool = True,
 ) -> Generator[None, None, None]:
     global global_var
@@ -111,6 +119,19 @@ def _to_static_mode_guard_(
         global_var._in_to_static_mode_ = original_val
 
 
+@signature_safe_contextmanager
+def sot_simulation_mode_guard(
+    is_sot_simulation: bool = True,
+) -> Generator[None, None, None]:
+    global global_var
+    original_val = global_var._in_sot_simulation_mode_
+    global_var._in_sot_simulation_mode_ = is_sot_simulation
+    try:
+        yield
+    finally:
+        global_var._in_sot_simulation_mode_ = original_val
+
+
 @signature_safe_contextmanager
 def param_guard(
     parameters: OrderedDict[str, Tensor]
 
@@ -209,6 +209,7 @@ def __init__(self):
         """
         global _dygraph_tracer_
         self._in_to_static_mode_ = False
+        self._in_sot_simulation_mode_ = False
         self._functional_dygraph_context_manager = None
         self._dygraph_tracer_ = _dygraph_tracer_
         self._use_pir_api_ = get_flags("FLAGS_enable_pir_api")[
@@ -218,6 +219,9 @@ def __init__(self):
     def __str__(self):
         strings = []
         strings.append("_in_to_static_mode_:" + str(self._in_to_static_mode_))
+        strings.append(
+            "_in_sot_simulation_mode_:" + str(self._in_sot_simulation_mode_)
+        )
         strings.append(
             "_functional_dygraph_context_manager:"
             + str(self._functional_dygraph_context_manager)
 
@@ -30,9 +30,9 @@
 from paddle.base import core, framework
 from paddle.base.data_feeder import check_type
 from paddle.base.dygraph.base import (
-    _to_static_mode_guard_,
     param_guard,
     switch_to_static_graph,
+    to_static_mode_guard,
 )
 from paddle.framework import in_dynamic_mode, use_pir_api
 from paddle.nn.layer import layers
@@ -1256,7 +1256,7 @@ def pir_from_func_spec(
         )
 
         with ir_static.program_guard(main_program, startup_program):
-            with _to_static_mode_guard_(
+            with to_static_mode_guard(
                 is_to_static=True
             ), static_op_arg_cast_guard(_convert_into_value):
                 # 1. Adds `paddle.static.data` layers for input if needed
@@ -1361,7 +1361,7 @@ def from_func_spec(
         ProgramTranslator.get_instance()._amp_records.clear()
 
         with framework.program_guard(main_program, startup_program):
-            with _to_static_mode_guard_(is_to_static=True):
+            with to_static_mode_guard(is_to_static=True):
                 # 1. Adds `paddle.static.data` layers for input if needed
                 static_inputs = func_spec.to_static_inputs_with_spec(
                     input_spec, main_program
Original file line number	Diff line number	Diff line change
`@@ -135,6 +135,7 @@ PhiKernelInstruction::PhiKernelInstruction(`
`135`	`135`	`.data();`
`136`	`136`	`auto kernel_result = phi::KernelFactory::Instance().SelectKernelOrThrowError(`
`137`	`137`	`kernel_name, kernel_key);`
	`138`	`+ kernel_name_ = kernel_name;`
`138`	`139`	`phi_kernel_ = new phi::Kernel(kernel_result.kernel);`
`139`	`140`	`PADDLE_ENFORCE_EQ(`
`140`	`141`	`phi_kernel_->IsValid(), true, "not found kernel for [%s]", kernel_name);`
`@@ -192,8 +193,8 @@ void PhiKernelInstruction::Run() {`
`192`	`193`	`}`
`193`	`194`	`VLOG(6) << "Begin run op " << phi_op_name_ << " kernel.";`
`194`	`195`	`{`
`195`		`- phi::RecordEvent record_event("PhiKernelInstruction::kernel launch",`
`196`		`- phi::TracerEventType::UserDefined,`
	`196`	`+ phi::RecordEvent record_event(kernel_name_ + " kernel launch",`
	`197`	`+ phi::TracerEventType::StaticKernelLaunch,`
`197`	`198`	`1);`
`198`	`199`	`(*(phi_kernel_))(&(kernel_context_));`
`199`	`200`	`}`