[SOT][3.11] clear shadow PyInterpreterFrame after eval to avoid memory leak (#57793)

SigureMo · web-flow · commit 29059a349810 · 2023-09-27T22:13:08.000+08:00
* clean interpreter frame

* move clear ahead of free
diff --git a/paddle/fluid/pybind/eval_frame.c b/paddle/fluid/pybind/eval_frame.c
@@ -51,6 +51,14 @@ static int Internal_PyFrame_OpAlreadyRan(_PyInterpreterFrame *frame,
                                          int opcode,
                                          int oparg);
 int Internal_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame);
+PyFrameObject *Internal_PyFrame_New_NoTrack(PyCodeObject *code);
+PyFrameObject *Internal_PyFrame_MakeAndSetFrameObject(
+    _PyInterpreterFrame *frame);
+static inline PyFrameObject *Internal_PyFrame_GetFrameObject(
+    _PyInterpreterFrame *frame);
+static void Internal_take_ownership(PyFrameObject *f,
+                                    _PyInterpreterFrame *frame);
+void Internal_PyFrame_Clear(_PyInterpreterFrame *frame);
 
 // clang-format off
 // Define a proxy PyObject to access _PyInterpreterFrame's properties.
@@ -245,6 +253,140 @@ int Internal_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
   return 0;
 }
 
+PyFrameObject *Internal_PyFrame_New_NoTrack(PyCodeObject *code) {
+  CALL_STAT_INC(frame_objects_created);
+  int slots = code->co_nlocalsplus + code->co_stacksize;
+  PyFrameObject *f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, slots);
+  if (f == NULL) {
+    return NULL;
+  }
+  f->f_back = NULL;
+  f->f_trace = NULL;
+  f->f_trace_lines = 1;
+  f->f_trace_opcodes = 0;
+  f->f_fast_as_locals = 0;
+  f->f_lineno = 0;
+  return f;
+}
+
+PyFrameObject *Internal_PyFrame_MakeAndSetFrameObject(
+    _PyInterpreterFrame *frame) {
+  assert(frame->frame_obj == NULL);
+  PyObject *error_type, *error_value, *error_traceback;
+  PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+  PyFrameObject *f = Internal_PyFrame_New_NoTrack(frame->f_code);
+  if (f == NULL) {
+    Py_XDECREF(error_type);
+    Py_XDECREF(error_value);
+    Py_XDECREF(error_traceback);
+    return NULL;
+  }
+  PyErr_Restore(error_type, error_value, error_traceback);
+  if (frame->frame_obj) {
+    // GH-97002: How did we get into this horrible situation? Most likely,
+    // allocating f triggered a GC collection, which ran some code that
+    // *also* created the same frame... while we were in the middle of
+    // creating it! See test_sneaky_frame_object in test_frame.py for a
+    // concrete example.
+    //
+    // Regardless, just throw f away and use that frame instead, since it's
+    // already been exposed to user code. It's actually a bit tricky to do
+    // this, since we aren't backed by a real _PyInterpreterFrame anymore.
+    // Just pretend that we have an owned, cleared frame so frame_dealloc
+    // doesn't make the situation worse:
+    f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data;
+    f->f_frame->owner = FRAME_CLEARED;
+    f->f_frame->frame_obj = f;
+    Py_DECREF(f);
+    return frame->frame_obj;
+  }
+  assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT);
+  assert(frame->owner != FRAME_CLEARED);
+  f->f_frame = frame;
+  frame->frame_obj = f;
+  return f;
+}
+
+static inline PyFrameObject *Internal_PyFrame_GetFrameObject(
+    _PyInterpreterFrame *frame) {
+  assert(!_PyFrame_IsIncomplete(frame));
+  PyFrameObject *res = frame->frame_obj;
+  if (res != NULL) {
+    return res;
+  }
+  return Internal_PyFrame_MakeAndSetFrameObject(frame);
+}
+
+static void Internal_take_ownership(PyFrameObject *f,
+                                    _PyInterpreterFrame *frame) {
+  assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT);
+  assert(frame->owner != FRAME_CLEARED);
+  Py_ssize_t size =
+      ((char *)&frame->localsplus[frame->stacktop]) - (char *)frame;
+  memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size);
+  frame = (_PyInterpreterFrame *)f->_f_frame_data;
+  f->f_frame = frame;
+  frame->owner = FRAME_OWNED_BY_FRAME_OBJECT;
+  if (_PyFrame_IsIncomplete(frame)) {
+    // This may be a newly-created generator or coroutine frame. Since it's
+    // dead anyways, just pretend that the first RESUME ran:
+    PyCodeObject *code = frame->f_code;
+    frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable;
+  }
+  assert(!_PyFrame_IsIncomplete(frame));
+  assert(f->f_back == NULL);
+  _PyInterpreterFrame *prev = frame->previous;
+  while (prev && _PyFrame_IsIncomplete(prev)) {
+    prev = prev->previous;
+  }
+  if (prev) {
+    /* Link PyFrameObjects.f_back and remove link through
+     * _PyInterpreterFrame.previous */
+    PyFrameObject *back = Internal_PyFrame_GetFrameObject(prev);
+    if (back == NULL) {
+      /* Memory error here. */
+      assert(PyErr_ExceptionMatches(PyExc_MemoryError));
+      /* Nothing we can do about it */
+      PyErr_Clear();
+    } else {
+      f->f_back = (PyFrameObject *)Py_NewRef(back);
+    }
+    frame->previous = NULL;
+  }
+  if (!PyObject_GC_IsTracked((PyObject *)f)) {
+    PyObject_GC_Track((PyObject *)f);
+  }
+}
+
+void Internal_PyFrame_Clear(_PyInterpreterFrame *frame) {
+  /* It is the responsibility of the owning generator/coroutine
+   * to have cleared the enclosing generator, if any. */
+  assert(frame->owner != FRAME_OWNED_BY_GENERATOR ||
+         _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED);
+  // GH-99729: Clearing this frame can expose the stack (via finalizers). It's
+  // crucial that this frame has been unlinked, and is no longer visible:
+  assert(_PyThreadState_GET()->cframe->current_frame != frame);
+  if (frame->frame_obj) {
+    PyFrameObject *f = frame->frame_obj;
+    frame->frame_obj = NULL;
+    if (Py_REFCNT(f) > 1) {
+      Internal_take_ownership(f, frame);
+      Py_DECREF(f);
+      return;
+    }
+    Py_DECREF(f);
+  }
+  assert(frame->stacktop >= 0);
+  for (int i = 0; i < frame->stacktop; i++) {
+    Py_XDECREF(frame->localsplus[i]);
+  }
+  Py_XDECREF(frame->frame_obj);
+  Py_XDECREF(frame->f_locals);
+  Py_DECREF(frame->f_func);
+  Py_DECREF(frame->f_code);
+}
+
 #else
 typedef PyFrameObject FrameObject;
 #endif
@@ -341,7 +483,9 @@ inline static PyObject *eval_custom_code_py311_plus(PyThreadState *tstate,
   }
 
   PyObject *result = eval_frame_default(tstate, shadow, throw_flag);
+  Internal_PyFrame_Clear(shadow);
   free(shadow);
+  Py_DECREF(func);
   Py_DECREF(namemap);
   return result;
 }
