gh-143939: Fix assignment to _PyThreadStateImpl.generator_return_kind (gh-143951)

The assignment to generator_return_kind has to be after any potentially
escaping calls to ensure that it's not overwritten.

Author: colesbury

Lib/test/test_coroutines.py

@@ -2265,6 +2265,20 @@ def c():
         # before fixing, visible stack from throw would be shorter than from send.
         self.assertEqual(len_send, len_throw)
 
+    def test_call_generator_in_frame_clear(self):
+        # gh-143939: Running a generator while clearing the coroutine's frame
+        # should not be misinterpreted as a yield.
+        class CallGeneratorOnDealloc:
+            def __del__(self):
+                next(x for x in [1])
+
+        async def coro():
+            obj = CallGeneratorOnDealloc()
+            return 42
+
+        yielded, result = run_async(coro())
+        self.assertEqual(yielded, [])
+        self.assertEqual(result, 42)
 
 @unittest.skipIf(
     support.is_emscripten or support.is_wasi,

Misc/NEWS.d/next/Core_and_Builtins/2026-01-16-23-19-38.gh-issue-143939.w9TWch.rst

@@ -0,0 +1,3 @@
+Fix erroneous "cannot reuse already awaited coroutine" error that could
+occur when a generator was run during the process of clearing a coroutine's
+frame.

Objects/genobject.c

@@ -280,6 +280,9 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, int exc)
 
     if (return_kind == GENERATOR_YIELD) {
         assert(result != NULL && !_PyErr_Occurred(tstate));
+#ifndef Py_GIL_DISABLED
+        assert(FRAME_STATE_SUSPENDED(gen->gi_frame_state));
+#endif
         *presult = result;
         return PYGEN_NEXT;
     }

Python/ceval.c

@@ -1914,14 +1914,16 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame)
     assert(frame->owner == FRAME_OWNED_BY_GENERATOR);
     PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame);
     FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_CLEARED);
-    ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_RETURN;
     assert(tstate->exc_info == &gen->gi_exc_state);
     tstate->exc_info = gen->gi_exc_state.previous_item;
     gen->gi_exc_state.previous_item = NULL;
     assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame);
     frame->previous = NULL;
     _PyFrame_ClearExceptCode(frame);
     _PyErr_ClearExcState(&gen->gi_exc_state);
+    // gh-143939: There must not be any escaping calls between setting
+    // the generator return kind and returning from _PyEval_EvalFrame.
+    ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_RETURN;
 }
 
 void