GH-150516: Reduce the work done to spill and reload the stack around calls (GH-151587)

Author: markshannon

Include/internal/pycore_interpframe.h

@@ -152,6 +152,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *
     // visited is GC bookkeeping for the current stack walk, not frame state.
     dest->visited = 0;
 #ifdef Py_DEBUG
+    dest->stackpointer_valid =  src->stackpointer_valid;
     dest->lltrace = src->lltrace;
 #endif
     for (int i = 0; i < stacktop; i++) {
@@ -207,6 +208,7 @@ _PyFrame_Initialize(
     frame->owner = FRAME_OWNED_BY_THREAD;
     frame->visited = 0;
 #ifdef Py_DEBUG
+    frame->stackpointer_valid = 1;
     frame->lltrace = 0;
 #endif
 
@@ -230,26 +232,51 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame)
 static inline _PyStackRef*
 _PyFrame_GetStackPointer(_PyInterpreterFrame *frame)
 {
+    return frame->stackpointer;
+}
+
+static inline void
+_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
+{
+    frame->stackpointer = stack_pointer;
+}
+
+static inline void
+_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame)
+{
+#ifdef Py_DEBUG
+/* Avoid bloating the JIT code */
 #ifndef _Py_JIT
-    assert(frame->stackpointer != NULL);
+    assert(frame->stackpointer_valid == 0);
 #endif
-    _PyStackRef *sp = frame->stackpointer;
-#ifndef NDEBUG
-    frame->stackpointer = NULL;
+    frame->stackpointer_valid = 1;
 #endif
-    return sp;
 }
 
 static inline void
-_PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
+_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame)
 {
+#ifdef Py_DEBUG
 /* Avoid bloating the JIT code */
 #ifndef _Py_JIT
-    assert(frame->stackpointer == NULL);
+    assert(frame->stackpointer_valid == 1);
+#endif
+    frame->stackpointer_valid = 0;
 #endif
-    frame->stackpointer = stack_pointer;
 }
 
+static inline void
+_PyFrame_StackAssertInvalid(_PyInterpreterFrame *frame)
+{
+#ifdef Py_DEBUG
+/* Avoid bloating the JIT code */
+#ifndef _Py_JIT
+    assert(frame->stackpointer_valid == 0);
+#endif
+#endif
+}
+
+
 /* Determine whether a frame is incomplete.
  * A frame is incomplete if it is part way through
  * creating cell objects or a generator or coroutine.
@@ -411,6 +438,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     frame->owner = FRAME_OWNED_BY_THREAD;
     frame->visited = 0;
 #ifdef Py_DEBUG
+    frame->stackpointer_valid = 1;
     frame->lltrace = 0;
 #endif
     frame->return_offset = 0;

Include/internal/pycore_interpframe_structs.h

@@ -44,7 +44,8 @@ struct _PyInterpreterFrame {
     char owner;
 #ifdef Py_DEBUG
     uint8_t visited:1;
-    uint8_t lltrace:7;
+    uint8_t stackpointer_valid:1;
+    uint8_t lltrace:6;
 #else
     uint8_t visited;
 #endif

Include/internal/pycore_opcode_metadata.h

@@ -312,7 +312,7 @@ int _Py_uop_analyze_and_optimize(
     _PyUOpInstruction *input, int trace_len, int curr_stackentries,
     _PyUOpInstruction *output, _PyBloomFilter *dependencies);
 
-extern PyTypeObject _PyUOpExecutor_Type;
+PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type;
 
 
 #define UOP_FORMAT_TARGET 0