bpo-44032: Move data stack to thread from FrameObject. (GH-26076)

* Remove 'zombie' frames. We won't need them once we are allocating fixed-size frames.

* Add co_nlocalplus field to code object to avoid recomputing size of locals + frees + cells.

* Move locals, cells and freevars out of frame object into separate memory buffer.

* Use per-threadstate allocated memory chunks for local variables.

* Move globals and builtins from frame object to per-thread stack.

* Move (slow) locals frame object to per-thread stack.

* Move internal frame functions to internal header.

Include/cpython/code.h

@@ -40,8 +40,8 @@ struct PyCodeObject {
     PyObject *co_name;          /* unicode (name, for reference) */
     PyObject *co_linetable;     /* string (encoding addr<->lineno mapping) See
                                    Objects/lnotab_notes.txt for details. */
+    int co_nlocalsplus;         /* Number of locals + free + cell variables */
     PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
-    void *co_zombieframe;       /* for optimization only (see frameobject.c) */
     PyObject *co_weakreflist;   /* to support weakrefs to code objects */
     /* Scratch space for extra data relating to the code object.
        Type is a void* to keep the format private in codeobject.c to force

Include/cpython/frameobject.h

@@ -20,12 +20,9 @@ enum _framestate {
 typedef signed char PyFrameState;
 
 struct _frame {
-    PyObject_VAR_HEAD
+    PyObject_HEAD
     struct _frame *f_back;      /* previous frame, or NULL */
     PyCodeObject *f_code;       /* code segment */
-    PyObject *f_builtins;       /* builtin symbol table (PyDictObject) */
-    PyObject *f_globals;        /* global symbol table (PyDictObject) */
-    PyObject *f_locals;         /* local symbol table (any mapping) */
     PyObject **f_valuestack;    /* points after the last local */
     PyObject *f_trace;          /* Trace function */
     /* Borrowed reference to a generator, or NULL */
@@ -36,7 +33,8 @@ struct _frame {
     PyFrameState f_state;       /* What state the frame is in */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
-    PyObject *f_localsplus[1];  /* locals+stack, dynamically sized */
+    char f_own_locals_memory;   /* This frame owns the memory for the locals */
+    PyObject **f_localsptr;     /* Pointer to locals, cells, free */
 };
 
 static inline int _PyFrame_IsRunnable(struct _frame *f) {
@@ -62,7 +60,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
 
 /* only internal use */
 PyFrameObject*
-_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *);
+_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *, PyObject **);
 
 
 /* The rest of the interface is specific for frame objects */

Include/cpython/pystate.h

@@ -57,6 +57,12 @@ typedef struct _err_stackitem {
 
 } _PyErr_StackItem;
 
+typedef struct _stack_chunk {
+    struct _stack_chunk *previous;
+    size_t size;
+    size_t top;
+    PyObject * data[1]; /* Variable sized */
+} _PyStackChunk;
 
 // The PyThreadState typedef is in Include/pystate.h.
 struct _ts {
@@ -149,6 +155,9 @@ struct _ts {
 
     CFrame root_cframe;
 
+    _PyStackChunk *datastack_chunk;
+    PyObject **datastack_top;
+    PyObject **datastack_limit;
     /* XXX signal handlers should also be here */
 
 };

Include/genobject.h

@@ -18,7 +18,7 @@ extern "C" {
     /* Note: gi_frame can be NULL if the generator is "finished" */         \
     PyFrameObject *prefix##_frame;                                          \
     /* The code object backing the generator */                             \
-    PyObject *prefix##_code;                                                \
+    PyCodeObject *prefix##_code;                                                \
     /* List of weak reference. */                                           \
     PyObject *prefix##_weakreflist;                                         \
     /* Name of the generator. */                                            \

Include/internal/pycore_frame.h

@@ -0,0 +1,38 @@
+#ifndef Py_INTERNAL_FRAME_H
+#define Py_INTERNAL_FRAME_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    FRAME_SPECIALS_GLOBALS_OFFSET = 0,
+    FRAME_SPECIALS_BUILTINS_OFFSET = 1,
+    FRAME_SPECIALS_LOCALS_OFFSET = 2,
+    FRAME_SPECIALS_SIZE = 3
+};
+
+static inline PyObject **
+_PyFrame_Specials(PyFrameObject *f) {
+    return &f->f_valuestack[-FRAME_SPECIALS_SIZE];
+}
+
+/* Returns a *borrowed* reference. */
+static inline PyObject *
+_PyFrame_GetGlobals(PyFrameObject *f)
+{
+    return _PyFrame_Specials(f)[FRAME_SPECIALS_GLOBALS_OFFSET];
+}
+
+/* Returns a *borrowed* reference. */
+static inline PyObject *
+_PyFrame_GetBuiltins(PyFrameObject *f)
+{
+    return _PyFrame_Specials(f)[FRAME_SPECIALS_BUILTINS_OFFSET];
+}
+
+int _PyFrame_TakeLocals(PyFrameObject *f);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_FRAME_H */

Include/internal/pycore_pymem.h

@@ -94,6 +94,11 @@ struct _PyTraceMalloc_Config {
 
 PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config;
 
+/* Allocate memory directly from the O/S virtual memory system,
+ * where supported. Otherwise fallback on malloc */
+void *_PyObject_VirtualAlloc(size_t size);
+void _PyObject_VirtualFree(void *, size_t size);
+
 
 #ifdef __cplusplus
 }

Include/internal/pycore_pystate.h

@@ -147,6 +147,9 @@ PyAPI_FUNC(int) _PyState_AddModule(
 
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
+PyObject **_PyThreadState_PushLocals(PyThreadState *, int size);
+void _PyThreadState_PopLocals(PyThreadState *, PyObject **);
+
 #ifdef __cplusplus
 }
 #endif