bpo-44032: Move data stack to thread from FrameObject. (GH-26076)

* Remove 'zombie' frames. We won't need them once we are allocating fixed-size frames.

* Add co_nlocalplus field to code object to avoid recomputing size of locals + frees + cells.

* Move locals, cells and freevars out of frame object into separate memory buffer.

* Use per-threadstate allocated memory chunks for local variables.

* Move globals and builtins from frame object to per-thread stack.

* Move (slow) locals frame object to per-thread stack.

* Move internal frame functions to internal header.

Include/cpython/code.h

@@ -40,8 +40,8 @@ struct PyCodeObject {
     PyObject *co_name;          /* unicode (name, for reference) */
     PyObject *co_linetable;     /* string (encoding addr<->lineno mapping) See
                                    Objects/lnotab_notes.txt for details. */
+    int co_nlocalsplus;         /* Number of locals + free + cell variables */
     PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
-    void *co_zombieframe;       /* for optimization only (see frameobject.c) */
     PyObject *co_weakreflist;   /* to support weakrefs to code objects */
     /* Scratch space for extra data relating to the code object.
        Type is a void* to keep the format private in codeobject.c to force

Include/cpython/frameobject.h

@@ -20,12 +20,9 @@ enum _framestate {
 typedef signed char PyFrameState;
 
 struct _frame {
-    PyObject_VAR_HEAD
+    PyObject_HEAD
     struct _frame *f_back;      /* previous frame, or NULL */
     PyCodeObject *f_code;       /* code segment */
-    PyObject *f_builtins;       /* builtin symbol table (PyDictObject) */
-    PyObject *f_globals;        /* global symbol table (PyDictObject) */
-    PyObject *f_locals;         /* local symbol table (any mapping) */
     PyObject **f_valuestack;    /* points after the last local */
     PyObject *f_trace;          /* Trace function */
     /* Borrowed reference to a generator, or NULL */
@@ -36,7 +33,8 @@ struct _frame {
     PyFrameState f_state;       /* What state the frame is in */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
-    PyObject *f_localsplus[1];  /* locals+stack, dynamically sized */
+    char f_own_locals_memory;   /* This frame owns the memory for the locals */
+    PyObject **f_localsptr;     /* Pointer to locals, cells, free */
 };
 
 static inline int _PyFrame_IsRunnable(struct _frame *f) {
@@ -62,7 +60,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
 
 /* only internal use */
 PyFrameObject*
-_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *);
+_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *, PyObject **);
 
 
 /* The rest of the interface is specific for frame objects */

Include/cpython/pystate.h

@@ -57,6 +57,12 @@ typedef struct _err_stackitem {
 
 } _PyErr_StackItem;
 
+typedef struct _stack_chunk {
+    struct _stack_chunk *previous;
+    size_t size;
+    size_t top;
+    PyObject * data[1]; /* Variable sized */
+} _PyStackChunk;
 
 // The PyThreadState typedef is in Include/pystate.h.
 struct _ts {
@@ -149,6 +155,9 @@ struct _ts {
 
     CFrame root_cframe;
 
+    _PyStackChunk *datastack_chunk;
+    PyObject **datastack_top;
+    PyObject **datastack_limit;
     /* XXX signal handlers should also be here */
 
 };

Include/genobject.h

@@ -18,7 +18,7 @@ extern "C" {
     /* Note: gi_frame can be NULL if the generator is "finished" */         \
     PyFrameObject *prefix##_frame;                                          \
     /* The code object backing the generator */                             \
-    PyObject *prefix##_code;                                                \
+    PyCodeObject *prefix##_code;                                                \
     /* List of weak reference. */                                           \
     PyObject *prefix##_weakreflist;                                         \
     /* Name of the generator. */                                            \

Include/internal/pycore_frame.h

@@ -0,0 +1,38 @@
+#ifndef Py_INTERNAL_FRAME_H
+#define Py_INTERNAL_FRAME_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    FRAME_SPECIALS_GLOBALS_OFFSET = 0,
+    FRAME_SPECIALS_BUILTINS_OFFSET = 1,
+    FRAME_SPECIALS_LOCALS_OFFSET = 2,
+    FRAME_SPECIALS_SIZE = 3
+};
+
+static inline PyObject **
+_PyFrame_Specials(PyFrameObject *f) {
+    return &f->f_valuestack[-FRAME_SPECIALS_SIZE];
+}
+
+/* Returns a *borrowed* reference. */
+static inline PyObject *
+_PyFrame_GetGlobals(PyFrameObject *f)
+{
+    return _PyFrame_Specials(f)[FRAME_SPECIALS_GLOBALS_OFFSET];
+}
+
+/* Returns a *borrowed* reference. */
+static inline PyObject *
+_PyFrame_GetBuiltins(PyFrameObject *f)
+{
+    return _PyFrame_Specials(f)[FRAME_SPECIALS_BUILTINS_OFFSET];
+}
+
+int _PyFrame_TakeLocals(PyFrameObject *f);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_FRAME_H */

Include/internal/pycore_pymem.h

@@ -94,6 +94,11 @@ struct _PyTraceMalloc_Config {
 
 PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config;
 
+/* Allocate memory directly from the O/S virtual memory system,
+ * where supported. Otherwise fallback on malloc */
+void *_PyObject_VirtualAlloc(size_t size);
+void _PyObject_VirtualFree(void *, size_t size);
+
 
 #ifdef __cplusplus
 }

Include/internal/pycore_pystate.h

@@ -147,6 +147,9 @@ PyAPI_FUNC(int) _PyState_AddModule(
 
 PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate);
 
+PyObject **_PyThreadState_PushLocals(PyThreadState *, int size);
+void _PyThreadState_PopLocals(PyThreadState *, PyObject **);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/test_gdb.py

@@ -666,15 +666,16 @@ def test_builtin_method(self):
 
     def test_frames(self):
         gdb_output = self.get_stack_trace('''
+import sys
 def foo(a, b, c):
-    pass
+    return sys._getframe(0)
 
-foo(3, 4, 5)
+f = foo(3, 4, 5)
-id(foo.__code__)''',
+id(f)''',
                                           breakpoint='builtin_id',
-                                          cmds_after_breakpoint=['print (PyFrameObject*)(((PyCodeObject*)v)->co_zombieframe)']
+                                          cmds_after_breakpoint=['print (PyFrameObject*)v']
                                           )
-        self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 3, in foo \(\)\s+.*',
+        self.assertTrue(re.match(r'.*\s+\$1 =\s+Frame 0x-?[0-9a-f]+, for file <string>, line 4, in foo \(a=3.*',
                                  gdb_output,
                                  re.DOTALL),
                         'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output))

Lib/test/test_sys.py

@@ -1274,11 +1274,7 @@ class C(object): pass
         # frame
         import inspect
         x = inspect.currentframe()
-        ncells = len(x.f_code.co_cellvars)
+        check(x, size('5P3i4cP'))
-        nfrees = len(x.f_code.co_freevars)
-        localsplus = x.f_code.co_stacksize + x.f_code.co_nlocals +\
-                  ncells + nfrees
-        check(x, vsize('8P3i3c' + localsplus*'P'))
         # function
         def func(): pass
         check(func, size('14P'))

Misc/NEWS.d/next/Core and Builtins/2021-05-14-20-03-32.bpo-44032.OzT1ob.rst

@@ -0,0 +1,2 @@
+Move 'fast' locals and other variables from the frame object to a per-thread
+datastack.

Objects/codeobject.c

@@ -280,6 +280,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
     co->co_posonlyargcount = posonlyargcount;
     co->co_kwonlyargcount = kwonlyargcount;
     co->co_nlocals = nlocals;
+    co->co_nlocalsplus = nlocals +
+        (int)PyTuple_GET_SIZE(freevars) + (int)PyTuple_GET_SIZE(cellvars);
     co->co_stacksize = stacksize;
     co->co_flags = flags;
     Py_INCREF(code);
@@ -304,7 +306,6 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
     co->co_linetable = linetable;
     Py_INCREF(exceptiontable);
     co->co_exceptiontable = exceptiontable;
-    co->co_zombieframe = NULL;
     co->co_weakreflist = NULL;
     co->co_extra = NULL;
 
@@ -968,8 +969,6 @@ code_dealloc(PyCodeObject *co)
     Py_XDECREF(co->co_exceptiontable);
     if (co->co_cell2arg != NULL)
         PyMem_Free(co->co_cell2arg);
-    if (co->co_zombieframe != NULL)
-        PyObject_GC_Del(co->co_zombieframe);
     if (co->co_weakreflist != NULL)
         PyObject_ClearWeakRefs((PyObject*)co);
     PyObject_Free(co);

Objects/frameobject.c

@@ -6,6 +6,7 @@
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
 
 #include "frameobject.h"          // PyFrameObject
+#include "pycore_frame.h"
 #include "opcode.h"               // EXTENDED_ARG
 #include "structmember.h"         // PyMemberDef
 
@@ -13,9 +14,6 @@
 
 static PyMemberDef frame_memberlist[] = {
     {"f_back",          T_OBJECT,       OFF(f_back),      READONLY},
-    {"f_code",          T_OBJECT,       OFF(f_code),      READONLY|PY_AUDIT_READ},
-    {"f_builtins",      T_OBJECT,       OFF(f_builtins),  READONLY},
-    {"f_globals",       T_OBJECT,       OFF(f_globals),   READONLY},
     {"f_trace_lines",   T_BOOL,         OFF(f_trace_lines), 0},
     {"f_trace_opcodes", T_BOOL,         OFF(f_trace_opcodes), 0},
     {NULL}      /* Sentinel */
@@ -34,8 +32,9 @@ frame_getlocals(PyFrameObject *f, void *closure)
 {
     if (PyFrame_FastToLocalsWithError(f) < 0)
         return NULL;
-    Py_INCREF(f->f_locals);
+    PyObject *locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
-    return f->f_locals;
+    Py_INCREF(locals);
+    return locals;
 }
 
 int
@@ -71,6 +70,36 @@ frame_getlasti(PyFrameObject *f, void *closure)
     return PyLong_FromLong(f->f_lasti*2);
 }
 
+static PyObject *
+frame_getglobals(PyFrameObject *f, void *closure)
+{
+    PyObject *globals = _PyFrame_GetGlobals(f);
+    if (globals == NULL) {
+        globals = Py_None;
+    }
+    Py_INCREF(globals);
+    return globals;
+}
+
+static PyObject *
+frame_getbuiltins(PyFrameObject *f, void *closure)
+{
+    PyObject *builtins = _PyFrame_GetBuiltins(f);
+    if (builtins == NULL) {
+        builtins = Py_None;
+    }
+    Py_INCREF(builtins);
+    return builtins;
+}
+
+static PyObject *
+frame_getcode(PyFrameObject *f, void *closure)
+{
+    if (PySys_Audit("object.__getattr__", "Os", f, "f_code") < 0) {
+        return NULL;
+    }
+    return (PyObject *)PyFrame_GetCode(f);
+}
 
 /* Given the index of the effective opcode,
    scan back to construct the oparg with EXTENDED_ARG */
@@ -554,50 +583,21 @@ static PyGetSetDef frame_getsetlist[] = {
                     (setter)frame_setlineno, NULL},
     {"f_trace",         (getter)frame_gettrace, (setter)frame_settrace, NULL},
     {"f_lasti",         (getter)frame_getlasti, NULL, NULL},
+    {"f_globals",       (getter)frame_getglobals, NULL, NULL},
+    {"f_builtins",      (getter)frame_getbuiltins, NULL, NULL},
+    {"f_code",          (getter)frame_getcode, NULL, NULL},
     {0}
 };
 
 /* Stack frames are allocated and deallocated at a considerable rate.
-   In an attempt to improve the speed of function calls, we:
+   In an attempt to improve the speed of function calls, we maintain
-
+   a separate free list of stack frames (just like floats are
-   1. Hold a single "zombie" frame on each code object. This retains
+   allocated in a special way -- see floatobject.c).  When a stack
-   the allocated and initialised frame object from an invocation of
+   frame is on the free list, only the following members have a meaning:
-   the code object. The zombie is reanimated the next time we need a
-   frame object for that code object. Doing this saves the malloc/
-   realloc required when using a free_list frame that isn't the
-   correct size. It also saves some field initialisation.
-
-   In zombie mode, no field of PyFrameObject holds a reference, but
-   the following fields are still valid:
-
-     * ob_type, ob_size, f_code, f_valuestack;
-
-     * f_locals, f_trace are NULL;
-
-     * f_localsplus does not require re-allocation and
-       the local variables in f_localsplus are NULL.
-
-   2. We also maintain a separate free list of stack frames (just like
-   floats are allocated in a special way -- see floatobject.c).  When
-   a stack frame is on the free list, only the following members have
-   a meaning:
     ob_type             == &Frametype
     f_back              next item on free list, or NULL
-    f_stacksize         size of value stack
-    ob_size             size of localsplus
-   Note that the value and block stacks are preserved -- this can save
-   another malloc() call or two (and two free() calls as well!).
-   Also note that, unlike for integers, each frame object is a
-   malloc'ed object in its own right -- it is only the actual calls to
-   malloc() that we are trying to save here, not the administration.
-   After all, while a typical program may make millions of calls, a
-   call depth of more than 20 or 30 is probably already exceptional
-   unless the program contains run-away recursion.  I hope.
-
-   Later, PyFrame_MAXFREELIST was added to bound the # of frames saved on
-   free_list.  Else programs creating lots of cyclic trash involving
-   frames could provoke free_list into growing without bound.
 */
+
 /* max value for numfree */
 #define PyFrame_MAXFREELIST 200
 
@@ -609,42 +609,37 @@ frame_dealloc(PyFrameObject *f)
     }
 
     Py_TRASHCAN_SAFE_BEGIN(f)
-    /* Kill all local variables */
+    PyCodeObject *co = f->f_code;
-    PyObject **valuestack = f->f_valuestack;
-    for (PyObject **p = f->f_localsplus; p < valuestack; p++) {
-        Py_CLEAR(*p);
-    }
 
-    /* Free stack */
+    /* Kill all local variables */
-    for (int i = 0; i < f->f_stackdepth; i++) {
+    if (f->f_localsptr) {
-        Py_XDECREF(f->f_valuestack[i]);
+        for (int i = 0; i < co->co_nlocalsplus+FRAME_SPECIALS_SIZE; i++) {
+            Py_CLEAR(f->f_localsptr[i]);
+        }
+        /* Free items on stack */
+        for (int i = 0; i < f->f_stackdepth; i++) {
+            Py_XDECREF(f->f_valuestack[i]);
+        }
+        if (f->f_own_locals_memory) {
+            PyMem_Free(f->f_localsptr);
+            f->f_own_locals_memory = 0;
+        }
     }
     f->f_stackdepth = 0;
-
     Py_XDECREF(f->f_back);
-    Py_DECREF(f->f_builtins);
-    Py_DECREF(f->f_globals);
-    Py_CLEAR(f->f_locals);
     Py_CLEAR(f->f_trace);
-
+    struct _Py_frame_state *state = get_frame_state();
-    PyCodeObject *co = f->f_code;
+#ifdef Py_DEBUG
-    if (co->co_zombieframe == NULL) {
+    // frame_dealloc() must not be called after _PyFrame_Fini()
-        co->co_zombieframe = f;
+    assert(state->numfree != -1);
+#endif
+    if (state->numfree < PyFrame_MAXFREELIST) {
+        ++state->numfree;
+        f->f_back = state->free_list;
+        state->free_list = f;
     }
     else {
-        struct _Py_frame_state *state = get_frame_state();
+        PyObject_GC_Del(f);
-#ifdef Py_DEBUG
-        // frame_dealloc() must not be called after _PyFrame_Fini()
-        assert(state->numfree != -1);
-#endif
-        if (state->numfree < PyFrame_MAXFREELIST) {
-            ++state->numfree;
-            f->f_back = state->free_list;
-            state->free_list = f;
-        }
-        else {
-            PyObject_GC_Del(f);
-        }
     }
 
     Py_DECREF(co);
@@ -654,24 +649,17 @@ frame_dealloc(PyFrameObject *f)
 static inline Py_ssize_t
 frame_nslots(PyFrameObject *frame)
 {
-    PyCodeObject *code = frame->f_code;
+    return frame->f_valuestack - frame->f_localsptr;
-    return (code->co_nlocals
-            + PyTuple_GET_SIZE(code->co_cellvars)
-            + PyTuple_GET_SIZE(code->co_freevars));
 }
 
 static int
 frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
 {
     Py_VISIT(f->f_back);
-    Py_VISIT(f->f_code);
-    Py_VISIT(f->f_builtins);
-    Py_VISIT(f->f_globals);
-    Py_VISIT(f->f_locals);
     Py_VISIT(f->f_trace);
 
     /* locals */
-    PyObject **fastlocals = f->f_localsplus;
+    PyObject **fastlocals = f->f_localsptr;
     for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++fastlocals) {
         Py_VISIT(*fastlocals);
     }
@@ -696,7 +684,7 @@ frame_tp_clear(PyFrameObject *f)
     Py_CLEAR(f->f_trace);
 
     /* locals */
-    PyObject **fastlocals = f->f_localsplus;
+    PyObject **fastlocals = f->f_localsptr;
     for (Py_ssize_t i = frame_nslots(f); --i >= 0; ++fastlocals) {
         Py_CLEAR(*fastlocals);
     }
@@ -731,15 +719,12 @@ PyDoc_STRVAR(clear__doc__,
 static PyObject *
 frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
 {
-    Py_ssize_t res, extras, ncells, nfrees;
+    Py_ssize_t res;
-
+    res = sizeof(PyFrameObject);
-    PyCodeObject *code = f->f_code;
+    if (f->f_own_locals_memory) {
-    ncells = PyTuple_GET_SIZE(code->co_cellvars);
+        PyCodeObject *code = f->f_code;
-    nfrees = PyTuple_GET_SIZE(code->co_freevars);
+        res += (code->co_nlocalsplus+code->co_stacksize) * sizeof(PyObject *);
-    extras = code->co_stacksize + code->co_nlocals + ncells + nfrees;
+    }
-    /* subtract one as it is already included in PyFrameObject */
-    res = sizeof(PyFrameObject) + (extras-1) * sizeof(PyObject *);
-
     return PyLong_FromSsize_t(res);
 }
 
@@ -802,24 +787,33 @@ PyTypeObject PyFrame_Type = {
 _Py_IDENTIFIER(__builtins__);
 
 static inline PyFrameObject*
-frame_alloc(PyCodeObject *code)
+frame_alloc(PyCodeObject *code, PyObject **localsarray)
 {
-    PyFrameObject *f = code->co_zombieframe;
+    int owns;
-    if (f != NULL) {
+    PyFrameObject *f;
-        code->co_zombieframe = NULL;
+    if (localsarray == NULL) {
-        _Py_NewReference((PyObject *)f);
+        int size = code->co_nlocalsplus+code->co_stacksize + FRAME_SPECIALS_SIZE;
-        assert(f->f_code == code);
+        localsarray = PyMem_Malloc(sizeof(PyObject *)*size);
-        return f;
+        if (localsarray == NULL) {
+            PyErr_NoMemory();
+            return NULL;
+        }
+        for (Py_ssize_t i=0; i < code->co_nlocalsplus; i++) {
+            localsarray[i] = NULL;
+        }
+        owns = 1;
+    }
+    else {
+        owns = 0;
     }
-
-    Py_ssize_t ncells = PyTuple_GET_SIZE(code->co_cellvars);
-    Py_ssize_t nfrees = PyTuple_GET_SIZE(code->co_freevars);
-    Py_ssize_t extras = code->co_stacksize + code->co_nlocals + ncells + nfrees;
     struct _Py_frame_state *state = get_frame_state();
     if (state->free_list == NULL)
     {
-        f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, extras);
+        f = PyObject_GC_New(PyFrameObject, &PyFrame_Type);
         if (f == NULL) {
+            if (owns) {
+                PyMem_Free(localsarray);
+            }
             return NULL;
         }
     }
@@ -832,46 +826,60 @@ frame_alloc(PyCodeObject *code)
         --state->numfree;
         f = state->free_list;
         state->free_list = state->free_list->f_back;
-        if (Py_SIZE(f) < extras) {
-            PyFrameObject *new_f = PyObject_GC_Resize(PyFrameObject, f, extras);
-            if (new_f == NULL) {
-                PyObject_GC_Del(f);
-                return NULL;
-            }
-            f = new_f;
-        }
         _Py_NewReference((PyObject *)f);
     }
-
+    f->f_localsptr = localsarray;
-    extras = code->co_nlocals + ncells + nfrees;
+    f->f_own_locals_memory = owns;
-    f->f_valuestack = f->f_localsplus + extras;
-    for (Py_ssize_t i=0; i < extras; i++) {
-        f->f_localsplus[i] = NULL;
-    }
     return f;
 }
 
+int
+_PyFrame_TakeLocals(PyFrameObject *f)
+{
+    assert(f->f_own_locals_memory == 0);
+    assert(f->f_stackdepth == 0);
+    int size = frame_nslots(f);
+    PyObject **copy = PyMem_Malloc(sizeof(PyObject *)*size);
+    if (copy == NULL) {
+        for (int i = 0; i < size; i++) {
+            PyObject *o = f->f_localsptr[i];
+            Py_XDECREF(o);
+        }
+        PyErr_NoMemory();
+        return -1;
+    }
+    for (int i = 0; i < size; i++) {
+        PyObject *o = f->f_localsptr[i];
+        copy[i] = o;
+    }
+    f->f_own_locals_memory = 1;
+    f->f_localsptr = copy;
+    f->f_valuestack = copy + size;
+    return 0;
+}
 
 PyFrameObject* _Py_HOT_FUNCTION
-_PyFrame_New_NoTrack(PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals)
+_PyFrame_New_NoTrack(PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals, PyObject **localsarray)
 {
     assert(con != NULL);
     assert(con->fc_globals != NULL);
     assert(con->fc_builtins != NULL);
     assert(con->fc_code != NULL);
     assert(locals == NULL || PyMapping_Check(locals));
+    PyCodeObject *code = (PyCodeObject *)con->fc_code;
 
-    PyFrameObject *f = frame_alloc((PyCodeObject *)con->fc_code);
+    PyFrameObject *f = frame_alloc(code, localsarray);
     if (f == NULL) {
         return NULL;
     }
 
+    PyObject **specials = f->f_localsptr + code->co_nlocalsplus;
+    f->f_valuestack = specials + FRAME_SPECIALS_SIZE;
     f->f_back = (PyFrameObject*)Py_XNewRef(tstate->frame);
     f->f_code = (PyCodeObject *)Py_NewRef(con->fc_code);
-    f->f_builtins = Py_NewRef(con->fc_builtins);
+    specials[FRAME_SPECIALS_BUILTINS_OFFSET] = Py_NewRef(con->fc_builtins);
-    f->f_globals = Py_NewRef(con->fc_globals);
+    specials[FRAME_SPECIALS_GLOBALS_OFFSET] = Py_NewRef(con->fc_globals);
-    f->f_locals = Py_XNewRef(locals);
+    specials[FRAME_SPECIALS_LOCALS_OFFSET] = Py_XNewRef(locals);
-    // f_valuestack initialized by frame_alloc()
     f->f_trace = NULL;
     f->f_stackdepth = 0;
     f->f_trace_lines = 1;
@@ -880,7 +888,6 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyFrameConstructor *con, PyObject *l
     f->f_lasti = -1;
     f->f_lineno = 0;
     f->f_state = FRAME_CREATED;
-    // f_blockstack and f_localsplus initialized by frame_alloc()
     return f;
 }
 
@@ -903,7 +910,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
         .fc_kwdefaults = NULL,
         .fc_closure = NULL
     };
-    PyFrameObject *f = _PyFrame_New_NoTrack(tstate, &desc, locals);
+    PyFrameObject *f = _PyFrame_New_NoTrack(tstate, &desc, locals, NULL);
     if (f) {
         _PyObject_GC_TRACK(f);
     }
@@ -1022,9 +1029,9 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
         PyErr_BadInternalCall();
         return -1;
     }
-    locals = f->f_locals;
+    locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
     if (locals == NULL) {
-        locals = f->f_locals = PyDict_New();
+        locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET] = PyDict_New();
         if (locals == NULL)
             return -1;
     }
@@ -1036,7 +1043,7 @@ PyFrame_FastToLocalsWithError(PyFrameObject *f)
                      Py_TYPE(map)->tp_name);
         return -1;
     }
-    fast = f->f_localsplus;
+    fast = f->f_localsptr;
     j = PyTuple_GET_SIZE(map);
     if (j > co->co_nlocals)
         j = co->co_nlocals;
@@ -1083,7 +1090,7 @@ PyFrame_FastToLocals(PyFrameObject *f)
 void
 PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 {
-    /* Merge f->f_locals into fast locals */
+    /* Merge locals into fast locals */
     PyObject *locals, *map;
     PyObject **fast;
     PyObject *error_type, *error_value, *error_traceback;
@@ -1092,7 +1099,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
     Py_ssize_t ncells, nfreevars;
     if (f == NULL)
         return;
-    locals = f->f_locals;
+    locals = _PyFrame_Specials(f)[FRAME_SPECIALS_LOCALS_OFFSET];
     co = f->f_code;
     map = co->co_varnames;
     if (locals == NULL)
@@ -1100,7 +1107,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
     if (!PyTuple_Check(map))
         return;
     PyErr_Fetch(&error_type, &error_value, &error_traceback);
-    fast = f->f_localsplus;
+    fast = f->f_localsptr;
     j = PyTuple_GET_SIZE(map);
     if (j > co->co_nlocals)
         j = co->co_nlocals;

Objects/genobject.c

@@ -176,7 +176,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     }
 
     assert(_PyFrame_IsRunnable(f));
-    assert(f->f_lasti >= 0 || ((unsigned char *)PyBytes_AS_STRING(f->f_code->co_code))[0] == GEN_START);
+    assert(f->f_lasti >= 0 || ((unsigned char *)PyBytes_AS_STRING(gen->gi_code->co_code))[0] == GEN_START);
     /* Push arg onto the frame's value stack */
     result = arg ? arg : Py_None;
     Py_INCREF(result);
@@ -331,7 +331,7 @@ _PyGen_yf(PyGenObject *gen)
     PyFrameObject *f = gen->gi_frame;
 
     if (f) {
-        PyObject *bytecode = f->f_code->co_code;
+        PyObject *bytecode = gen->gi_code->co_code;
         unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
 
         if (f->f_lasti < 0) {
@@ -826,8 +826,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
     }
     gen->gi_frame = f;
     f->f_gen = (PyObject *) gen;
-    Py_INCREF(f->f_code);
+    gen->gi_code = PyFrame_GetCode(f);
-    gen->gi_code = (PyObject *)(f->f_code);
     gen->gi_weakreflist = NULL;
     gen->gi_exc_state.exc_type = NULL;
     gen->gi_exc_state.exc_value = NULL;
@@ -836,7 +835,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
     if (name != NULL)
         gen->gi_name = name;
     else
-        gen->gi_name = ((PyCodeObject *)gen->gi_code)->co_name;
+        gen->gi_name = gen->gi_code->co_name;
     Py_INCREF(gen->gi_name);
     if (qualname != NULL)
         gen->gi_qualname = qualname;
@@ -1167,11 +1166,12 @@ compute_cr_origin(int origin_depth)
     }
     frame = PyEval_GetFrame();
     for (int i = 0; i < frame_count; ++i) {
-        PyCodeObject *code = frame->f_code;
+        PyCodeObject *code = PyFrame_GetCode(frame);
         PyObject *frameinfo = Py_BuildValue("OiO",
                                             code->co_filename,
                                             PyFrame_GetLineNumber(frame),
                                             code->co_name);
+        Py_DECREF(code);
         if (!frameinfo) {
             Py_DECREF(cr_origin);
             return NULL;

Objects/obmalloc.c

@@ -552,6 +552,18 @@ PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
     *allocator = _PyObject_Arena;
 }
 
+void *
+_PyObject_VirtualAlloc(size_t size)
+{
+    return _PyObject_Arena.alloc(_PyObject_Arena.ctx, size);
+}
+
+void
+_PyObject_VirtualFree(void *obj, size_t size)
+{
+    _PyObject_Arena.free(_PyObject_Arena.ctx, obj, size);
+}
+
 void
 PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
 {
@@ -3035,7 +3047,7 @@ _PyObject_DebugMallocStats(FILE *out)
 
     fputc('\n', out);
 
-    /* Account for what all of those arena bytes are being used for. */ 
+    /* Account for what all of those arena bytes are being used for. */
     total = printone(out, "# bytes in allocated blocks", allocated_bytes);
     total += printone(out, "# bytes in available blocks", available_bytes);
 

Objects/typeobject.c

@@ -8836,14 +8836,14 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co,
         return -1;
     }
 
-    PyObject *obj = f->f_localsplus[0];
+    PyObject *obj = f->f_localsptr[0];
     Py_ssize_t i, n;
     if (obj == NULL && co->co_cell2arg) {
         /* The first argument might be a cell. */
         n = PyTuple_GET_SIZE(co->co_cellvars);
         for (i = 0; i < n; i++) {
             if (co->co_cell2arg[i] == 0) {
-                PyObject *cell = f->f_localsplus[co->co_nlocals + i];
+                PyObject *cell = f->f_localsptr[co->co_nlocals + i];
                 assert(PyCell_Check(cell));
                 obj = PyCell_GET(cell);
                 break;
@@ -8871,7 +8871,7 @@ super_init_without_args(PyFrameObject *f, PyCodeObject *co,
         if (_PyUnicode_EqualToASCIIId(name, &PyId___class__)) {
             Py_ssize_t index = co->co_nlocals +
                 PyTuple_GET_SIZE(co->co_cellvars) + i;
-            PyObject *cell = f->f_localsplus[index];
+            PyObject *cell = f->f_localsptr[index];
             if (cell == NULL || !PyCell_Check(cell)) {
                 PyErr_SetString(PyExc_RuntimeError,
                   "super(): bad __class__ cell");

Python/_warnings.c

@@ -5,6 +5,7 @@
 #include "pycore_pyerrors.h"
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "frameobject.h"          // PyFrame_GetBack()
+#include "pycore_frame.h"
 #include "clinic/_warnings.c.h"
 
 #define MODULE_NAME "_warnings"
@@ -853,7 +854,7 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno,
         *lineno = 1;
     }
     else {
-        globals = f->f_globals;
+        globals = _PyFrame_GetGlobals(f);
         PyCodeObject *code = PyFrame_GetCode(f);
         *filename = code->co_filename;
         Py_DECREF(code);

Python/ceval.c

@@ -26,6 +26,7 @@
 #include "code.h"
 #include "dictobject.h"
 #include "frameobject.h"
+#include "pycore_frame.h"
 #include "opcode.h"
 #include "pydtrace.h"
 #include "setobject.h"
@@ -1547,6 +1548,9 @@ eval_frame_handle_pending(PyThreadState *tstate)
 
 #endif
 
+#define GLOBALS() specials[FRAME_SPECIALS_GLOBALS_OFFSET]
+#define BUILTINS() specials[FRAME_SPECIALS_BUILTINS_OFFSET]
+#define LOCALS() specials[FRAME_SPECIALS_LOCALS_OFFSET]
 
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
@@ -1565,7 +1569,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
     const _Py_CODEUNIT *next_instr;
     int opcode;        /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
-    PyObject **fastlocals, **freevars;
+    PyObject **fastlocals, **freevars, **specials;
     PyObject *retval = NULL;            /* Return value */
     _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
     PyCodeObject *co;
@@ -1598,6 +1602,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
     /* push frame */
     tstate->frame = f;
+    specials = f->f_valuestack - FRAME_SPECIALS_SIZE;
     co = f->f_code;
 
     if (trace_info.cframe.use_tracing) {
@@ -1641,8 +1646,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
     names = co->co_names;
     consts = co->co_consts;
-    fastlocals = f->f_localsplus;
+    fastlocals = f->f_localsptr;
-    freevars = f->f_localsplus + co->co_nlocals;
+    freevars = f->f_localsptr + co->co_nlocals;
     assert(PyBytes_Check(co->co_code));
     assert(PyBytes_GET_SIZE(co->co_code) <= INT_MAX);
     assert(PyBytes_GET_SIZE(co->co_code) % sizeof(_Py_CODEUNIT) == 0);
@@ -1692,7 +1697,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
 #ifdef LLTRACE
     {
-        int r = _PyDict_ContainsId(f->f_globals, &PyId___ltrace__);
+        int r = _PyDict_ContainsId(GLOBALS(), &PyId___ltrace__);
         if (r < 0) {
             goto exit_eval_frame;
         }
@@ -2726,8 +2731,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
             _Py_IDENTIFIER(__build_class__);
 
             PyObject *bc;
-            if (PyDict_CheckExact(f->f_builtins)) {
+            if (PyDict_CheckExact(BUILTINS())) {
-                bc = _PyDict_GetItemIdWithError(f->f_builtins, &PyId___build_class__);
+                bc = _PyDict_GetItemIdWithError(BUILTINS(), &PyId___build_class__);
                 if (bc == NULL) {
                     if (!_PyErr_Occurred(tstate)) {
                         _PyErr_SetString(tstate, PyExc_NameError,
@@ -2741,7 +2746,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                 PyObject *build_class_str = _PyUnicode_FromId(&PyId___build_class__);
                 if (build_class_str == NULL)
                     goto error;
-                bc = PyObject_GetItem(f->f_builtins, build_class_str);
+                bc = PyObject_GetItem(BUILTINS(), build_class_str);
                 if (bc == NULL) {
                     if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError))
                         _PyErr_SetString(tstate, PyExc_NameError,
@@ -2756,7 +2761,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
         case TARGET(STORE_NAME): {
             PyObject *name = GETITEM(names, oparg);
             PyObject *v = POP();
-            PyObject *ns = f->f_locals;
+            PyObject *ns = LOCALS();
             int err;
             if (ns == NULL) {
                 _PyErr_Format(tstate, PyExc_SystemError,
@@ -2776,7 +2781,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
         case TARGET(DELETE_NAME): {
             PyObject *name = GETITEM(names, oparg);
-            PyObject *ns = f->f_locals;
+            PyObject *ns = LOCALS();
             int err;
             if (ns == NULL) {
                 _PyErr_Format(tstate, PyExc_SystemError,
@@ -2868,7 +2873,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
             PyObject *name = GETITEM(names, oparg);
             PyObject *v = POP();
             int err;
-            err = PyDict_SetItem(f->f_globals, name, v);
+            err = PyDict_SetItem(GLOBALS(), name, v);
             Py_DECREF(v);
             if (err != 0)
                 goto error;
@@ -2878,7 +2883,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
         case TARGET(DELETE_GLOBAL): {
             PyObject *name = GETITEM(names, oparg);
             int err;
-            err = PyDict_DelItem(f->f_globals, name);
+            err = PyDict_DelItem(GLOBALS(), name);
             if (err != 0) {
                 if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                     format_exc_check_arg(tstate, PyExc_NameError,
@@ -2891,7 +2896,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
         case TARGET(LOAD_NAME): {
             PyObject *name = GETITEM(names, oparg);
-            PyObject *locals = f->f_locals;
+            PyObject *locals = LOCALS();
             PyObject *v;
             if (locals == NULL) {
                 _PyErr_Format(tstate, PyExc_SystemError,
@@ -2916,7 +2921,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                 }
             }
             if (v == NULL) {
-                v = PyDict_GetItemWithError(f->f_globals, name);
+                v = PyDict_GetItemWithError(GLOBALS(), name);
                 if (v != NULL) {
                     Py_INCREF(v);
                 }
@@ -2924,8 +2929,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                     goto error;
                 }
                 else {
-                    if (PyDict_CheckExact(f->f_builtins)) {
+                    if (PyDict_CheckExact(BUILTINS())) {
-                        v = PyDict_GetItemWithError(f->f_builtins, name);
+                        v = PyDict_GetItemWithError(BUILTINS(), name);
                         if (v == NULL) {
                             if (!_PyErr_Occurred(tstate)) {
                                 format_exc_check_arg(
@@ -2937,7 +2942,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                         Py_INCREF(v);
                     }
                     else {
-                        v = PyObject_GetItem(f->f_builtins, name);
+                        v = PyObject_GetItem(BUILTINS(), name);
                         if (v == NULL) {
                             if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                                 format_exc_check_arg(
@@ -2956,17 +2961,17 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
         case TARGET(LOAD_GLOBAL): {
             PyObject *name;
             PyObject *v;
-            if (PyDict_CheckExact(f->f_globals)
+            if (PyDict_CheckExact(GLOBALS())
-                && PyDict_CheckExact(f->f_builtins))
+                && PyDict_CheckExact(BUILTINS()))
             {
                 OPCACHE_CHECK();
                 if (co_opcache != NULL && co_opcache->optimized > 0) {
                     _PyOpcache_LoadGlobal *lg = &co_opcache->u.lg;
 
                     if (lg->globals_ver ==
-                            ((PyDictObject *)f->f_globals)->ma_version_tag
+                            ((PyDictObject *)GLOBALS())->ma_version_tag
                         && lg->builtins_ver ==
-                           ((PyDictObject *)f->f_builtins)->ma_version_tag)
+                           ((PyDictObject *)BUILTINS())->ma_version_tag)
                     {
                         PyObject *ptr = lg->ptr;
                         OPCACHE_STAT_GLOBAL_HIT();
@@ -2978,8 +2983,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                 }
 
                 name = GETITEM(names, oparg);
-                v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals,
+                v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
-                                       (PyDictObject *)f->f_builtins,
+                                       (PyDictObject *)BUILTINS(),
                                        name);
                 if (v == NULL) {
                     if (!_PyErr_Occurred(tstate)) {
@@ -3003,9 +3008,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
                     co_opcache->optimized = 1;
                     lg->globals_ver =
-                        ((PyDictObject *)f->f_globals)->ma_version_tag;
+                        ((PyDictObject *)GLOBALS())->ma_version_tag;
                     lg->builtins_ver =
-                        ((PyDictObject *)f->f_builtins)->ma_version_tag;
+                        ((PyDictObject *)BUILTINS())->ma_version_tag;
                     lg->ptr = v; /* borrowed */
                 }
 
@@ -3016,7 +3021,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
 
                 /* namespace 1: globals */
                 name = GETITEM(names, oparg);
-                v = PyObject_GetItem(f->f_globals, name);
+                v = PyObject_GetItem(GLOBALS(), name);
                 if (v == NULL) {
                     if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                         goto error;
@@ -3024,7 +3029,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                     _PyErr_Clear(tstate);
 
                     /* namespace 2: builtins */
-                    v = PyObject_GetItem(f->f_builtins, name);
+                    v = PyObject_GetItem(BUILTINS(), name);
                     if (v == NULL) {
                         if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                             format_exc_check_arg(
@@ -3073,7 +3078,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
         }
 
         case TARGET(LOAD_CLASSDEREF): {
-            PyObject *name, *value, *locals = f->f_locals;
+            PyObject *name, *value, *locals = LOCALS();
             Py_ssize_t idx;
             assert(locals);
             assert(oparg >= PyTuple_GET_SIZE(co->co_cellvars));
@@ -3266,14 +3271,14 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
             _Py_IDENTIFIER(__annotations__);
             int err;
             PyObject *ann_dict;
-            if (f->f_locals == NULL) {
+            if (LOCALS() == NULL) {
                 _PyErr_Format(tstate, PyExc_SystemError,
                               "no locals found when setting up annotations");
                 goto error;
             }
             /* check if __annotations__ in locals()... */
-            if (PyDict_CheckExact(f->f_locals)) {
+            if (PyDict_CheckExact(LOCALS())) {
-                ann_dict = _PyDict_GetItemIdWithError(f->f_locals,
+                ann_dict = _PyDict_GetItemIdWithError(LOCALS(),
                                              &PyId___annotations__);
                 if (ann_dict == NULL) {
                     if (_PyErr_Occurred(tstate)) {
@@ -3284,7 +3289,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                     if (ann_dict == NULL) {
                         goto error;
                     }
-                    err = _PyDict_SetItemId(f->f_locals,
+                    err = _PyDict_SetItemId(LOCALS(),
                                             &PyId___annotations__, ann_dict);
                     Py_DECREF(ann_dict);
                     if (err != 0) {
@@ -3298,7 +3303,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                 if (ann_str == NULL) {
                     goto error;
                 }
-                ann_dict = PyObject_GetItem(f->f_locals, ann_str);
+                ann_dict = PyObject_GetItem(LOCALS(), ann_str);
                 if (ann_dict == NULL) {
                     if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
                         goto error;
@@ -3308,7 +3313,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                     if (ann_dict == NULL) {
                         goto error;
                     }
-                    err = PyObject_SetItem(f->f_locals, ann_str, ann_dict);
+                    err = PyObject_SetItem(LOCALS(), ann_str, ann_dict);
                     Py_DECREF(ann_dict);
                     if (err != 0) {
                         goto error;
@@ -3707,7 +3712,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
                 goto error;
             }
 
-            locals = f->f_locals;
+            locals = LOCALS();
             if (locals == NULL) {
                 _PyErr_SetString(tstate, PyExc_SystemError,
                                  "no locals found during 'import *'");
@@ -4313,7 +4318,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
             PyObject *qualname = POP();
             PyObject *codeobj = POP();
             PyFunctionObject *func = (PyFunctionObject *)
-                PyFunction_NewWithQualName(codeobj, f->f_globals, qualname);
+                PyFunction_NewWithQualName(codeobj, GLOBALS(), qualname);
 
             Py_DECREF(codeobj);
             Py_DECREF(qualname);
@@ -4869,25 +4874,14 @@ get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, i
     return 0;
 }
 
-PyFrameObject *
+static int
-_PyEval_MakeFrameVector(PyThreadState *tstate,
+initialize_locals(PyThreadState *tstate, PyFrameConstructor *con,
-           PyFrameConstructor *con, PyObject *locals,
+    PyObject **fastlocals, PyObject *const *args,
-           PyObject *const *args, Py_ssize_t argcount,
+    Py_ssize_t argcount, PyObject *kwnames)
-           PyObject *kwnames)
 {
-    assert(is_tstate_valid(tstate));
-
     PyCodeObject *co = (PyCodeObject*)con->fc_code;
-    assert(con->fc_defaults == NULL || PyTuple_CheckExact(con->fc_defaults));
     const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount;
-
+    PyObject **freevars = fastlocals + co->co_nlocals;
-    /* Create the frame */
-    PyFrameObject *f = _PyFrame_New_NoTrack(tstate, con, locals);
-    if (f == NULL) {
-        return NULL;
-    }
-    PyObject **fastlocals = f->f_localsplus;
-    PyObject **freevars = f->f_localsplus + co->co_nlocals;
 
     /* Create a dictionary for keyword parameters (**kwags) */
     PyObject *kwdict;
@@ -5093,25 +5087,33 @@ _PyEval_MakeFrameVector(PyThreadState *tstate,
         freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o;
     }
 
-    return f;
+    return 0;
 
 fail: /* Jump here from prelude on failure */
+    return -1;
 
-    /* decref'ing the frame can cause __del__ methods to get invoked,
+}
-       which can call back into Python.  While we're done with the
+
-       current Python frame (f), the associated C stack is still in use,
+
-       so recursion_depth must be boosted for the duration.
+PyFrameObject *
-    */
+_PyEval_MakeFrameVector(PyThreadState *tstate,
-    if (Py_REFCNT(f) > 1) {
+           PyFrameConstructor *con, PyObject *locals,
-        Py_DECREF(f);
+           PyObject *const *args, Py_ssize_t argcount,
-        _PyObject_GC_TRACK(f);
+           PyObject *kwnames, PyObject** localsarray)
+{
+    assert(is_tstate_valid(tstate));
+    assert(con->fc_defaults == NULL || PyTuple_CheckExact(con->fc_defaults));
+
+    /* Create the frame */
+    PyFrameObject *f = _PyFrame_New_NoTrack(tstate, con, locals, localsarray);
+    if (f == NULL) {
+        return NULL;
     }
-    else {
+    if (initialize_locals(tstate, con, f->f_localsptr, args, argcount, kwnames)) {
-        ++tstate->recursion_depth;
         Py_DECREF(f);
-        --tstate->recursion_depth;
+        return NULL;
     }
-    return NULL;
+    return f;
 }
 
 static PyObject *
@@ -5149,30 +5151,59 @@ _PyEval_Vector(PyThreadState *tstate, PyFrameConstructor *con,
                PyObject* const* args, size_t argcount,
                PyObject *kwnames)
 {
+    PyObject **localsarray;
+    PyCodeObject *code = (PyCodeObject *)con->fc_code;
+    int is_coro = code->co_flags &
+        (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR);
+    if (is_coro) {
+        localsarray = NULL;
+    }
+    else {
+        int size = code->co_nlocalsplus + code->co_stacksize +
+            FRAME_SPECIALS_SIZE;
+        localsarray = _PyThreadState_PushLocals(tstate, size);
+        if (localsarray == NULL) {
+            return NULL;
+        }
+    }
     PyFrameObject *f = _PyEval_MakeFrameVector(
-        tstate, con, locals, args, argcount, kwnames);
+        tstate, con, locals, args, argcount, kwnames, localsarray);
     if (f == NULL) {
+        if (!is_coro) {
+            _PyThreadState_PopLocals(tstate, localsarray);
+        }
         return NULL;
     }
-    if (((PyCodeObject *)con->fc_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
+    if (is_coro) {
         return make_coro(con, f);
     }
     PyObject *retval = _PyEval_EvalFrame(tstate, f, 0);
+    assert(f->f_stackdepth == 0);
 
     /* decref'ing the frame can cause __del__ methods to get invoked,
        which can call back into Python.  While we're done with the
        current Python frame (f), the associated C stack is still in use,
        so recursion_depth must be boosted for the duration.
     */
+    assert (!is_coro);
+    assert(f->f_own_locals_memory == 0);
     if (Py_REFCNT(f) > 1) {
         Py_DECREF(f);
         _PyObject_GC_TRACK(f);
+        if (_PyFrame_TakeLocals(f)) {
+            Py_CLEAR(retval);
+        }
     }
     else {
         ++tstate->recursion_depth;
+        f->f_localsptr = NULL;
+        for (int i = 0; i < code->co_nlocalsplus + FRAME_SPECIALS_SIZE; i++) {
+            Py_XDECREF(localsarray[i]);
+        }
         Py_DECREF(f);
         --tstate->recursion_depth;
     }
+    _PyThreadState_PopLocals(tstate, localsarray);
     return retval;
 }
 
@@ -5778,7 +5809,7 @@ _PyEval_GetBuiltins(PyThreadState *tstate)
 {
     PyFrameObject *frame = tstate->frame;
     if (frame != NULL) {
-        return frame->f_builtins;
+        return _PyFrame_GetBuiltins(frame);
     }
     return tstate->interp->builtins;
 }
@@ -5819,8 +5850,10 @@ PyEval_GetLocals(void)
         return NULL;
     }
 
-    assert(current_frame->f_locals != NULL);
+    PyObject *locals = current_frame->f_valuestack[
-    return current_frame->f_locals;
+        FRAME_SPECIALS_LOCALS_OFFSET-FRAME_SPECIALS_SIZE];
+    assert(locals != NULL);
+    return locals;
 }
 
 PyObject *
@@ -5831,9 +5864,7 @@ PyEval_GetGlobals(void)
     if (current_frame == NULL) {
         return NULL;
     }
-
+    return _PyFrame_GetGlobals(current_frame);
-    assert(current_frame->f_globals != NULL);
-    return current_frame->f_globals;
 }
 
 int
@@ -6084,14 +6115,15 @@ import_name(PyThreadState *tstate, PyFrameObject *f,
     PyObject *import_func, *res;
     PyObject* stack[5];
 
-    import_func = _PyDict_GetItemIdWithError(f->f_builtins, &PyId___import__);
+    import_func = _PyDict_GetItemIdWithError(_PyFrame_GetBuiltins(f), &PyId___import__);
     if (import_func == NULL) {
         if (!_PyErr_Occurred(tstate)) {
             _PyErr_SetString(tstate, PyExc_ImportError, "__import__ not found");
         }
         return NULL;
     }
-
+    PyObject *locals = f->f_valuestack[
+        FRAME_SPECIALS_LOCALS_OFFSET-FRAME_SPECIALS_SIZE];
     /* Fast path for not overloaded __import__. */
     if (import_func == tstate->interp->import_func) {
         int ilevel = _PyLong_AsInt(level);
@@ -6100,8 +6132,8 @@ import_name(PyThreadState *tstate, PyFrameObject *f,
         }
         res = PyImport_ImportModuleLevelObject(
                         name,
-                        f->f_globals,
+                        _PyFrame_GetGlobals(f),
-                        f->f_locals == NULL ? Py_None : f->f_locals,
+                        locals == NULL ? Py_None :locals,
                         fromlist,
                         ilevel);
         return res;
@@ -6110,8 +6142,8 @@ import_name(PyThreadState *tstate, PyFrameObject *f,
     Py_INCREF(import_func);
 
     stack[0] = name;
-    stack[1] = f->f_globals;
+    stack[1] = _PyFrame_GetGlobals(f);
-    stack[2] = f->f_locals == NULL ? Py_None : f->f_locals;
+    stack[2] = locals == NULL ? Py_None : locals;
     stack[3] = fromlist;
     stack[4] = level;
     res = _PyObject_FastCall(import_func, stack, 5);
@@ -6436,14 +6468,14 @@ unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w,
         switch (opcode) {
         case STORE_FAST:
         {
-            PyObject **fastlocals = f->f_localsplus;
+            PyObject **fastlocals = f->f_localsptr;
             if (GETLOCAL(oparg) == v)
                 SETLOCAL(oparg, NULL);
             break;
         }
         case STORE_DEREF:
         {
-            PyObject **freevars = (f->f_localsplus +
+            PyObject **freevars = (f->f_localsptr +
                                    f->f_code->co_nlocals);
             PyObject *c = freevars[oparg];
             if (PyCell_GET(c) ==  v) {
@@ -6456,7 +6488,8 @@ unicode_concatenate(PyThreadState *tstate, PyObject *v, PyObject *w,
         {
             PyObject *names = f->f_code->co_names;
             PyObject *name = GETITEM(names, oparg);
-            PyObject *locals = f->f_locals;
+            PyObject *locals = f->f_valuestack[
+                FRAME_SPECIALS_LOCALS_OFFSET-FRAME_SPECIALS_SIZE];
             if (locals && PyDict_CheckExact(locals)) {
                 PyObject *w = PyDict_GetItemWithError(locals, name);
                 if ((w == v && PyDict_DelItem(locals, name) != 0) ||

Python/pystate.c

@@ -607,6 +607,23 @@ PyInterpreterState_GetDict(PyInterpreterState *interp)
     return interp->dict;
 }
 
+/* Minimum size of data stack chunk */
+#define DATA_STACK_CHUNK_SIZE (16*1024)
+
+static _PyStackChunk*
+allocate_chunk(int size_in_bytes, _PyStackChunk* previous)
+{
+    assert(size_in_bytes % sizeof(PyObject **) == 0);
+    _PyStackChunk *res = _PyObject_VirtualAlloc(size_in_bytes);
+    if (res == NULL) {
+        return NULL;
+    }
+    res->previous = previous;
+    res->size = size_in_bytes;
+    res->top = 0;
+    return res;
+}
+
 static PyThreadState *
 new_threadstate(PyInterpreterState *interp, int init)
 {
@@ -658,6 +675,14 @@ new_threadstate(PyInterpreterState *interp, int init)
 
     tstate->context = NULL;
     tstate->context_ver = 1;
+    tstate->datastack_chunk = allocate_chunk(DATA_STACK_CHUNK_SIZE, NULL);
+    if (tstate->datastack_chunk == NULL) {
+        PyMem_RawFree(tstate);
+        return NULL;
+    }
+    /* If top points to entry 0, then _PyThreadState_PopLocals will try to pop this chunk */
+    tstate->datastack_top = &tstate->datastack_chunk->data[1];
+    tstate->datastack_limit = (PyObject **)(((char *)tstate->datastack_chunk) + DATA_STACK_CHUNK_SIZE);
 
     if (init) {
         _PyThreadState_Init(tstate);
@@ -872,6 +897,13 @@ PyThreadState_Clear(PyThreadState *tstate)
     if (tstate->on_delete != NULL) {
         tstate->on_delete(tstate->on_delete_data);
     }
+    _PyStackChunk *chunk = tstate->datastack_chunk;
+    tstate->datastack_chunk = NULL;
+    while (chunk != NULL) {
+        _PyStackChunk *prev = chunk->previous;
+        _PyObject_VirtualFree(chunk, chunk->size);
+        chunk = prev;
+    }
 }
 
 
@@ -906,7 +938,6 @@ tstate_delete_common(PyThreadState *tstate,
     }
 }
 
-
 static void
 _PyThreadState_Delete(PyThreadState *tstate, int check_current)
 {
@@ -1969,6 +2000,59 @@ _Py_GetConfig(void)
     return _PyInterpreterState_GetConfig(tstate->interp);
 }
 
+#define MINIMUM_OVERHEAD 1000
+
+PyObject **
+_PyThreadState_PushLocals(PyThreadState *tstate, int size)
+{
+    assert(((unsigned)size) < INT_MAX/sizeof(PyObject*)/2);
+    PyObject **res = tstate->datastack_top;
+    PyObject **top = res + size;
+    if (top >= tstate->datastack_limit) {
+        int allocate_size = DATA_STACK_CHUNK_SIZE;
+        while (allocate_size < (int)sizeof(PyObject*)*(size + MINIMUM_OVERHEAD)) {
+            allocate_size *= 2;
+        }
+        _PyStackChunk *new = allocate_chunk(allocate_size, tstate->datastack_chunk);
+        if (new == NULL) {
+            goto error;
+        }
+        tstate->datastack_chunk->top = tstate->datastack_top - &tstate->datastack_chunk->data[0];
+        tstate->datastack_chunk = new;
+        tstate->datastack_limit = (PyObject **)(((char *)new) + allocate_size);
+        res = &new->data[0];
+        tstate->datastack_top = res + size;
+    }
+    else {
+        tstate->datastack_top = top;
+    }
+    for (int i=0; i < size; i++) {
+        res[i] = NULL;
+    }
+    return res;
+error:
+    _PyErr_SetString(tstate, PyExc_MemoryError, "Out of memory");
+    return NULL;
+}
+
+void
+_PyThreadState_PopLocals(PyThreadState *tstate, PyObject **locals)
+{
+    if (locals == &tstate->datastack_chunk->data[0]) {
+        _PyStackChunk *chunk = tstate->datastack_chunk;
+        _PyStackChunk *previous = chunk->previous;
+        tstate->datastack_top = &previous->data[previous->top];
+        tstate->datastack_chunk = previous;
+        _PyObject_VirtualFree(chunk, chunk->size);
+        tstate->datastack_limit = (PyObject **)(((char *)previous) + previous->size);
+    }
+    else {
+        assert(tstate->datastack_top >= locals);
+        tstate->datastack_top = locals;
+    }
+}
+
+
 #ifdef __cplusplus
 }
 #endif

Python/suggestions.c

@@ -1,5 +1,6 @@
 #include "Python.h"
 #include "frameobject.h"
+#include "pycore_frame.h"
 
 #include "pycore_pyerrors.h"
 
@@ -208,9 +209,10 @@ offer_suggestions_for_name_error(PyNameErrorObject *exc)
 
     PyFrameObject *frame = traceback->tb_frame;
     assert(frame != NULL);
-    PyCodeObject *code = frame->f_code;
+    PyCodeObject *code = PyFrame_GetCode(frame);
     assert(code != NULL && code->co_varnames != NULL);
     PyObject *dir = PySequence_List(code->co_varnames);
+    Py_DECREF(code);
     if (dir == NULL) {
         return NULL;
     }
@@ -221,7 +223,7 @@ offer_suggestions_for_name_error(PyNameErrorObject *exc)
         return suggestions;
     }
 
-    dir = PySequence_List(frame->f_globals);
+    dir = PySequence_List(_PyFrame_GetGlobals(frame));
     if (dir == NULL) {
         return NULL;
     }
@@ -231,7 +233,7 @@ offer_suggestions_for_name_error(PyNameErrorObject *exc)
         return suggestions;
     }
 
-    dir = PySequence_List(frame->f_builtins);
+    dir = PySequence_List(_PyFrame_GetBuiltins(frame));
     if (dir == NULL) {
         return NULL;
     }

Tools/gdb/libpython.py

@@ -854,6 +854,8 @@ class PyNoneStructPtr(PyObjectPtr):
     def proxyval(self, visited):
         return None
 
+FRAME_SPECIALS_GLOBAL_OFFSET = 0
+FRAME_SPECIALS_BUILTINS_OFFSET = 1
 
 class PyFrameObjectPtr(PyObjectPtr):
     _typename = 'PyFrameObject'
@@ -879,13 +881,19 @@ def iter_locals(self):
         if self.is_optimized_out():
             return
 
-        f_localsplus = self.field('f_localsplus')
+        f_localsplus = self.field('f_localsptr')
         for i in safe_range(self.co_nlocals):
             pyop_value = PyObjectPtr.from_pyobject_ptr(f_localsplus[i])
             if not pyop_value.is_null():
                 pyop_name = PyObjectPtr.from_pyobject_ptr(self.co_varnames[i])
                 yield (pyop_name, pyop_value)
 
+    def _f_globals(self):
+        f_localsplus = self.field('f_localsptr')
+        nlocalsplus = int_from_int(self.co.field('co_nlocalsplus'))
+        index = nlocalsplus + FRAME_SPECIALS_GLOBAL_OFFSET
+        return PyObjectPtr.from_pyobject_ptr(f_localsplus[index])
+
     def iter_globals(self):
         '''
         Yield a sequence of (name,value) pairs of PyObjectPtr instances, for
@@ -894,9 +902,15 @@ def iter_globals(self):
         if self.is_optimized_out():
             return ()
 
-        pyop_globals = self.pyop_field('f_globals')
+        pyop_globals = self._f_globals()
         return pyop_globals.iteritems()
 
+    def _f_builtins(self):
+        f_localsplus = self.field('f_localsptr')
+        nlocalsplus = int_from_int(self.co.field('co_nlocalsplus'))
+        index = nlocalsplus + FRAME_SPECIALS_BUILTINS_OFFSET
+        return PyObjectPtr.from_pyobject_ptr(f_localsplus[index])
+
     def iter_builtins(self):
         '''
         Yield a sequence of (name,value) pairs of PyObjectPtr instances, for
@@ -905,7 +919,7 @@ def iter_builtins(self):
         if self.is_optimized_out():
             return ()
 
-        pyop_builtins = self.pyop_field('f_builtins')
+        pyop_builtins = self._f_builtins()
         return pyop_builtins.iteritems()
 
     def get_var_by_name(self, name):