GH-131729: Code-gen better liveness analysis (GH-131732)

* Rename 'defined' attribute to 'in_local' to more accurately reflect how it is used

* Make death of variables explicit even for array variables.

* Convert in_memory from boolean to stack offset

* Don't apply liveness analysis to optimizer generated code

* Fix RETURN_VALUE in optimizer

Include/internal/pycore_uop_metadata.h

@@ -930,7 +930,7 @@ int _PyUop_num_popped(int opcode, int oparg)
         case _LOAD_ATTR_METHOD_LAZY_DICT:
             return 1;
         case _MAYBE_EXPAND_METHOD:
-            return 2 + oparg;
+            return 0;
         case _PY_FRAME_GENERAL:
             return 2 + oparg;
         case _CHECK_FUNCTION_VERSION:
@@ -976,7 +976,7 @@ int _PyUop_num_popped(int opcode, int oparg)
         case _CALL_TUPLE_1:
             return 3;
         case _CHECK_AND_ALLOCATE_OBJECT:
-            return 2 + oparg;
+            return 0;
         case _CREATE_INIT_FRAME:
             return 2 + oparg;
         case _EXIT_INIT_CHECK:
@@ -1004,7 +1004,7 @@ int _PyUop_num_popped(int opcode, int oparg)
         case _CALL_METHOD_DESCRIPTOR_FAST:
             return 2 + oparg;
         case _MAYBE_EXPAND_METHOD_KW:
-            return 3 + oparg;
+            return 1;
         case _PY_FRAME_KW:
             return 3 + oparg;
         case _CHECK_FUNCTION_VERSION_KW:

Lib/test/test_generated_cases.py

@@ -285,9 +285,11 @@ def test_overlap(self):
     def test_predictions(self):
         input = """
         inst(OP1, (arg -- res)) {
+            DEAD(arg);
             res = Py_None;
         }
         inst(OP3, (arg -- res)) {
+            DEAD(arg);
             DEOPT_IF(xxx);
             res = Py_None;
         }
@@ -303,7 +305,9 @@ def test_predictions(self):
             next_instr += 1;
             INSTRUCTION_STATS(OP1);
             PREDICTED_OP1:;
+            _PyStackRef arg;
             _PyStackRef res;
+            arg = stack_pointer[-1];
             res = Py_None;
             stack_pointer[-1] = res;
             DISPATCH();
@@ -320,7 +324,9 @@ def test_predictions(self):
             next_instr += 1;
             INSTRUCTION_STATS(OP3);
             static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size");
+            _PyStackRef arg;
             _PyStackRef res;
+            arg = stack_pointer[-1];
             if (xxx) {
                 UPDATE_MISS_STATS(OP1);
                 assert(_PyOpcode_Deopt[opcode] == (OP1));
@@ -336,11 +342,13 @@ def test_predictions(self):
     def test_sync_sp(self):
         input = """
         inst(A, (arg -- res)) {
+            DEAD(arg);
             SYNC_SP();
             escaping_call();
             res = Py_None;
         }
         inst(B, (arg -- res)) {
+            DEAD(arg);
             res = Py_None;
             SYNC_SP();
             escaping_call();
@@ -355,7 +363,9 @@ def test_sync_sp(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(A);
+            _PyStackRef arg;
             _PyStackRef res;
+            arg = stack_pointer[-1];
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
             _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -376,7 +386,9 @@ def test_sync_sp(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(B);
+            _PyStackRef arg;
             _PyStackRef res;
+            arg = stack_pointer[-1];
             res = Py_None;
             stack_pointer[-1] = res;
             _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -522,6 +534,7 @@ def test_error_if_pop_with_result(self):
     def test_cache_effect(self):
         input = """
         inst(OP, (counter/1, extra/2, value --)) {
+            DEAD(value);
         }
     """
         output = """
@@ -535,6 +548,8 @@ def test_cache_effect(self):
             frame->instr_ptr = next_instr;
             next_instr += 4;
             INSTRUCTION_STATS(OP);
+            _PyStackRef value;
+            value = stack_pointer[-1];
             uint16_t counter = read_u16(&this_instr[1].cache);
             (void)counter;
             uint32_t extra = read_u32(&this_instr[2].cache);
@@ -793,6 +808,9 @@ def test_array_input(self):
         input = """
         inst(OP, (below, values[oparg*2], above --)) {
             SPAM(values, oparg);
+            DEAD(below);
+            DEAD(values);
+            DEAD(above);
         }
     """
         output = """
@@ -804,8 +822,12 @@ def test_array_input(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(OP);
+            _PyStackRef below;
             _PyStackRef *values;
+            _PyStackRef above;
+            above = stack_pointer[-1];
             values = &stack_pointer[-1 - oparg*2];
+            below = stack_pointer[-2 - oparg*2];
             SPAM(values, oparg);
             stack_pointer += -2 - oparg*2;
             assert(WITHIN_STACK_BOUNDS());
@@ -879,6 +901,8 @@ def test_array_input_output(self):
     def test_array_error_if(self):
         input = """
         inst(OP, (extra, values[oparg] --)) {
+            DEAD(extra);
+            DEAD(values);
             ERROR_IF(oparg == 0, somewhere);
         }
     """
@@ -891,6 +915,10 @@ def test_array_error_if(self):
             frame->instr_ptr = next_instr;
             next_instr += 1;
             INSTRUCTION_STATS(OP);
+            _PyStackRef extra;
+            _PyStackRef *values;
+            values = &stack_pointer[-oparg];
+            extra = stack_pointer[-1 - oparg];
             if (oparg == 0) {
                 stack_pointer += -1 - oparg;
                 assert(WITHIN_STACK_BOUNDS());
@@ -1056,6 +1084,7 @@ def test_array_of_one(self):
         input = """
         inst(OP, (arg[1] -- out[1])) {
             out[0] = arg[0];
+            DEAD(arg);
         }
         """
         output = """

Python/bytecodes.c

@@ -1927,6 +1927,7 @@ dummy_func(
                     PyStackRef_CLOSE(value);
                 }
             }
+            DEAD(values);
             if (err) {
                 Py_DECREF(set_o);
                 ERROR_IF(true, error);
@@ -3583,15 +3584,15 @@ dummy_func(
             #endif  /* ENABLE_SPECIALIZATION_FT */
         }
 
-        op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) {
+        op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) {
             (void)args;
             if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                 PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                 PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                 _PyStackRef temp = callable[0];
-                func[0] = PyStackRef_FromPyObjectNew(method);
+                callable[0] = PyStackRef_FromPyObjectNew(method);
                 PyStackRef_CLOSE(temp);
             }
         }
@@ -3618,6 +3619,9 @@ dummy_func(
                     tstate, callable[0], locals,
                     arguments, total_args, NULL, frame
                 );
+                DEAD(args);
+                DEAD(self_or_null);
+                DEAD(callable);
                 // Manipulate stack directly since we leave using DISPATCH_INLINED().
                 SYNC_SP();
                 // The frame has stolen all the arguments from the stack,
@@ -3950,10 +3954,10 @@ dummy_func(
             _CALL_TUPLE_1 +
             _CHECK_PERIODIC;
 
-        op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], null[1], args[oparg] -- init[1], self[1], args[oparg])) {
+        op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) {
             (void)args;
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
-            DEOPT_IF(!PyStackRef_IsNull(null[0]));
+            DEOPT_IF(!PyStackRef_IsNull(self_or_null[0]));
             DEOPT_IF(!PyType_Check(callable_o));
             PyTypeObject *tp = (PyTypeObject *)callable_o;
             DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version);
@@ -3969,9 +3973,9 @@ dummy_func(
             if (self_o == NULL) {
                 ERROR_NO_POP();
             }
-            self[0] = PyStackRef_FromPyObjectSteal(self_o);
+            self_or_null[0] = PyStackRef_FromPyObjectSteal(self_o);
             _PyStackRef temp = callable[0];
-            init[0] = PyStackRef_FromPyObjectNew(init_func);
+            callable[0] = PyStackRef_FromPyObjectNew(init_func);
             PyStackRef_CLOSE(temp);
         }
 
@@ -3982,10 +3986,11 @@ dummy_func(
             assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE);
             /* Push self onto stack of shim */
             shim->localsplus[0] = PyStackRef_DUP(self[0]);
-            DEAD(init);
-            DEAD(self);
             _PyInterpreterFrame *temp = _PyEvalFramePushAndInit(
                 tstate, init[0], NULL, args-1, oparg+1, NULL, shim);
+            DEAD(init);
+            DEAD(self);
+            DEAD(args);
             SYNC_SP();
             if (temp == NULL) {
                 _PyEval_FrameClearAndPop(tstate, shim);
@@ -4187,9 +4192,9 @@ dummy_func(
             res = PyStackRef_FromPyObjectSteal(res_o);
         }
 
-        inst(CALL_ISINSTANCE, (unused/1, unused/2, callable[1], self_or_null[1], args[oparg] -- res)) {
+        inst(CALL_ISINSTANCE, (unused/1, unused/2, callable, self_or_null[1], args[oparg] -- res)) {
             /* isinstance(o, o2) */
-            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
+            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
 
             int total_args = oparg;
             _PyStackRef *arguments = args;
@@ -4420,15 +4425,15 @@ dummy_func(
             ERROR_IF(err, error);
         }
 
-        op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- func[1], maybe_self[1], args[oparg], kwnames_out)) {
+        op(_MAYBE_EXPAND_METHOD_KW, (callable[1], self_or_null[1], args[oparg], kwnames_in -- callable[1], self_or_null[1], args[oparg], kwnames_out)) {
             (void)args;
             if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                 PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                 PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                 _PyStackRef temp = callable[0];
-                func[0] = PyStackRef_FromPyObjectNew(method);
+                callable[0] = PyStackRef_FromPyObjectNew(method);
                 PyStackRef_CLOSE(temp);
             }
             kwnames_out = kwnames_in;
@@ -4458,6 +4463,9 @@ dummy_func(
                     tstate, callable[0], locals,
                     arguments, positional_args, kwnames_o, frame
                 );
+                DEAD(args);
+                DEAD(self_or_null);
+                DEAD(callable);
                 PyStackRef_CLOSE(kwnames);
                 // Sync stack explicitly since we leave using DISPATCH_INLINED().
                 SYNC_SP();
@@ -4525,6 +4533,9 @@ dummy_func(
             PyStackRef_CLOSE(kwnames);
             // The frame has stolen all the arguments from the stack,
             // so there is no need to clean them up.
+            DEAD(args);
+            DEAD(self_or_null);
+            DEAD(callable);
             SYNC_SP();
             ERROR_IF(temp == NULL, error);
             new_frame = temp;

Python/executor_cases.c.h

@@ -635,7 +635,6 @@
             stack_pointer[-1] = value;
             PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             break;
         }
 
@@ -1259,13 +1258,13 @@
             list_st = res;
             stack_pointer[-2] = list_st;
             PyStackRef_CLOSE(tmp);
+            tmp = sub_st;
+            sub_st = PyStackRef_NULL;
+            stack_pointer[-1] = sub_st;
+            PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            PyStackRef_CLOSE(sub_st);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             break;
         }
 
@@ -1354,7 +1353,6 @@
             stack_pointer[-1] = tuple_st;
             PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             break;
         }
 
@@ -2054,6 +2052,7 @@
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(seq);
             stack_pointer = _PyFrame_GetStackPointer(frame);
+            stack_pointer[-1] = val0;
             break;
         }
 
@@ -2077,13 +2076,11 @@
             for (int i = oparg; --i >= 0; ) {
                 *values++ = PyStackRef_FromPyObjectNew(items[i]);
             }
-            stack_pointer += -1;
+            stack_pointer += -1 + oparg;
             assert(WITHIN_STACK_BOUNDS());
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(seq);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += oparg;
-            assert(WITHIN_STACK_BOUNDS());
             break;
         }
 
@@ -2115,13 +2112,11 @@
                 *values++ = PyStackRef_FromPyObjectNew(items[i]);
             }
             UNLOCK_OBJECT(seq_o);
-            stack_pointer += -1;
+            stack_pointer += -1 + oparg;
             assert(WITHIN_STACK_BOUNDS());
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(seq);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += oparg;
-            assert(WITHIN_STACK_BOUNDS());
             break;
         }
 
@@ -2701,11 +2696,11 @@
                 }
             }
             if (err) {
+                stack_pointer += -oparg;
+                assert(WITHIN_STACK_BOUNDS());
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 Py_DECREF(set_o);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer += -oparg;
-                assert(WITHIN_STACK_BOUNDS());
                 JUMP_TO_ERROR();
             }
             set = PyStackRef_FromPyObjectStealMortal(set_o);
@@ -3002,13 +2997,12 @@
             global_super_st = self_or_null;
             stack_pointer[-2] = global_super_st;
             PyStackRef_CLOSE(tmp);
+            tmp = class_st;
+            class_st = PyStackRef_NULL;
+            stack_pointer[-1] = class_st;
+            PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += -1;
+            stack_pointer += -2;
-            assert(WITHIN_STACK_BOUNDS());
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            PyStackRef_CLOSE(class_st);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
             attr = PyStackRef_FromPyObjectSteal(attr_o);
             stack_pointer[0] = attr;
@@ -3296,7 +3290,6 @@
             stack_pointer[-1] = owner;
             PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = attr;
             break;
         }
 
@@ -3331,7 +3324,6 @@
             stack_pointer[-1] = owner;
             PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = attr;
             break;
         }
 
@@ -3936,9 +3928,7 @@
                 stack_pointer[-1] = value;
                 PyStackRef_CLOSE(tmp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer[-1] = b;
             }
-            stack_pointer[-1] = b;
             break;
         }
 
@@ -4120,10 +4110,8 @@
                     stack_pointer[-1] = iterable;
                     PyStackRef_CLOSE(tmp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer[-1] = iter;
                 }
             }
-            stack_pointer[-1] = iter;
             break;
         }
 
@@ -4667,23 +4655,18 @@
             _PyStackRef *args;
             _PyStackRef *self_or_null;
             _PyStackRef *callable;
-            _PyStackRef *func;
-            _PyStackRef *maybe_self;
             oparg = CURRENT_OPARG();
             args = &stack_pointer[-oparg];
             self_or_null = &stack_pointer[-1 - oparg];
             callable = &stack_pointer[-2 - oparg];
-            func = &stack_pointer[-2 - oparg];
-            maybe_self = &stack_pointer[-1 - oparg];
-            args = &stack_pointer[-oparg];
             (void)args;
             if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                 PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                 PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                 _PyStackRef temp = callable[0];
-                func[0] = PyStackRef_FromPyObjectNew(method);
+                callable[0] = PyStackRef_FromPyObjectNew(method);
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 PyStackRef_CLOSE(temp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -5252,21 +5235,16 @@
 
         case _CHECK_AND_ALLOCATE_OBJECT: {
             _PyStackRef *args;
-            _PyStackRef *null;
+            _PyStackRef *self_or_null;
             _PyStackRef *callable;
-            _PyStackRef *init;
-            _PyStackRef *self;
             oparg = CURRENT_OPARG();
             args = &stack_pointer[-oparg];
-            null = &stack_pointer[-1 - oparg];
+            self_or_null = &stack_pointer[-1 - oparg];
             callable = &stack_pointer[-2 - oparg];
-            init = &stack_pointer[-2 - oparg];
-            self = &stack_pointer[-1 - oparg];
-            args = &stack_pointer[-oparg];
             uint32_t type_version = (uint32_t)CURRENT_OPERAND0();
             (void)args;
             PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
-            if (!PyStackRef_IsNull(null[0])) {
+            if (!PyStackRef_IsNull(self_or_null[0])) {
                 UOP_STAT_INC(uopcode, miss);
                 JUMP_TO_JUMP_TARGET();
             }
@@ -5296,9 +5274,9 @@
             if (self_o == NULL) {
                 JUMP_TO_ERROR();
             }
-            self[0] = PyStackRef_FromPyObjectSteal(self_o);
+            self_or_null[0] = PyStackRef_FromPyObjectSteal(self_o);
             _PyStackRef temp = callable[0];
-            init[0] = PyStackRef_FromPyObjectNew(init_func);
+            callable[0] = PyStackRef_FromPyObjectNew(init_func);
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(temp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -5720,14 +5698,14 @@
         case _CALL_ISINSTANCE: {
             _PyStackRef *args;
             _PyStackRef *self_or_null;
-            _PyStackRef *callable;
+            _PyStackRef callable;
             _PyStackRef res;
             oparg = CURRENT_OPARG();
             args = &stack_pointer[-oparg];
             self_or_null = &stack_pointer[-1 - oparg];
-            callable = &stack_pointer[-2 - oparg];
+            callable = stack_pointer[-2 - oparg];
             /* isinstance(o, o2) */
-            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
+            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             int total_args = oparg;
             _PyStackRef *arguments = args;
             if (!PyStackRef_IsNull(self_or_null[0])) {
@@ -5755,8 +5733,9 @@
             res = retval ? PyStackRef_True : PyStackRef_False;
             assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            _PyStackRef tmp = callable[0];
+            _PyStackRef tmp = callable;
-            callable[0] = res;
+            callable = res;
+            stack_pointer[-2 - oparg] = callable;
             PyStackRef_CLOSE(tmp);
             for (int _i = oparg; --_i >= 0;) {
                 tmp = args[_i];
@@ -5767,7 +5746,6 @@
             self_or_null[0] = PyStackRef_NULL;
             PyStackRef_XCLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-2 - oparg] = res;
             stack_pointer += -1 - oparg;
             assert(WITHIN_STACK_BOUNDS());
             break;
@@ -6147,25 +6125,20 @@
             _PyStackRef *args;
             _PyStackRef *self_or_null;
             _PyStackRef *callable;
-            _PyStackRef *func;
-            _PyStackRef *maybe_self;
             _PyStackRef kwnames_out;
             oparg = CURRENT_OPARG();
             kwnames_in = stack_pointer[-1];
             args = &stack_pointer[-1 - oparg];
             self_or_null = &stack_pointer[-2 - oparg];
             callable = &stack_pointer[-3 - oparg];
-            func = &stack_pointer[-3 - oparg];
-            maybe_self = &stack_pointer[-2 - oparg];
-            args = &stack_pointer[-1 - oparg];
             (void)args;
             if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                 PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                 PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                 _PyStackRef temp = callable[0];
-                func[0] = PyStackRef_FromPyObjectNew(method);
+                callable[0] = PyStackRef_FromPyObjectNew(method);
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 PyStackRef_CLOSE(temp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -6668,13 +6641,13 @@
             lhs = res;
             stack_pointer[-2] = lhs;
             PyStackRef_CLOSE(tmp);
+            tmp = rhs;
+            rhs = PyStackRef_NULL;
+            stack_pointer[-1] = rhs;
+            PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            PyStackRef_CLOSE(rhs);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             break;
         }
 

Python/generated_cases.c.h

@@ -70,13 +70,13 @@
                 lhs = res;
                 stack_pointer[-2] = lhs;
                 PyStackRef_CLOSE(tmp);
+                tmp = rhs;
+                rhs = PyStackRef_NULL;
+                stack_pointer[-1] = rhs;
+                PyStackRef_CLOSE(tmp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
                 stack_pointer += -1;
                 assert(WITHIN_STACK_BOUNDS());
-                _PyFrame_SetStackPointer(frame, stack_pointer);
-                PyStackRef_CLOSE(rhs);
-                stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer[-1] = res;
             }
             DISPATCH();
         }
@@ -708,13 +708,13 @@
             list_st = res;
             stack_pointer[-2] = list_st;
             PyStackRef_CLOSE(tmp);
+            tmp = sub_st;
+            sub_st = PyStackRef_NULL;
+            stack_pointer[-1] = sub_st;
+            PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            PyStackRef_CLOSE(sub_st);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             DISPATCH();
         }
 
@@ -834,7 +834,6 @@
             stack_pointer[-1] = tuple_st;
             PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             DISPATCH();
         }
 
@@ -1125,11 +1124,11 @@
                 }
             }
             if (err) {
+                stack_pointer += -oparg;
+                assert(WITHIN_STACK_BOUNDS());
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 Py_DECREF(set_o);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer += -oparg;
-                assert(WITHIN_STACK_BOUNDS());
                 JUMP_TO_LABEL(error);
             }
             set = PyStackRef_FromPyObjectStealMortal(set_o);
@@ -1271,8 +1270,6 @@
             _PyStackRef *callable;
             _PyStackRef *self_or_null;
             _PyStackRef *args;
-            _PyStackRef *func;
-            _PyStackRef *maybe_self;
             _PyStackRef res;
             // _SPECIALIZE_CALL
             {
@@ -1295,18 +1292,15 @@
             /* Skip 2 cache entries */
             // _MAYBE_EXPAND_METHOD
             {
-                args = &stack_pointer[-oparg];
-                func = &stack_pointer[-2 - oparg];
-                maybe_self = &stack_pointer[-1 - oparg];
                 args = &stack_pointer[-oparg];
                 (void)args;
                 if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                     PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                     PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                    maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                    self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                     PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                     _PyStackRef temp = callable[0];
-                    func[0] = PyStackRef_FromPyObjectNew(method);
+                    callable[0] = PyStackRef_FromPyObjectNew(method);
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     PyStackRef_CLOSE(temp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -1315,8 +1309,6 @@
             // _DO_CALL
             {
                 args = &stack_pointer[-oparg];
-                self_or_null = &stack_pointer[-1 - oparg];
-                callable = &stack_pointer[-2 - oparg];
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                 // oparg counts all of the args, but *not* self:
                 int total_args = oparg;
@@ -1458,7 +1450,7 @@
             INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT);
             static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
             _PyStackRef *callable;
-            _PyStackRef *null;
+            _PyStackRef *self_or_null;
             _PyStackRef *args;
             _PyStackRef *init;
             _PyStackRef *self;
@@ -1476,15 +1468,12 @@
             // _CHECK_AND_ALLOCATE_OBJECT
             {
                 args = &stack_pointer[-oparg];
-                null = &stack_pointer[-1 - oparg];
+                self_or_null = &stack_pointer[-1 - oparg];
                 callable = &stack_pointer[-2 - oparg];
-                init = &stack_pointer[-2 - oparg];
-                self = &stack_pointer[-1 - oparg];
-                args = &stack_pointer[-oparg];
                 uint32_t type_version = read_u32(&this_instr[2].cache);
                 (void)args;
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
-                if (!PyStackRef_IsNull(null[0])) {
+                if (!PyStackRef_IsNull(self_or_null[0])) {
                     UPDATE_MISS_STATS(CALL);
                     assert(_PyOpcode_Deopt[opcode] == (CALL));
                     JUMP_TO_PREDICTED(CALL);
@@ -1518,18 +1507,17 @@
                 if (self_o == NULL) {
                     JUMP_TO_LABEL(error);
                 }
-                self[0] = PyStackRef_FromPyObjectSteal(self_o);
+                self_or_null[0] = PyStackRef_FromPyObjectSteal(self_o);
                 _PyStackRef temp = callable[0];
-                init[0] = PyStackRef_FromPyObjectNew(init_func);
+                callable[0] = PyStackRef_FromPyObjectNew(init_func);
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 PyStackRef_CLOSE(temp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
             }
             // _CREATE_INIT_FRAME
             {
-                args = &stack_pointer[-oparg];
+                self = self_or_null;
-                self = &stack_pointer[-1 - oparg];
+                init = callable;
-                init = &stack_pointer[-2 - oparg];
                 _PyFrame_SetStackPointer(frame, stack_pointer);
                 _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked(
                     tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame);
@@ -2547,7 +2535,7 @@
             next_instr += 4;
             INSTRUCTION_STATS(CALL_ISINSTANCE);
             static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size");
-            _PyStackRef *callable;
+            _PyStackRef callable;
             _PyStackRef *self_or_null;
             _PyStackRef *args;
             _PyStackRef res;
@@ -2555,9 +2543,9 @@
             /* Skip 2 cache entries */
             args = &stack_pointer[-oparg];
             self_or_null = &stack_pointer[-1 - oparg];
-            callable = &stack_pointer[-2 - oparg];
+            callable = stack_pointer[-2 - oparg];
             /* isinstance(o, o2) */
-            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
+            PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable);
             int total_args = oparg;
             _PyStackRef *arguments = args;
             if (!PyStackRef_IsNull(self_or_null[0])) {
@@ -2587,8 +2575,9 @@
             res = retval ? PyStackRef_True : PyStackRef_False;
             assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL));
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            _PyStackRef tmp = callable[0];
+            _PyStackRef tmp = callable;
-            callable[0] = res;
+            callable = res;
+            stack_pointer[-2 - oparg] = callable;
             PyStackRef_CLOSE(tmp);
             for (int _i = oparg; --_i >= 0;) {
                 tmp = args[_i];
@@ -2599,7 +2588,6 @@
             self_or_null[0] = PyStackRef_NULL;
             PyStackRef_XCLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-2 - oparg] = res;
             stack_pointer += -1 - oparg;
             assert(WITHIN_STACK_BOUNDS());
             DISPATCH();
@@ -2622,8 +2610,6 @@
             _PyStackRef *args;
             _PyStackRef kwnames;
             _PyStackRef kwnames_in;
-            _PyStackRef *func;
-            _PyStackRef *maybe_self;
             _PyStackRef kwnames_out;
             _PyStackRef res;
             // _SPECIALIZE_CALL_KW
@@ -2649,17 +2635,14 @@
             {
                 kwnames_in = stack_pointer[-1];
                 args = &stack_pointer[-1 - oparg];
-                func = &stack_pointer[-3 - oparg];
-                maybe_self = &stack_pointer[-2 - oparg];
-                args = &stack_pointer[-1 - oparg];
                 (void)args;
                 if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                     PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                     PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                    maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                    self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                     PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                     _PyStackRef temp = callable[0];
-                    func[0] = PyStackRef_FromPyObjectNew(method);
+                    callable[0] = PyStackRef_FromPyObjectNew(method);
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     PyStackRef_CLOSE(temp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -2670,8 +2653,6 @@
             {
                 kwnames = kwnames_out;
                 args = &stack_pointer[-1 - oparg];
-                self_or_null = &stack_pointer[-2 - oparg];
-                callable = &stack_pointer[-3 - oparg];
                 PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                 PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames);
                 // oparg counts all of the args, but *not* self:
@@ -2696,14 +2677,12 @@
                         arguments, positional_args, kwnames_o, frame
                     );
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer += -1;
+                    stack_pointer += -3 - oparg;
                     assert(WITHIN_STACK_BOUNDS());
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     PyStackRef_CLOSE(kwnames);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
                     // Sync stack explicitly since we leave using DISPATCH_INLINED().
-                    stack_pointer += -2 - oparg;
-                    assert(WITHIN_STACK_BOUNDS());
                     // The frame has stolen all the arguments from the stack,
                     // so there is no need to clean them up.
                     if (new_frame == NULL) {
@@ -3958,6 +3937,7 @@
             // _INIT_CALL_PY_EXACT_ARGS
             {
                 args = &stack_pointer[-oparg];
+                self_or_null = &stack_pointer[-1 - oparg];
                 int has_self = !PyStackRef_IsNull(self_or_null[0]);
                 STAT_INC(CALL, hit);
                 new_frame = _PyFrame_PushUnchecked(tstate, callable[0], oparg + has_self, frame);
@@ -6027,10 +6007,8 @@
                     stack_pointer[-1] = iterable;
                     PyStackRef_CLOSE(tmp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer[-1] = iter;
                 }
             }
-            stack_pointer[-1] = iter;
             DISPATCH();
         }
 
@@ -6121,17 +6099,14 @@
                 args = &stack_pointer[-oparg];
                 self_or_null = &stack_pointer[-1 - oparg];
                 callable = &stack_pointer[-2 - oparg];
-                func = &stack_pointer[-2 - oparg];
-                maybe_self = &stack_pointer[-1 - oparg];
-                args = &stack_pointer[-oparg];
                 (void)args;
                 if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                     PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                     PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                    maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                    self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                     PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                     _PyStackRef temp = callable[0];
-                    func[0] = PyStackRef_FromPyObjectNew(method);
+                    callable[0] = PyStackRef_FromPyObjectNew(method);
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     PyStackRef_CLOSE(temp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -6139,9 +6114,8 @@
             }
             // _MONITOR_CALL
             {
-                args = &stack_pointer[-oparg];
+                maybe_self = self_or_null;
-                maybe_self = &stack_pointer[-1 - oparg];
+                func = callable;
-                func = &stack_pointer[-2 - oparg];
                 int is_meth = !PyStackRef_IsNull(maybe_self[0]);
                 PyObject *function = PyStackRef_AsPyObjectBorrow(func[0]);
                 PyObject *arg0;
@@ -6506,8 +6480,6 @@
             _PyStackRef *self_or_null;
             _PyStackRef *args;
             _PyStackRef kwnames_in;
-            _PyStackRef *func;
-            _PyStackRef *maybe_self;
             _PyStackRef kwnames_out;
             _PyStackRef kwnames;
             _PyStackRef res;
@@ -6519,17 +6491,14 @@
                 args = &stack_pointer[-1 - oparg];
                 self_or_null = &stack_pointer[-2 - oparg];
                 callable = &stack_pointer[-3 - oparg];
-                func = &stack_pointer[-3 - oparg];
-                maybe_self = &stack_pointer[-2 - oparg];
-                args = &stack_pointer[-1 - oparg];
                 (void)args;
                 if (PyStackRef_TYPE(callable[0]) == &PyMethod_Type && PyStackRef_IsNull(self_or_null[0])) {
                     PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable[0]);
                     PyObject *self = ((PyMethodObject *)callable_o)->im_self;
-                    maybe_self[0] = PyStackRef_FromPyObjectNew(self);
+                    self_or_null[0] = PyStackRef_FromPyObjectNew(self);
                     PyObject *method = ((PyMethodObject *)callable_o)->im_func;
                     _PyStackRef temp = callable[0];
-                    func[0] = PyStackRef_FromPyObjectNew(method);
+                    callable[0] = PyStackRef_FromPyObjectNew(method);
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     PyStackRef_CLOSE(temp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
@@ -6538,9 +6507,6 @@
             }
             // _MONITOR_CALL_KW
             {
-                args = &stack_pointer[-1 - oparg];
-                self_or_null = &stack_pointer[-2 - oparg];
-                callable = &stack_pointer[-3 - oparg];
                 int is_meth = !PyStackRef_IsNull(self_or_null[0]);
                 PyObject *arg;
                 if (is_meth) {
@@ -6591,14 +6557,12 @@
                         arguments, positional_args, kwnames_o, frame
                     );
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer += -1;
+                    stack_pointer += -3 - oparg;
                     assert(WITHIN_STACK_BOUNDS());
                     _PyFrame_SetStackPointer(frame, stack_pointer);
                     PyStackRef_CLOSE(kwnames);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
                     // Sync stack explicitly since we leave using DISPATCH_INLINED().
-                    stack_pointer += -2 - oparg;
-                    assert(WITHIN_STACK_BOUNDS());
                     // The frame has stolen all the arguments from the stack,
                     // so there is no need to clean them up.
                     if (new_frame == NULL) {
@@ -7873,7 +7837,6 @@
                 stack_pointer[-1] = owner;
                 PyStackRef_CLOSE(tmp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer[-1] = attr;
             }
             // _PUSH_NULL_CONDITIONAL
             {
@@ -7942,7 +7905,6 @@
                 stack_pointer[-1] = owner;
                 PyStackRef_CLOSE(tmp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer[-1] = attr;
             }
             // _PUSH_NULL_CONDITIONAL
             {
@@ -8603,7 +8565,6 @@
                 stack_pointer[-1] = owner;
                 PyStackRef_CLOSE(tmp);
                 stack_pointer = _PyFrame_GetStackPointer(frame);
-                stack_pointer[-1] = attr;
             }
             /* Skip 5 cache entries */
             // _PUSH_NULL_CONDITIONAL
@@ -9655,13 +9616,12 @@
             global_super_st = self_or_null;
             stack_pointer[-2] = global_super_st;
             PyStackRef_CLOSE(tmp);
+            tmp = class_st;
+            class_st = PyStackRef_NULL;
+            stack_pointer[-1] = class_st;
+            PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += -1;
+            stack_pointer += -2;
-            assert(WITHIN_STACK_BOUNDS());
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            PyStackRef_CLOSE(class_st);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
             attr = PyStackRef_FromPyObjectSteal(attr_o);
             stack_pointer[0] = attr;
@@ -9992,7 +9952,6 @@
                     stack_pointer[-1] = value;
                     PyStackRef_CLOSE(tmp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer[-1] = b;
                 }
             }
             // _POP_JUMP_IF_TRUE
@@ -10036,7 +9995,6 @@
                     stack_pointer[-1] = value;
                     PyStackRef_CLOSE(tmp);
                     stack_pointer = _PyFrame_GetStackPointer(frame);
-                    stack_pointer[-1] = b;
                 }
             }
             // _POP_JUMP_IF_FALSE
@@ -11611,7 +11569,6 @@
             stack_pointer[-1] = value;
             PyStackRef_CLOSE(tmp);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer[-1] = res;
             DISPATCH();
         }
 
@@ -11880,13 +11837,11 @@
                 *values++ = PyStackRef_FromPyObjectNew(items[i]);
             }
             UNLOCK_OBJECT(seq_o);
-            stack_pointer += -1;
+            stack_pointer += -1 + oparg;
             assert(WITHIN_STACK_BOUNDS());
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(seq);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += oparg;
-            assert(WITHIN_STACK_BOUNDS());
             DISPATCH();
         }
 
@@ -11922,13 +11877,11 @@
             for (int i = oparg; --i >= 0; ) {
                 *values++ = PyStackRef_FromPyObjectNew(items[i]);
             }
-            stack_pointer += -1;
+            stack_pointer += -1 + oparg;
             assert(WITHIN_STACK_BOUNDS());
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(seq);
             stack_pointer = _PyFrame_GetStackPointer(frame);
-            stack_pointer += oparg;
-            assert(WITHIN_STACK_BOUNDS());
             DISPATCH();
         }
 
@@ -11970,6 +11923,7 @@
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(seq);
             stack_pointer = _PyFrame_GetStackPointer(frame);
+            stack_pointer[-1] = val0;
             DISPATCH();
         }
 

Python/optimizer_bytecodes.c

@@ -717,6 +717,8 @@ dummy_func(void) {
     }
 
     op(_RETURN_VALUE, (retval -- res)) {
+        JitOptSymbol *temp = retval;
+        DEAD(retval);
         SAVE_STACK();
         ctx->frame->stack_pointer = stack_pointer;
         frame_pop(ctx);
@@ -736,7 +738,7 @@ dummy_func(void) {
             ctx->done = true;
         }
         RELOAD_STACK();
-        res = retval;
+        res = temp;
     }
 
     op(_RETURN_GENERATOR, ( -- res)) {

Python/optimizer_cases.c.h

@@ -276,16 +276,14 @@
             {
                 assert(PyLong_CheckExact(sym_get_const(ctx, left)));
                 assert(PyLong_CheckExact(sym_get_const(ctx, right)));
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
                 PyObject *temp = _PyLong_Multiply((PyLongObject *)sym_get_const(ctx, left),
                     (PyLongObject *)sym_get_const(ctx, right));
                 if (temp == NULL) {
                     goto error;
                 }
                 res = sym_new_const(ctx, temp);
-                stack_pointer[0] = res;
+                stack_pointer[-2] = res;
-                stack_pointer += 1;
+                stack_pointer += -1;
                 assert(WITHIN_STACK_BOUNDS());
                 Py_DECREF(temp);
                 // TODO gh-115506:
@@ -311,16 +309,14 @@
             {
                 assert(PyLong_CheckExact(sym_get_const(ctx, left)));
                 assert(PyLong_CheckExact(sym_get_const(ctx, right)));
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
                 PyObject *temp = _PyLong_Add((PyLongObject *)sym_get_const(ctx, left),
                     (PyLongObject *)sym_get_const(ctx, right));
                 if (temp == NULL) {
                     goto error;
                 }
                 res = sym_new_const(ctx, temp);
-                stack_pointer[0] = res;
+                stack_pointer[-2] = res;
-                stack_pointer += 1;
+                stack_pointer += -1;
                 assert(WITHIN_STACK_BOUNDS());
                 Py_DECREF(temp);
                 // TODO gh-115506:
@@ -346,16 +342,14 @@
             {
                 assert(PyLong_CheckExact(sym_get_const(ctx, left)));
                 assert(PyLong_CheckExact(sym_get_const(ctx, right)));
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
                 PyObject *temp = _PyLong_Subtract((PyLongObject *)sym_get_const(ctx, left),
                     (PyLongObject *)sym_get_const(ctx, right));
                 if (temp == NULL) {
                     goto error;
                 }
                 res = sym_new_const(ctx, temp);
-                stack_pointer[0] = res;
+                stack_pointer[-2] = res;
-                stack_pointer += 1;
+                stack_pointer += -1;
                 assert(WITHIN_STACK_BOUNDS());
                 Py_DECREF(temp);
                 // TODO gh-115506:
@@ -557,17 +551,15 @@
                     goto error;
                 }
                 res = sym_new_const(ctx, temp);
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
                 Py_DECREF(temp);
             }
             else {
                 res = sym_new_type(ctx, &PyUnicode_Type);
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
             }
             // _STORE_FAST:
             GETLOCAL(this_instr->operand0) = res;
+            stack_pointer += -2;
+            assert(WITHIN_STACK_BOUNDS());
             break;
         }
 
@@ -710,6 +702,7 @@
             JitOptSymbol *retval;
             JitOptSymbol *res;
             retval = stack_pointer[-1];
+            JitOptSymbol *temp = retval;
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
             ctx->frame->stack_pointer = stack_pointer;
@@ -727,7 +720,7 @@
                 // might be impossible, but bailing is still safe
                 ctx->done = true;
             }
-            res = retval;
+            res = temp;
             stack_pointer[0] = res;
             stack_pointer += 1;
             assert(WITHIN_STACK_BOUNDS());
@@ -1259,11 +1252,7 @@
                 res = sym_new_type(ctx, &PyBool_Type);
             }
             else {
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
                 res = _Py_uop_sym_new_not_null(ctx);
-                stack_pointer += 2;
-                assert(WITHIN_STACK_BOUNDS());
             }
             stack_pointer[-2] = res;
             stack_pointer += -1;
@@ -1290,8 +1279,6 @@
             {
                 assert(PyLong_CheckExact(sym_get_const(ctx, left)));
                 assert(PyLong_CheckExact(sym_get_const(ctx, right)));
-                stack_pointer += -2;
-                assert(WITHIN_STACK_BOUNDS());
                 PyObject *tmp = PyObject_RichCompare(sym_get_const(ctx, left),
                     sym_get_const(ctx, right),
                     oparg >> 5);
@@ -1302,8 +1289,8 @@
                 assert(_Py_IsImmortal(tmp));
                 REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)tmp);
                 res = sym_new_const(ctx, tmp);
-                stack_pointer[0] = res;
+                stack_pointer[-2] = res;
-                stack_pointer += 1;
+                stack_pointer += -1;
                 assert(WITHIN_STACK_BOUNDS());
                 Py_DECREF(tmp);
             }
@@ -1673,16 +1660,14 @@
             _Py_UOpsAbstractFrame *new_frame;
             PyCodeObject *co = NULL;
             assert((this_instr + 2)->opcode == _PUSH_FRAME);
-            stack_pointer += -2 - oparg;
-            assert(WITHIN_STACK_BOUNDS());
             co = get_code_with_logging((this_instr + 2));
             if (co == NULL) {
                 ctx->done = true;
                 break;
             }
             new_frame = frame_new(ctx, co, 0, NULL, 0);
-            stack_pointer[0] = (JitOptSymbol *)new_frame;
+            stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame;
-            stack_pointer += 1;
+            stack_pointer += -1 - oparg;
             assert(WITHIN_STACK_BOUNDS());
             break;
         }
@@ -1787,8 +1772,6 @@
             int argcount = oparg;
             PyCodeObject *co = NULL;
             assert((this_instr + 2)->opcode == _PUSH_FRAME);
-            stack_pointer += -2 - oparg;
-            assert(WITHIN_STACK_BOUNDS());
             co = get_code_with_logging((this_instr + 2));
             if (co == NULL) {
                 ctx->done = true;
@@ -1806,8 +1789,8 @@
             } else {
                 new_frame = frame_new(ctx, co, 0, NULL, 0);
             }
-            stack_pointer[0] = (JitOptSymbol *)new_frame;
+            stack_pointer[-2 - oparg] = (JitOptSymbol *)new_frame;
-            stack_pointer += 1;
+            stack_pointer += -1 - oparg;
             assert(WITHIN_STACK_BOUNDS());
             break;
         }
@@ -2008,18 +1991,7 @@
         /* _MONITOR_CALL_KW is not a viable micro-op for tier 2 */
 
         case _MAYBE_EXPAND_METHOD_KW: {
-            JitOptSymbol **func;
-            JitOptSymbol **maybe_self;
-            JitOptSymbol **args;
             JitOptSymbol *kwnames_out;
-            func = &stack_pointer[-3 - oparg];
-            maybe_self = &stack_pointer[-2 - oparg];
-            args = &stack_pointer[-1 - oparg];
-            func[0] = sym_new_not_null(ctx);
-            maybe_self[0] = sym_new_not_null(ctx);
-            for (int _i = oparg; --_i >= 0;) {
-                args[_i] = sym_new_not_null(ctx);
-            }
             kwnames_out = sym_new_not_null(ctx);
             stack_pointer[-1] = kwnames_out;
             break;
@@ -2111,6 +2083,7 @@
                 // might be impossible, but bailing is still safe
                 ctx->done = true;
             }
+            stack_pointer[-1] = res;
             break;
         }
 
@@ -2269,11 +2242,7 @@
             if (sym_is_const(ctx, flag)) {
                 PyObject *value = sym_get_const(ctx, flag);
                 assert(value != NULL);
-                stack_pointer += -1;
-                assert(WITHIN_STACK_BOUNDS());
                 eliminate_pop_guard(this_instr, value != Py_True);
-                stack_pointer += 1;
-                assert(WITHIN_STACK_BOUNDS());
             }
             sym_set_const(flag, Py_True);
             stack_pointer += -1;
@@ -2287,11 +2256,7 @@
             if (sym_is_const(ctx, flag)) {
                 PyObject *value = sym_get_const(ctx, flag);
                 assert(value != NULL);
-                stack_pointer += -1;
-                assert(WITHIN_STACK_BOUNDS());
                 eliminate_pop_guard(this_instr, value != Py_False);
-                stack_pointer += 1;
-                assert(WITHIN_STACK_BOUNDS());
             }
             sym_set_const(flag, Py_False);
             stack_pointer += -1;
@@ -2305,23 +2270,17 @@
             if (sym_is_const(ctx, flag)) {
                 PyObject *value = sym_get_const(ctx, flag);
                 assert(value != NULL);
-                stack_pointer += -1;
-                assert(WITHIN_STACK_BOUNDS());
                 eliminate_pop_guard(this_instr, !Py_IsNone(value));
             }
             else {
                 if (sym_has_type(flag)) {
                     assert(!sym_matches_type(flag, &_PyNone_Type));
-                    stack_pointer += -1;
-                    assert(WITHIN_STACK_BOUNDS());
                     eliminate_pop_guard(this_instr, true);
-                    stack_pointer += 1;
-                    assert(WITHIN_STACK_BOUNDS());
                 }
-                stack_pointer += -1;
-                assert(WITHIN_STACK_BOUNDS());
             }
             sym_set_const(flag, Py_None);
+            stack_pointer += -1;
+            assert(WITHIN_STACK_BOUNDS());
             break;
         }
 
@@ -2331,22 +2290,16 @@
             if (sym_is_const(ctx, flag)) {
                 PyObject *value = sym_get_const(ctx, flag);
                 assert(value != NULL);
-                stack_pointer += -1;
-                assert(WITHIN_STACK_BOUNDS());
                 eliminate_pop_guard(this_instr, Py_IsNone(value));
             }
             else {
                 if (sym_has_type(flag)) {
                     assert(!sym_matches_type(flag, &_PyNone_Type));
-                    stack_pointer += -1;
-                    assert(WITHIN_STACK_BOUNDS());
                     eliminate_pop_guard(this_instr, false);
-                    stack_pointer += 1;
+                }
-                    assert(WITHIN_STACK_BOUNDS());
             }
             stack_pointer += -1;
             assert(WITHIN_STACK_BOUNDS());
-            }
             break;
         }
 

Tools/cases_generator/generators_common.py

@@ -248,6 +248,7 @@ def decref_inputs(
         except Exception as ex:
             ex.args = (ex.args[0] + str(tkn),)
             raise
+        self._print_storage(storage)
         return True
 
     def kill_inputs(
@@ -303,7 +304,7 @@ def stackref_kill(
                         f"'{live}' is still live", name)
                 var.kill()
                 break
-            if var.defined:
+            if var.in_local:
                 live = var.name
         return True
 
@@ -402,6 +403,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None:
         self.out.emit(")")
 
     def emit_save(self, storage: Storage) -> None:
+        storage.flush(self.out)
         storage.save(self.out)
         self._print_storage(storage)
 
@@ -498,6 +500,9 @@ def _emit_if(
                 else:
                     if PRINT_STACKS:
                         self.emit("/* Merge */\n")
+                        self.out.emit(if_storage.as_comment())
+                        self.out.emit("\n")
+                        self.out.emit(else_storage.as_comment())
                     else_storage.merge(if_storage, self.out)
                     storage = else_storage
                     self._print_storage(storage)
@@ -513,7 +518,7 @@ def _emit_if(
                     reachable = True
         except StackError as ex:
             self._print_storage(if_storage)
-            raise analysis_error(ex.args[0], rbrace) # from None
+            raise analysis_error(ex.args[0], rbrace) from None
         return reachable, rbrace, storage
 
     def _emit_block(
@@ -577,16 +582,16 @@ def _emit_block(
                         if tkn in local_stores:
                             for var in storage.inputs:
                                 if var.name == tkn.text:
-                                    if var.defined or var.in_memory:
+                                    if var.in_local or var.in_memory():
                                         msg = f"Cannot assign to already defined input variable '{tkn.text}'"
                                         raise analysis_error(msg, tkn)
-                                    var.defined = True
+                                    var.in_local = True
-                                    var.in_memory = False
+                                    var.memory_offset = None
                                     break
                             for var in storage.outputs:
                                 if var.name == tkn.text:
-                                    var.defined = True
+                                    var.in_local = True
-                                    var.in_memory = False
+                                    var.memory_offset = None
                                     break
                         if tkn.text.startswith("DISPATCH"):
                             self._print_storage(storage)

Tools/cases_generator/optimizer_generator.py

@@ -84,7 +84,7 @@ def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None:
         local = Local.undefined(var)
         stack.push(local)
         if var.name != "unused" and not var.peek:
-            local.defined = True
+            local.in_local = True
             if var.is_array():
                 if var.size == "1":
                     out.emit(f"{var.name}[0] = sym_new_not_null(ctx);\n")
@@ -123,7 +123,7 @@ def write_uop(
     try:
         out.start_line()
         if override:
-            code_list, storage = Storage.for_uop(stack, prototype)
+            code_list, storage = Storage.for_uop(stack, prototype, check_liveness=False)
             for code in code_list:
                 out.emit(code)
         if debug:
@@ -145,7 +145,7 @@ def write_uop(
             emitter = OptimizerEmitter(out, {})
             # No reference management of inputs needed.
             for var in storage.inputs:  # type: ignore[possibly-undefined]
-                var.defined = False
+                var.in_local = False
             storage = emitter.emit_tokens(override, storage, None)
             out.start_line()
             storage.flush(out)

Tools/cases_generator/stack.py

@@ -29,74 +29,6 @@ def var_size(var: StackItem) -> str:
         return "1"
 
 
-@dataclass
-class Local:
-    item: StackItem
-    cached: bool
-    in_memory: bool
-    defined: bool
-
-    def __repr__(self) -> str:
-        return f"Local('{self.item.name}', mem={self.in_memory}, defined={self.defined}, array={self.is_array()})"
-
-    def compact_str(self) -> str:
-        mtag = "M" if self.in_memory else ""
-        dtag = "D" if self.defined else ""
-        atag = "A" if self.is_array() else ""
-        return f"'{self.item.name}'{mtag}{dtag}{atag}"
-
-    @staticmethod
-    def unused(defn: StackItem) -> "Local":
-        return Local(defn, False, defn.is_array(), False)
-
-    @staticmethod
-    def undefined(defn: StackItem) -> "Local":
-        array = defn.is_array()
-        return Local(defn, not array, array, False)
-
-    @staticmethod
-    def redefinition(var: StackItem, prev: "Local") -> "Local":
-        assert var.is_array() == prev.is_array()
-        return Local(var, prev.cached, prev.in_memory, True)
-
-    @staticmethod
-    def from_memory(defn: StackItem) -> "Local":
-        return Local(defn, True, True, True)
-
-    def kill(self) -> None:
-        self.defined = False
-        self.in_memory = False
-
-    def copy(self) -> "Local":
-        return Local(
-            self.item,
-            self.cached,
-            self.in_memory,
-            self.defined
-        )
-
-    @property
-    def size(self) -> str:
-        return self.item.size
-
-    @property
-    def name(self) -> str:
-        return self.item.name
-
-    def is_array(self) -> bool:
-        return self.item.is_array()
-
-    def __eq__(self, other: object) -> bool:
-        if not isinstance(other, Local):
-            return NotImplemented
-        return (
-            self.item is other.item
-            and self.cached is other.cached
-            and self.in_memory is other.in_memory
-            and self.defined is other.defined
-        )
-
-
 @dataclass
 class StackOffset:
     "The stack offset of the virtual base of the stack from the physical stack pointer"
@@ -206,6 +138,73 @@ def __eq__(self, other: object) -> bool:
         return self.to_c() == other.to_c()
 
 
+@dataclass
+class Local:
+    item: StackItem
+    memory_offset: StackOffset | None
+    in_local: bool
+
+    def __repr__(self) -> str:
+        return f"Local('{self.item.name}', mem={self.memory_offset}, local={self.in_local}, array={self.is_array()})"
+
+    #def compact_str(self) -> str:
+        #mtag = "M" if self.memory_offset else ""
+        #dtag = "D" if self.in_local else ""
+        #atag = "A" if self.is_array() else ""
+        #return f"'{self.item.name}'{mtag}{dtag}{atag}"
+
+    compact_str = __repr__
+
+    @staticmethod
+    def unused(defn: StackItem, offset: StackOffset) -> "Local":
+        return Local(defn, offset, False)
+
+    @staticmethod
+    def undefined(defn: StackItem) -> "Local":
+        return Local(defn, None, False)
+
+    @staticmethod
+    def from_memory(defn: StackItem, offset: StackOffset) -> "Local":
+        return Local(defn, offset, True)
+
+    def kill(self) -> None:
+        self.in_local = False
+        self.memory_offset = None
+
+    def in_memory(self) -> bool:
+        return self.memory_offset is not None or self.is_array()
+
+    def is_dead(self) -> bool:
+        return not self.in_local and self.memory_offset is None
+
+    def copy(self) -> "Local":
+        return Local(
+            self.item,
+            self.memory_offset,
+            self.in_local
+        )
+
+    @property
+    def size(self) -> str:
+        return self.item.size
+
+    @property
+    def name(self) -> str:
+        return self.item.name
+
+    def is_array(self) -> bool:
+        return self.item.is_array()
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Local):
+            return NotImplemented
+        return (
+            self.item is other.item
+            and self.memory_offset == other.memory_offset
+            and self.in_local == other.in_local
+        )
+
+
 class StackError(Exception):
     pass
 
@@ -221,6 +220,15 @@ def __init__(self, extract_bits: bool=True, cast_type: str = "uintptr_t") -> Non
         self.extract_bits = extract_bits
         self.cast_type = cast_type
 
+    def drop(self, var: StackItem, check_liveness: bool) -> None:
+        self.top_offset.pop(var)
+        if self.variables:
+            popped = self.variables.pop()
+            if popped.is_dead() or not var.used:
+                return
+        if check_liveness:
+            raise StackError(f"Dropping live value '{var.name}'")
+
     def pop(self, var: StackItem) -> tuple[str, Local]:
         self.top_offset.pop(var)
         indirect = "&" if var.is_array() else ""
@@ -245,28 +253,31 @@ def pop(self, var: StackItem) -> tuple[str, Local]:
             if not var.used:
                 return "", popped
             self.defined.add(var.name)
-            if popped.defined:
+            if popped.name != var.name:
-                if popped.name == var.name:
+                rename = f"{var.name} = {popped.name};\n"
-                    return "", popped
+                popped.item = var
-                else:
-                    defn = f"{var.name} = {popped.name};\n"
             else:
+                rename = ""
+            if not popped.in_local:
+                assert popped.memory_offset is not None
                 if var.is_array():
                     defn = f"{var.name} = &stack_pointer[{self.top_offset.to_c()}];\n"
                 else:
                     defn = f"{var.name} = stack_pointer[{self.top_offset.to_c()}];\n"
-                    popped.in_memory = True
+                    popped.in_local = True
-            return defn, Local.redefinition(var, popped)
+            else:
+                defn = rename
+            return defn, popped
 
         self.base_offset.pop(var)
         if var.name in UNUSED or not var.used:
-            return "", Local.unused(var)
+            return "", Local.unused(var, self.base_offset)
         self.defined.add(var.name)
         cast = f"({var.type})" if (not indirect and var.type) else ""
         bits = ".bits" if cast and self.extract_bits else ""
         assign = f"{var.name} = {cast}{indirect}stack_pointer[{self.base_offset.to_c()}]{bits};"
         assign = f"{assign}\n"
-        return assign, Local.from_memory(var)
+        return assign, Local.from_memory(var, self.base_offset.copy())
 
     def push(self, var: Local) -> None:
         assert(var not in self.variables)
@@ -298,11 +309,12 @@ def flush(self, out: CWriter) -> None:
         var_offset = self.base_offset.copy()
         for var in self.variables:
             if (
-                var.defined and
+                var.in_local and
-                not var.in_memory
+                not var.memory_offset and
+                not var.is_array()
             ):
                 Stack._do_emit(out, var.item, var_offset, self.cast_type, self.extract_bits)
-                var.in_memory = True
+                var.memory_offset = var_offset.copy()
             var_offset.push(var.item)
         number = self.top_offset.to_c()
         self._adjust_stack_pointer(out, number)
@@ -358,9 +370,14 @@ def merge(self, other: "Stack", out: CWriter) -> None:
         for self_var, other_var in zip(self.variables, other.variables):
             if self_var.name != other_var.name:
                 raise StackError(f"Mismatched variables on stack: {self_var.name} and {other_var.name}")
-            self_var.defined = self_var.defined and other_var.defined
+            self_var.in_local = self_var.in_local and other_var.in_local
-            self_var.in_memory = self_var.in_memory and other_var.in_memory
+            if other_var.memory_offset is None:
+                self_var.memory_offset = None
         self.align(other, out)
+        for self_var, other_var in zip(self.variables, other.variables):
+            if self_var.memory_offset is not None:
+                if self_var.memory_offset != other_var.memory_offset:
+                    raise StackError(f"Mismatched stack depths for {self_var.name}: {self_var.memory_offset} and {other_var.memory_offset}")
 
 
 def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]:
@@ -383,7 +400,7 @@ def apply_stack_effect(stack: Stack, effect: StackEffect) -> None:
         if var.name in locals:
             local = locals[var.name]
         else:
-            local = Local.unused(var)
+            local = Local.unused(var, stack.base_offset)
         stack.push(local)
 
 
@@ -400,13 +417,13 @@ class Storage:
     stack: Stack
     inputs: list[Local]
     outputs: list[Local]
-    peeks: list[Local]
+    check_liveness: bool
     spilled: int = 0
 
     @staticmethod
     def needs_defining(var: Local) -> bool:
         return (
-            not var.defined and
+            not var.in_local and
             not var.is_array() and
             var.name != "unused"
         )
@@ -414,24 +431,21 @@ def needs_defining(var: Local) -> bool:
     @staticmethod
     def is_live(var: Local) -> bool:
         return (
-            var.defined and
+            var.name != "unused" and
-            var.name != "unused"
+            (
+                var.in_local or
+                var.memory_offset is not None
+            )
         )
-
-    def first_input_not_cleared(self) -> str:
-        for input in self.inputs:
-            if input.defined:
-                return input.name
-        return ""
 
     def clear_inputs(self, reason:str) -> None:
         while self.inputs:
             tos = self.inputs.pop()
-            if self.is_live(tos) and not tos.is_array():
+            if self.is_live(tos) and self.check_liveness:
                 raise StackError(
                     f"Input '{tos.name}' is still live {reason}"
                 )
-            self.stack.pop(tos.item)
+            self.stack.drop(tos.item, self.check_liveness)
 
     def clear_dead_inputs(self) -> None:
         live = ""
@@ -441,9 +455,9 @@ def clear_dead_inputs(self) -> None:
                 live = tos.name
                 break
             self.inputs.pop()
-            self.stack.pop(tos.item)
+            self.stack.drop(tos.item, self.check_liveness)
         for var in self.inputs:
-            if not var.defined and not var.is_array() and var.name != "unused":
+            if not self.is_live(var):
                 raise StackError(
                     f"Input '{var.name}' is not live, but '{live}' is"
                 )
@@ -451,14 +465,14 @@ def clear_dead_inputs(self) -> None:
     def _push_defined_outputs(self) -> None:
         defined_output = ""
         for output in self.outputs:
-            if output.defined and not output.in_memory:
+            if output.in_local and not output.memory_offset:
                 defined_output = output.name
         if not defined_output:
             return
         self.clear_inputs(f"when output '{defined_output}' is defined")
         undefined = ""
         for out in self.outputs:
-            if out.defined:
+            if out.in_local:
                 if undefined:
                     f"Locals not defined in stack order. "
                     f"Expected '{undefined}' to be defined before '{out.name}'"
@@ -470,7 +484,7 @@ def _push_defined_outputs(self) -> None:
 
     def locals_cached(self) -> bool:
         for out in self.outputs:
-            if out.defined:
+            if out.in_local:
                 return True
         return False
 
@@ -482,7 +496,6 @@ def flush(self, out: CWriter) -> None:
     def save(self, out: CWriter) -> None:
         assert self.spilled >= 0
         if self.spilled == 0:
-            self.flush(out)
             out.start_line()
             out.emit_spill()
         self.spilled += 1
@@ -506,7 +519,7 @@ def reload(self, out: CWriter) -> None:
             out.emit_reload()
 
     @staticmethod
-    def for_uop(stack: Stack, uop: Uop) -> tuple[list[str], "Storage"]:
+    def for_uop(stack: Stack, uop: Uop, check_liveness: bool = True) -> tuple[list[str], "Storage"]:
         code_list: list[str] = []
         inputs: list[Local] = []
         peeks: list[Local] = []
@@ -532,7 +545,7 @@ def for_uop(stack: Stack, uop: Uop) -> tuple[list[str], "Storage"]:
         for var in inputs:
             stack.push(var)
         outputs = [ Local.undefined(var) for var in uop.stack.outputs if not var.peek ]
-        return code_list, Storage(stack, inputs, outputs, peeks)
+        return code_list, Storage(stack, inputs, outputs, check_liveness)
 
     @staticmethod
     def copy_list(arg: list[Local]) -> list[Local]:
@@ -544,8 +557,8 @@ def copy(self) -> "Storage":
         inputs = [ variables[var.name] for var in self.inputs]
         assert [v.name for v in inputs] == [v.name for v in self.inputs], (inputs, self.inputs)
         return Storage(
-            new_stack, inputs,
+            new_stack, inputs, self.copy_list(self.outputs),
-            self.copy_list(self.outputs), self.copy_list(self.peeks), self.spilled
+            self.check_liveness, self.spilled
         )
 
     def sanity_check(self) -> None:
@@ -567,7 +580,7 @@ def sanity_check(self) -> None:
 
     def is_flushed(self) -> bool:
         for var in self.outputs:
-            if var.defined and not var.in_memory:
+            if var.in_local and not var.memory_offset:
                 return False
         return self.stack.is_flushed()
 
@@ -576,11 +589,11 @@ def merge(self, other: "Storage", out: CWriter) -> None:
         if len(self.inputs) != len(other.inputs):
             self.clear_dead_inputs()
             other.clear_dead_inputs()
-        if len(self.inputs) != len(other.inputs):
+        if len(self.inputs) != len(other.inputs) and self.check_liveness:
             diff = self.inputs[-1] if len(self.inputs) > len(other.inputs) else other.inputs[-1]
             raise StackError(f"Unmergeable inputs. Differing state of '{diff.name}'")
         for var, other_var in zip(self.inputs, other.inputs):
-            if var.defined != other_var.defined:
+            if var.in_local != other_var.in_local:
                 raise StackError(f"'{var.name}' is cleared on some paths, but not all")
         if len(self.outputs) != len(other.outputs):
             self._push_defined_outputs()
@@ -595,7 +608,7 @@ def push_outputs(self) -> None:
         if self.spilled:
             raise StackError(f"Unbalanced stack spills")
         self.clear_inputs("at the end of the micro-op")
-        if self.inputs:
+        if self.inputs and self.check_liveness:
             raise StackError(f"Input variable '{self.inputs[-1].name}' is still live")
         self._push_defined_outputs()
         if self.outputs:
@@ -610,10 +623,10 @@ def as_comment(self) -> str:
         next_line = "\n               "
         inputs = ", ".join([var.compact_str() for var in self.inputs])
         outputs = ", ".join([var.compact_str() for var in self.outputs])
-        peeks = ", ".join([var.name for var in self.peeks])
+        return f"{stack_comment[:-2]}{next_line}inputs: {inputs}{next_line}outputs: {outputs}*/"
-        return f"{stack_comment[:-2]}{next_line}inputs: {inputs}{next_line}outputs: {outputs}{next_line}peeks: {peeks} */"
 
     def close_inputs(self, out: CWriter) -> None:
+
         tmp_defined = False
         def close_named(close: str, name: str, overwrite: str) -> None:
             nonlocal tmp_defined
@@ -623,9 +636,7 @@ def close_named(close: str, name: str, overwrite: str) -> None:
                     tmp_defined = True
                 out.emit(f"tmp = {name};\n")
                 out.emit(f"{name} = {overwrite};\n")
-                if not var.is_array():
+                self.stack.flush(out)
-                    var.in_memory = False
-                    self.flush(out)
                 out.emit(f"{close}(tmp);\n")
             else:
                 out.emit(f"{close}({name});\n")
@@ -635,6 +646,9 @@ def close_variable(var: Local, overwrite: str) -> None:
             close = "PyStackRef_CLOSE"
             if "null" in var.name:
                 close = "PyStackRef_XCLOSE"
+            var.memory_offset = None
+            self.save(out)
+            out.start_line()
             if var.size:
                 if var.size == "1":
                     close_named(close, f"{var.name}[0]", overwrite)
@@ -647,54 +661,58 @@ def close_variable(var: Local, overwrite: str) -> None:
                     out.emit("}\n")
             else:
                 close_named(close, var.name, overwrite)
+            self.reload(out)
 
         self.clear_dead_inputs()
         if not self.inputs:
             return
+        lowest = self.inputs[0]
         output: Local | None = None
         for var in self.outputs:
             if var.is_array():
                 if len(self.inputs) > 1:
-                    raise StackError("Cannot call DECREF_INPUTS with multiple live input(s) and array output")
+                    raise StackError("Cannot call DECREF_INPUTS with array output and more than one input")
-            elif var.defined:
+                output = var
+            elif var.in_local:
                 if output is not None:
                     raise StackError("Cannot call DECREF_INPUTS with more than one live output")
                 output = var
-        self.save_inputs(out)
+        self.stack.flush(out)
         if output is not None:
-            lowest = self.inputs[0]
+            if output.is_array():
-            if lowest.is_array():
+                assert len(self.inputs) == 1
-                try:
+                self.stack.drop(self.inputs[0].item, False)
-                    size = int(lowest.size)
+                self.stack.push(output)
-                except:
+                self.stack.flush(out)
-                    size = -1
+                close_variable(self.inputs[0], "")
-                if size <= 0:
+                self.stack.drop(output.item, self.check_liveness)
-                    raise StackError("Cannot call DECREF_INPUTS with non fixed size array as lowest input on stack")
+                self.inputs = []
-                if size > 1:
+                return
-                    raise StackError("Cannot call DECREF_INPUTS with array size > 1 as lowest input on stack")
+            if var_size(lowest.item) != var_size(output.item):
-                output.defined = False
+                raise StackError("Cannot call DECREF_INPUTS with live output not matching first input size")
+            lowest.in_local = True
             close_variable(lowest, output.name)
+            assert lowest.memory_offset is not None
+        for input in reversed(self.inputs[1:]):
+            close_variable(input, "PyStackRef_NULL")
+        if output is None:
+            close_variable(self.inputs[0], "PyStackRef_NULL")
+        for input in reversed(self.inputs[1:]):
+            input.kill()
+            self.stack.drop(input.item, self.check_liveness)
+        if output is None:
+            self.inputs[0].kill()
+        self.stack.drop(self.inputs[0].item, False)
+        output_in_place = self.outputs and output is self.outputs[0] and lowest.memory_offset is not None
+        if output_in_place:
+            output.memory_offset = lowest.memory_offset.copy()  # type: ignore[union-attr]
         else:
-                lowest.in_memory = False
+            self.stack.flush(out)
-                output.defined = False
-                close_variable(lowest, output.name)
-        to_close = self.inputs[: 0 if output is not None else None: -1]
-        if len(to_close) == 1 and not to_close[0].is_array():
-            self.reload(out)
-            to_close[0].defined = False
-            self.flush(out)
-            self.save_inputs(out)
-            close_variable(to_close[0], "")
-            self.reload(out)
-        else:
-            for var in to_close:
-                assert var.defined or var.is_array()
-                close_variable(var, "PyStackRef_NULL")
-            self.reload(out)
-        for var in self.inputs:
-            var.defined = False
         if output is not None:
-            output.defined = True
+            self.stack.push(output)
-            # MyPy false positive
+        self.inputs = []
-            lowest.defined = False  # type: ignore[possibly-undefined]
+        if output_in_place:
-        self.flush(out)
+            self.stack.flush(out)
+        if output is not None:
+            code, output = self.stack.pop(output.item)
+            out.emit(code)

Tools/cases_generator/tier1_generator.py

@@ -205,7 +205,7 @@ def generate_tier1_labels(
     for name, label in analysis.labels.items():
         emitter.emit(f"LABEL({name})\n")
         emitter.emit("{\n")
-        storage = Storage(Stack(), [], [], [])
+        storage = Storage(Stack(), [], [], False)
         if label.spilled:
             storage.spilled = 1
             emitter.emit("/* STACK SPILLED */\n")

Tools/cases_generator/tier2_generator.py

@@ -157,6 +157,7 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack:
                 emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n")
                 idx += 1
         storage = emitter.emit_tokens(uop, storage, None)
+        storage.flush(emitter.out)
     except StackError as ex:
         raise analysis_error(ex.args[0], uop.body[0]) from None
     return storage.stack
@@ -196,7 +197,6 @@ def generate_tier2(
         stack = write_uop(uop, emitter, stack)
         out.start_line()
         if not uop.properties.always_exits:
-            stack.flush(out)
             out.emit("break;\n")
         out.start_line()
         out.emit("}")