gh-117139: Add _PyTuple_FromStackRefSteal and use it (#121244)

Avoids the extra conversion from stack refs to PyObjects.

Include/internal/pycore_stackref.h

@@ -48,7 +48,7 @@ extern "C" {
    CPython refcounting operations on it!
 */
 
-typedef union {
+typedef union _PyStackRef {
     uintptr_t bits;
 } _PyStackRef;
 

Include/internal/pycore_tuple.h

@@ -21,6 +21,7 @@ extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *);
 #define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item)
 
 extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t);
+PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefSteal(const union _PyStackRef *, Py_ssize_t);
 PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);
 
 typedef struct {

Objects/tupleobject.c

@@ -390,6 +390,27 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n)
     return (PyObject *)tuple;
 }
 
+PyObject *
+_PyTuple_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n)
+{
+    if (n == 0) {
+        return tuple_get_empty();
+    }
+    PyTupleObject *tuple = tuple_alloc(n);
+    if (tuple == NULL) {
+        for (Py_ssize_t i = 0; i < n; i++) {
+            PyStackRef_CLOSE(src[i]);
+        }
+        return NULL;
+    }
+    PyObject **dst = tuple->ob_item;
+    for (Py_ssize_t i = 0; i < n; i++) {
+        dst[i] = PyStackRef_AsPyObjectSteal(src[i]);
+    }
+    _PyObject_GC_TRACK(tuple);
+    return (PyObject *)tuple;
+}
+
 PyObject *
 _PyTuple_FromArraySteal(PyObject *const *src, Py_ssize_t n)
 {

Python/bytecodes.c

@@ -1780,13 +1780,7 @@ dummy_func(
         }
 
         inst(BUILD_TUPLE, (values[oparg] -- tup)) {
-            STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+            PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg);
-            if (CONVERSION_FAILED(values_o)) {
-                DECREF_INPUTS();
-                ERROR_IF(true, error);
-            }
-            PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
-            STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
             ERROR_IF(tup_o == NULL, error);
             tup = PyStackRef_FromPyObjectSteal(tup_o);
         }

Python/ceval.c

@@ -1500,13 +1500,7 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func,
             u = (PyObject *)&_Py_SINGLETON(tuple_empty);
         }
         else {
-            assert(args != NULL);
+            u = _PyTuple_FromStackRefSteal(args + n, argcount - n);
-            STACKREFS_TO_PYOBJECTS((_PyStackRef *)args, argcount, args_o);
-            if (args_o == NULL) {
-                goto fail_pre_positional;
-            }
-            u = _PyTuple_FromArraySteal((args_o + n), argcount - n);
-            STACKREFS_TO_PYOBJECTS_CLEANUP(args_o);
         }
         if (u == NULL) {
             goto fail_post_positional;

Python/executor_cases.c.h

@@ -1816,15 +1816,7 @@
             _PyStackRef tup;
             oparg = CURRENT_OPARG();
             values = &stack_pointer[-oparg];
-            STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+            PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg);
-            if (CONVERSION_FAILED(values_o)) {
-                for (int _i = oparg; --_i >= 0;) {
-                    PyStackRef_CLOSE(values[_i]);
-                }
-                if (true) JUMP_TO_ERROR();
-            }
-            PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
-            STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
             if (tup_o == NULL) JUMP_TO_ERROR();
             tup = PyStackRef_FromPyObjectSteal(tup_o);
             stack_pointer[-oparg] = tup;

Python/generated_cases.c.h

@@ -769,15 +769,7 @@
             _PyStackRef *values;
             _PyStackRef tup;
             values = &stack_pointer[-oparg];
-            STACKREFS_TO_PYOBJECTS(values, oparg, values_o);
+            PyObject *tup_o = _PyTuple_FromStackRefSteal(values, oparg);
-            if (CONVERSION_FAILED(values_o)) {
-                for (int _i = oparg; --_i >= 0;) {
-                    PyStackRef_CLOSE(values[_i]);
-                }
-                if (true) { stack_pointer += -oparg; goto error; }
-            }
-            PyObject *tup_o = _PyTuple_FromArraySteal(values_o, oparg);
-            STACKREFS_TO_PYOBJECTS_CLEANUP(values_o);
             if (tup_o == NULL) { stack_pointer += -oparg; goto error; }
             tup = PyStackRef_FromPyObjectSteal(tup_o);
             stack_pointer[-oparg] = tup;

Tools/cases_generator/analyzer.py

@@ -431,6 +431,7 @@ NON_ESCAPING_FUNCTIONS = (
     "CONVERSION_FAILED",
     "_PyList_FromArraySteal",
     "_PyTuple_FromArraySteal",
+    "_PyTuple_FromStackRefSteal",
 )
 
 ESCAPING_FUNCTIONS = (