bpo-33720: Refactor marshalling/unmarshalling floats. (GH-8071)

Include/pyport.h

@@ -525,7 +525,9 @@ extern "C" {
  * Usage:
  *    int _Py_NO_INLINE x(void) { return 3; }
  */
-#if defined(__GNUC__) || defined(__clang__)
+#if defined(_MSC_VER)
+#  define _Py_NO_INLINE __declspec(noinline)
+#elif defined(__GNUC__) || defined(__clang__)
 #  define _Py_NO_INLINE __attribute__ ((noinline))
 #else
 #  define _Py_NO_INLINE

Objects/call.c

@@ -1271,7 +1271,7 @@ PyObject_CallFunctionObjArgs(PyObject *callable, ...)
 
 /* Issue #29234: Inlining _PyStack_AsTuple() into callers increases their
    stack consumption, Disable inlining to optimize the stack consumption. */
-PyObject* _Py_NO_INLINE
+_Py_NO_INLINE PyObject *
 _PyStack_AsTuple(PyObject *const *stack, Py_ssize_t nargs)
 {
     PyObject *args;

Python/marshal.c

@@ -266,6 +266,32 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
     } while (d != 0);
 }
 
+static void
+w_float_bin(double v, WFILE *p)
+{
+    unsigned char buf[8];
+    if (_PyFloat_Pack8(v, buf, 1) < 0) {
+        p->error = WFERR_UNMARSHALLABLE;
+        return;
+    }
+    w_string((const char *)buf, 8, p);
+}
+
+static void
+w_float_str(double v, WFILE *p)
+{
+    int n;
+    char *buf = PyOS_double_to_string(v, 'g', 17, 0, NULL);
+    if (!buf) {
+        p->error = WFERR_NOMEMORY;
+        return;
+    }
+    n = (int)strlen(buf);
+    w_byte(n, p);
+    w_string(buf, n, p);
+    PyMem_Free(buf);
+}
+
 static int
 w_ref(PyObject *v, char *flag, WFILE *p)
 {
@@ -375,69 +401,24 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
     }
     else if (PyFloat_CheckExact(v)) {
         if (p->version > 1) {
-            unsigned char buf[8];
-            if (_PyFloat_Pack8(PyFloat_AsDouble(v),
-                               buf, 1) < 0) {
-                p->error = WFERR_UNMARSHALLABLE;
-                return;
-            }
             W_TYPE(TYPE_BINARY_FLOAT, p);
-            w_string((char*)buf, 8, p);
+            w_float_bin(PyFloat_AS_DOUBLE(v), p);
         }
         else {
-            char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
-                                              'g', 17, 0, NULL);
-            if (!buf) {
-                p->error = WFERR_NOMEMORY;
-                return;
-            }
-            n = strlen(buf);
             W_TYPE(TYPE_FLOAT, p);
-            w_byte((int)n, p);
+            w_float_str(PyFloat_AS_DOUBLE(v), p);
-            w_string(buf, n, p);
-            PyMem_Free(buf);
         }
     }
     else if (PyComplex_CheckExact(v)) {
         if (p->version > 1) {
-            unsigned char buf[8];
-            if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
-                               buf, 1) < 0) {
-                p->error = WFERR_UNMARSHALLABLE;
-                return;
-            }
             W_TYPE(TYPE_BINARY_COMPLEX, p);
-            w_string((char*)buf, 8, p);
+            w_float_bin(PyComplex_RealAsDouble(v), p);
-            if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
+            w_float_bin(PyComplex_ImagAsDouble(v), p);
-                               buf, 1) < 0) {
-                p->error = WFERR_UNMARSHALLABLE;
-                return;
-            }
-            w_string((char*)buf, 8, p);
         }
         else {
-            char *buf;
             W_TYPE(TYPE_COMPLEX, p);
-            buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
+            w_float_str(PyComplex_RealAsDouble(v), p);
-                                        'g', 17, 0, NULL);
+            w_float_str(PyComplex_ImagAsDouble(v), p);
-            if (!buf) {
-                p->error = WFERR_NOMEMORY;
-                return;
-            }
-            n = strlen(buf);
-            w_byte((int)n, p);
-            w_string(buf, n, p);
-            PyMem_Free(buf);
-            buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
-                                        'g', 17, 0, NULL);
-            if (!buf) {
-                p->error = WFERR_NOMEMORY;
-                return;
-            }
-            n = strlen(buf);
-            w_byte((int)n, p);
-            w_string(buf, n, p);
-            PyMem_Free(buf);
         }
     }
     else if (PyBytes_CheckExact(v)) {
@@ -880,6 +861,38 @@ r_PyLong(RFILE *p)
     return NULL;
 }
 
+static double
+r_float_bin(RFILE *p)
+{
+    const unsigned char *buf = (const unsigned char *) r_string(8, p);
+    if (buf == NULL)
+        return -1;
+    return _PyFloat_Unpack8(buf, 1);
+}
+
+/* Issue #33720: Disable inlining for reducing the C stack consumption
+   on PGO builds. */
+_Py_NO_INLINE static double
+r_float_str(RFILE *p)
+{
+    int n;
+    char buf[256];
+    const char *ptr;
+    n = r_byte(p);
+    if (n == EOF) {
+        PyErr_SetString(PyExc_EOFError,
+            "EOF read where object expected");
+        return -1;
+    }
+    ptr = r_string(n, p);
+    if (ptr == NULL) {
+        return -1;
+    }
+    memcpy(buf, ptr, n);
+    buf[n] = '\0';
+    return PyOS_string_to_double(buf, NULL, NULL);
+}
+
 /* allocate the reflist index for a new object. Return -1 on failure */
 static Py_ssize_t
 r_ref_reserve(int flag, RFILE *p)
@@ -1016,36 +1029,17 @@ r_object(RFILE *p)
 
     case TYPE_FLOAT:
         {
-            char buf[256];
+            double x = r_float_str(p);
-            const char *ptr;
+            if (x == -1.0 && PyErr_Occurred())
-            double dx;
-            n = r_byte(p);
-            if (n == EOF) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
                 break;
-            }
+            retval = PyFloat_FromDouble(x);
-            ptr = r_string(n, p);
-            if (ptr == NULL)
-                break;
-            memcpy(buf, ptr, n);
-            buf[n] = '\0';
-            dx = PyOS_string_to_double(buf, NULL, NULL);
-            if (dx == -1.0 && PyErr_Occurred())
-                break;
-            retval = PyFloat_FromDouble(dx);
             R_REF(retval);
             break;
         }
 
     case TYPE_BINARY_FLOAT:
         {
-            const unsigned char *buf;
+            double x = r_float_bin(p);
-            double x;
-            buf = (const unsigned char *) r_string(8, p);
-            if (buf == NULL)
-                break;
-            x = _PyFloat_Unpack8(buf, 1);
             if (x == -1.0 && PyErr_Occurred())
                 break;
             retval = PyFloat_FromDouble(x);
@@ -1055,35 +1049,11 @@ r_object(RFILE *p)
 
     case TYPE_COMPLEX:
         {
-            char buf[256];
-            const char *ptr;
             Py_complex c;
-            n = r_byte(p);
+            c.real = r_float_str(p);
-            if (n == EOF) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
-                break;
-            }
-            ptr = r_string(n, p);
-            if (ptr == NULL)
-                break;
-            memcpy(buf, ptr, n);
-            buf[n] = '\0';
-            c.real = PyOS_string_to_double(buf, NULL, NULL);
             if (c.real == -1.0 && PyErr_Occurred())
                 break;
-            n = r_byte(p);
+            c.imag = r_float_str(p);
-            if (n == EOF) {
-                PyErr_SetString(PyExc_EOFError,
-                    "EOF read where object expected");
-                break;
-            }
-            ptr = r_string(n, p);
-            if (ptr == NULL)
-                break;
-            memcpy(buf, ptr, n);
-            buf[n] = '\0';
-            c.imag = PyOS_string_to_double(buf, NULL, NULL);
             if (c.imag == -1.0 && PyErr_Occurred())
                 break;
             retval = PyComplex_FromCComplex(c);
@@ -1093,18 +1063,11 @@ r_object(RFILE *p)
 
     case TYPE_BINARY_COMPLEX:
         {
-            const unsigned char *buf;
             Py_complex c;
-            buf = (const unsigned char *) r_string(8, p);
+            c.real = r_float_bin(p);
-            if (buf == NULL)
-                break;
-            c.real = _PyFloat_Unpack8(buf, 1);
             if (c.real == -1.0 && PyErr_Occurred())
                 break;
-            buf = (const unsigned char *) r_string(8, p);
+            c.imag = r_float_bin(p);
-            if (buf == NULL)
-                break;
-            c.imag = _PyFloat_Unpack8(buf, 1);
             if (c.imag == -1.0 && PyErr_Occurred())
                 break;
             retval = PyComplex_FromCComplex(c);