MERGE: Closes #15512: Correct __sizeof__ support for parser

Include/node.h

@@ -20,6 +20,9 @@ PyAPI_FUNC(node *) PyNode_New(int type);
 PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
                                       char *str, int lineno, int col_offset);
 PyAPI_FUNC(void) PyNode_Free(node *n);
+#ifndef Py_LIMITED_API
+Py_ssize_t _PyNode_SizeOf(node *n);
+#endif
 
 /* Node access functions */
 #define NCH(n)		((n)->n_nchildren)

Lib/test/test_parser.py

@@ -2,6 +2,7 @@ import parser
 import unittest
 import sys
 import operator
+import struct
 from test import support
 
 #
@@ -679,6 +680,46 @@ class STObjectTestCase(unittest.TestCase):
         self.assertRaises(TypeError, operator.lt, st1, 1815)
         self.assertRaises(TypeError, operator.gt, b'waterloo', st2)
 
+    check_sizeof = support.check_sizeof
+
+    @support.cpython_only
+    def test_sizeof(self):
+        def XXXROUNDUP(n):
+            if n <= 1:
+                return n
+            if n <= 128:
+                return (n + 3) & ~3
+            return 1 << (n - 1).bit_length()
+
+        basesize = support.calcobjsize('Pii')
+        nodesize = struct.calcsize('hP3iP0h')
+        def sizeofchildren(node):
+            if node is None:
+                return 0
+            res = 0
+            hasstr = len(node) > 1 and isinstance(node[-1], str)
+            if hasstr:
+                res += len(node[-1]) + 1
+            children = node[1:-1] if hasstr else node[1:]
+            if children:
+                res += XXXROUNDUP(len(children)) * nodesize
+            res1 = res
+            if children:
+                for child in children:
+                    res += sizeofchildren(child)
+            return res
+
+        def check_st_sizeof(st):
+            self.check_sizeof(st, basesize + nodesize +
+                                  sizeofchildren(st.totuple()))
+
+        check_st_sizeof(parser.expr('2 + 3'))
+        check_st_sizeof(parser.expr('2 + 3 + 4'))
+        check_st_sizeof(parser.suite('x = 2 + 3'))
+        check_st_sizeof(parser.suite(''))
+        check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-'))
+        check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']'))
+
 
     # XXX tests for pickling and unpickling of ST objects should go here
 

Misc/NEWS

@@ -367,6 +367,9 @@ Library
 - Issue #12288: Consider '0' and '0.0' as valid initialvalue
   for tkinter SimpleDialog.
 
+- Issue #15512: Add a __sizeof__ implementation for parser.
+  Patch by Serhiy Storchaka.
+
 - Issue #15489: Add a __sizeof__ implementation for BytesIO objects.
   Patch by Serhiy Storchaka.
 

Modules/parsermodule.c

@@ -167,6 +167,7 @@ typedef struct {
 
 
 static void parser_free(PyST_Object *st);
+static PyObject* parser_sizeof(PyST_Object *, void *);
 static PyObject* parser_richcompare(PyObject *left, PyObject *right, int op);
 static PyObject* parser_compilest(PyST_Object *, PyObject *, PyObject *);
 static PyObject* parser_isexpr(PyST_Object *, PyObject *, PyObject *);
@@ -187,7 +188,8 @@ static PyMethodDef parser_methods[] = {
         PyDoc_STR("Creates a list-tree representation of this ST.")},
     {"totuple",         (PyCFunction)parser_st2tuple,   PUBLIC_METHOD_TYPE,
         PyDoc_STR("Creates a tuple-tree representation of this ST.")},
-
+    {"__sizeof__",      (PyCFunction)parser_sizeof,     METH_NOARGS,
+        PyDoc_STR("Returns size in memory, in bytes.")},
     {NULL, NULL, 0, NULL}
 };
 
@@ -361,6 +363,15 @@ parser_free(PyST_Object *st)
     PyObject_Del(st);
 }
 
+static PyObject *
+parser_sizeof(PyST_Object *st, void *unused)
+{
+    Py_ssize_t res;
+
+    res = sizeof(PyST_Object) + _PyNode_SizeOf(st->st_node);
+    return PyLong_FromSsize_t(res);
+}
+
 
 /*  parser_st2tuple(PyObject* self, PyObject* args, PyObject* kw)
  *

Parser/node.c

@@ -114,6 +114,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offs
 
 /* Forward */
 static void freechildren(node *);
+static Py_ssize_t sizeofchildren(node *n);
 
 
 void
@@ -125,6 +126,16 @@ PyNode_Free(node *n)
     }
 }
 
+Py_ssize_t
+_PyNode_SizeOf(node *n)
+{
+    Py_ssize_t res = 0;
+
+    if (n != NULL)
+        res = sizeof(node) + sizeofchildren(n);
+    return res;
+}
+
 static void
 freechildren(node *n)
 {
@@ -136,3 +147,18 @@ freechildren(node *n)
     if (STR(n) != NULL)
         PyObject_FREE(STR(n));
 }
+
+static Py_ssize_t
+sizeofchildren(node *n)
+{
+    Py_ssize_t res = 0;
+    int i;
+    for (i = NCH(n); --i >= 0; )
+        res += sizeofchildren(CHILD(n, i));
+    if (n->n_child != NULL)
+        /* allocated size of n->n_child array */
+        res += XXXROUNDUP(NCH(n)) * sizeof(node);
+    if (STR(n) != NULL)
+        res += strlen(STR(n)) + 1;
+    return res;
+}