Skip to content

Commit 3e3192d

Browse files
jceajcea
committed
Closes python#15512: Correct __sizeof__ support for parser
1 parent 1fa9f7b commit 3e3192d

5 files changed

Lines changed: 103 additions & 4 deletions

File tree

‎Include/node.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ PyAPI_FUNC(node *) PyNode_New(int type);
2020
PyAPI_FUNC(int) PyNode_AddChild(node *n, int type,
2121
char *str, int lineno, int col_offset);
2222
PyAPI_FUNC(void) PyNode_Free(node *n);
23+
#ifndef Py_LIMITED_API
24+
Py_ssize_t _PyNode_SizeOf(node *n);
25+
#endif
2326

2427
/* Node access functions */
2528
#define NCH(n) ((n)->n_nchildren)

‎Lib/test/test_parser.py‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import parser
22
import unittest
33
import sys
4-
from test import test_support
4+
import struct
5+
from test import test_support as support
56

67
#
78
# First, we test that we can generate trees from valid source fragments,
@@ -583,12 +584,59 @@ def test_trigger_memory_error(self):
583584
print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line"
584585
self.assertRaises(MemoryError, parser.expr, e)
585586

587+
class STObjectTestCase(unittest.TestCase):
588+
"""Test operations on ST objects themselves"""
589+
590+
check_sizeof = support.check_sizeof
591+
592+
@support.cpython_only
593+
def test_sizeof(self):
594+
def XXXROUNDUP(n):
595+
if n <= 1:
596+
return n
597+
if n <= 128:
598+
return (n + 3) & ~3
599+
return 1 << (n - 1).bit_length()
600+
601+
basesize = support.calcobjsize('Pii')
602+
nodesize = struct.calcsize('hP3iP0h')
603+
def sizeofchildren(node):
604+
if node is None:
605+
return 0
606+
res = 0
607+
hasstr = len(node) > 1 and isinstance(node[-1], str)
608+
if hasstr:
609+
res += len(node[-1]) + 1
610+
children = node[1:-1] if hasstr else node[1:]
611+
if children:
612+
res += XXXROUNDUP(len(children)) * nodesize
613+
res1 = res
614+
if children:
615+
for child in children:
616+
res += sizeofchildren(child)
617+
return res
618+
619+
def check_st_sizeof(st):
620+
self.check_sizeof(st, basesize + nodesize +
621+
sizeofchildren(st.totuple()))
622+
623+
check_st_sizeof(parser.expr('2 + 3'))
624+
check_st_sizeof(parser.expr('2 + 3 + 4'))
625+
check_st_sizeof(parser.suite('x = 2 + 3'))
626+
check_st_sizeof(parser.suite(''))
627+
check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-'))
628+
check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']'))
629+
630+
631+
# XXX tests for pickling and unpickling of ST objects should go here
632+
586633
def test_main():
587-
test_support.run_unittest(
634+
support.run_unittest(
588635
RoundtripLegalSyntaxTestCase,
589636
IllegalSyntaxTestCase,
590637
CompileTestCase,
591638
ParserStackLimitTestCase,
639+
STObjectTestCase,
592640
)
593641

594642

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ Library
107107
- Issue #15487: Add a __sizeof__ implementation for buffered I/O objects.
108108
Patch by Serhiy Storchaka.
109109

110+
- Issue #15512: Add a __sizeof__ implementation for parser.
111+
Patch by Serhiy Storchaka.
112+
110113
- Issue #15402: An issue in the struct module that caused sys.getsizeof to
111114
return incorrect results for struct.Struct instances has been fixed.
112115
Initial patch by Serhiy Storchaka.

‎Modules/parsermodule.c‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,10 @@ typedef struct {
169169

170170

171171
static void parser_free(PyST_Object *st);
172+
static PyObject* parser_sizeof(PyST_Object *, void *);
172173
static int parser_compare(PyST_Object *left, PyST_Object *right);
173174
static PyObject *parser_getattr(PyObject *self, char *name);
175+
static PyMethodDef parser_methods[];
174176

175177

176178
static
@@ -200,7 +202,14 @@ PyTypeObject PyST_Type = {
200202
Py_TPFLAGS_DEFAULT, /* tp_flags */
201203

202204
/* __doc__ */
203-
"Intermediate representation of a Python parse tree."
205+
"Intermediate representation of a Python parse tree.",
206+
0, /* tp_traverse */
207+
0, /* tp_clear */
208+
0, /* tp_richcompare */
209+
0, /* tp_weaklistoffset */
210+
0, /* tp_iter */
211+
0, /* tp_iternext */
212+
parser_methods, /* tp_methods */
204213
}; /* PyST_Type */
205214

206215

@@ -508,7 +517,8 @@ parser_methods[] = {
508517
PyDoc_STR("Creates a list-tree representation of this ST.")},
509518
{"totuple", (PyCFunction)parser_st2tuple, PUBLIC_METHOD_TYPE,
510519
PyDoc_STR("Creates a tuple-tree representation of this ST.")},
511-
520+
{"__sizeof__", (PyCFunction)parser_sizeof, METH_NOARGS,
521+
PyDoc_STR("Returns size in memory, in bytes.")},
512522
{NULL, NULL, 0, NULL}
513523
};
514524

@@ -695,6 +705,15 @@ parser_tuple2ast(PyST_Object *self, PyObject *args, PyObject *kw)
695705
return parser_tuple2st(self, args, kw);
696706
}
697707

708+
static PyObject *
709+
parser_sizeof(PyST_Object *st, void *unused)
710+
{
711+
Py_ssize_t res;
712+
713+
res = sizeof(PyST_Object) + _PyNode_SizeOf(st->st_node);
714+
return PyLong_FromSsize_t(res);
715+
}
716+
698717

699718
/* node* build_node_children()
700719
*

‎Parser/node.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offs
114114

115115
/* Forward */
116116
static void freechildren(node *);
117+
static Py_ssize_t sizeofchildren(node *n);
117118

118119

119120
void
@@ -125,6 +126,16 @@ PyNode_Free(node *n)
125126
}
126127
}
127128

129+
Py_ssize_t
130+
_PyNode_SizeOf(node *n)
131+
{
132+
Py_ssize_t res = 0;
133+
134+
if (n != NULL)
135+
res = sizeof(node) + sizeofchildren(n);
136+
return res;
137+
}
138+
128139
static void
129140
freechildren(node *n)
130141
{
@@ -136,3 +147,18 @@ freechildren(node *n)
136147
if (STR(n) != NULL)
137148
PyObject_FREE(STR(n));
138149
}
150+
151+
static Py_ssize_t
152+
sizeofchildren(node *n)
153+
{
154+
Py_ssize_t res = 0;
155+
int i;
156+
for (i = NCH(n); --i >= 0; )
157+
res += sizeofchildren(CHILD(n, i));
158+
if (n->n_child != NULL)
159+
/* allocated size of n->n_child array */
160+
res += XXXROUNDUP(NCH(n)) * sizeof(node);
161+
if (STR(n) != NULL)
162+
res += strlen(STR(n)) + 1;
163+
return res;
164+
}

0 commit comments

Comments
 (0)