From 43b068648e81d951f78d9f1cf1cd8f42731e164e Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 27 May 2011 09:08:01 -0500 Subject: [PATCH] try to use the same str object for all code filenames when compiling or unmarshalling (#12190) This should reduce memory usage. --- Lib/test/test_compile.py | 9 +++++++++ Lib/test/test_marshal.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ Python/compile.c | 27 +++++++++------------------ Python/marshal.c | 19 +++++++++++++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 58ef2979a05..c5f9189fcc0 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import unittest import sys import _ast +import types from test import support class TestSpecifics(unittest.TestCase): @@ -433,6 +434,14 @@ if 1: ast.body = [_ast.BoolOp()] self.assertRaises(TypeError, compile, ast, '', 'exec') + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + c = compile(s, "myfile", "exec") + for obj in c.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(obj.co_filename, c.co_filename) + def test_main(): support.run_unittest(TestSpecifics) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 81cf598402a..8a5590a4a0e 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -5,6 +5,7 @@ import marshal import sys import unittest import os +import types class HelperMixin: def helper(self, sample, *extra): @@ -113,6 +114,22 @@ class CodeTestCase(unittest.TestCase): codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) + def test_different_filenames(self): + co1 = compile("x", "f1", "exec") + co2 = compile("y", "f2", "exec") + co1, co2 = marshal.loads(marshal.dumps((co1, co2))) + self.assertEqual(co1.co_filename, "f1") + self.assertEqual(co2.co_filename, "f2") + + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + co = compile(s, "myfile", "exec") + co = marshal.loads(marshal.dumps(co)) + for obj in co.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(co.co_filename, obj.co_filename) + class ContainerTestCase(unittest.TestCase, HelperMixin): d = {'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, diff --git a/Misc/NEWS b/Misc/NEWS index 0514c965279..df8bca8b7fc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12190: Try to use the same filename object when compiling unmarshalling + a code objects in the same file. + - Issue #12166: Move implementations of dir() specialized for various types into the __dir__() methods of those types. diff --git a/Python/compile.c b/Python/compile.c index d195967e7c4..96d01cdcc87 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -135,6 +135,7 @@ managed by compiler_enter_scope() and compiler_exit_scope(). struct compiler { const char *c_filename; + PyObject *c_filename_obj; struct symtable *c_st; PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; @@ -272,6 +273,9 @@ PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags, if (!compiler_init(&c)) return NULL; c.c_filename = filename; + c.c_filename_obj = PyUnicode_DecodeFSDefault(filename); + if (!c.c_filename_obj) + goto finally; c.c_arena = arena; c.c_future = PyFuture_FromAST(mod, filename); if (c.c_future == NULL) @@ -324,6 +328,8 @@ compiler_free(struct compiler *c) PySymtable_Free(c->c_st); if (c->c_future) PyObject_Free(c->c_future); + if (c->c_filename_obj) + Py_DECREF(c->c_filename_obj); Py_DECREF(c->c_stack); } @@ -3361,7 +3367,7 @@ compiler_in_loop(struct compiler *c) { static int compiler_error(struct compiler *c, const char *errstr) { - PyObject *loc, *filename; + PyObject *loc; PyObject *u = NULL, *v = NULL; loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno); @@ -3369,16 +3375,7 @@ compiler_error(struct compiler *c, const char *errstr) Py_INCREF(Py_None); loc = Py_None; } - if (c->c_filename != NULL) { - filename = PyUnicode_DecodeFSDefault(c->c_filename); - if (!filename) - goto exit; - } - else { - Py_INCREF(Py_None); - filename = Py_None; - } - u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno, + u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno, c->u->u_col_offset, loc); if (!u) goto exit; @@ -3927,7 +3924,6 @@ makecode(struct compiler *c, struct assembler *a) PyObject *consts = NULL; PyObject *names = NULL; PyObject *varnames = NULL; - PyObject *filename = NULL; PyObject *name = NULL; PyObject *freevars = NULL; PyObject *cellvars = NULL; @@ -3951,10 +3947,6 @@ makecode(struct compiler *c, struct assembler *a) freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars)); if (!freevars) goto error; - filename = PyUnicode_DecodeFSDefault(c->c_filename); - if (!filename) - goto error; - nlocals = PyDict_Size(c->u->u_varnames); flags = compute_code_flags(c); if (flags < 0) @@ -3974,14 +3966,13 @@ makecode(struct compiler *c, struct assembler *a) nlocals, stackdepth(c), flags, bytecode, consts, names, varnames, freevars, cellvars, - filename, c->u->u_name, + c->c_filename_obj, c->u->u_name, c->u->u_firstlineno, a->a_lnotab); error: Py_XDECREF(consts); Py_XDECREF(names); Py_XDECREF(varnames); - Py_XDECREF(filename); Py_XDECREF(name); Py_XDECREF(freevars); Py_XDECREF(cellvars); diff --git a/Python/marshal.c b/Python/marshal.c index f66b765b7dc..7b327ade01d 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -58,6 +58,7 @@ typedef struct { int depth; /* If fp == NULL, the following are valid: */ PyObject *str; + PyObject *current_filename; char *ptr; char *end; int version; @@ -976,6 +977,18 @@ r_object(RFILE *p) filename = r_object(p); if (filename == NULL) goto code_error; + if (PyUnicode_CheckExact(filename)) { + if (p->current_filename != NULL) { + if (!PyUnicode_Compare(filename, p->current_filename)) { + Py_DECREF(filename); + Py_INCREF(p->current_filename); + filename = p->current_filename; + } + } + else { + p->current_filename = filename; + } + } name = r_object(p); if (name == NULL) goto code_error; @@ -1037,6 +1050,7 @@ PyMarshal_ReadShortFromFile(FILE *fp) RFILE rf; assert(fp); rf.fp = fp; + rf.current_filename = NULL; rf.end = rf.ptr = NULL; return r_short(&rf); } @@ -1046,6 +1060,7 @@ PyMarshal_ReadLongFromFile(FILE *fp) { RFILE rf; rf.fp = fp; + rf.current_filename = NULL; rf.ptr = rf.end = NULL; return r_long(&rf); } @@ -1106,6 +1121,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) RFILE rf; PyObject *result; rf.fp = fp; + rf.current_filename = NULL; rf.depth = 0; rf.ptr = rf.end = NULL; result = r_object(&rf); @@ -1118,6 +1134,7 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len) RFILE rf; PyObject *result; rf.fp = NULL; + rf.current_filename = NULL; rf.ptr = str; rf.end = str + len; rf.depth = 0; @@ -1214,6 +1231,7 @@ marshal_load(PyObject *self, PyObject *f) if (data == NULL) return NULL; rf.fp = NULL; + rf.current_filename = NULL; if (PyBytes_Check(data)) { rf.ptr = PyBytes_AS_STRING(data); rf.end = rf.ptr + PyBytes_GET_SIZE(data); @@ -1282,6 +1300,7 @@ marshal_loads(PyObject *self, PyObject *args) s = p.buf; n = p.len; rf.fp = NULL; + rf.current_filename = NULL; rf.ptr = s; rf.end = s + n; rf.depth = 0;