try to use the same str object for all code filenames when compiling or unmarshalling (#12190)
This should reduce memory usage.
This commit is contained in:
parent
d408503b2c
commit
43b068648e
|
@ -1,6 +1,7 @@
|
|||
import unittest
|
||||
import sys
|
||||
import _ast
|
||||
import types
|
||||
from test import support
|
||||
|
||||
class TestSpecifics(unittest.TestCase):
|
||||
|
@ -433,6 +434,14 @@ if 1:
|
|||
ast.body = [_ast.BoolOp()]
|
||||
self.assertRaises(TypeError, compile, ast, '<ast>', 'exec')
|
||||
|
||||
@support.cpython_only
|
||||
def test_same_filename_used(self):
|
||||
s = """def f(): pass\ndef g(): pass"""
|
||||
c = compile(s, "myfile", "exec")
|
||||
for obj in c.co_consts:
|
||||
if isinstance(obj, types.CodeType):
|
||||
self.assertIs(obj.co_filename, c.co_filename)
|
||||
|
||||
|
||||
def test_main():
|
||||
support.run_unittest(TestSpecifics)
|
||||
|
|
|
@ -5,6 +5,7 @@ import marshal
|
|||
import sys
|
||||
import unittest
|
||||
import os
|
||||
import types
|
||||
|
||||
class HelperMixin:
|
||||
def helper(self, sample, *extra):
|
||||
|
@ -113,6 +114,22 @@ class CodeTestCase(unittest.TestCase):
|
|||
codes = (ExceptionTestCase.test_exceptions.__code__,) * count
|
||||
marshal.loads(marshal.dumps(codes))
|
||||
|
||||
def test_different_filenames(self):
|
||||
co1 = compile("x", "f1", "exec")
|
||||
co2 = compile("y", "f2", "exec")
|
||||
co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
|
||||
self.assertEqual(co1.co_filename, "f1")
|
||||
self.assertEqual(co2.co_filename, "f2")
|
||||
|
||||
@support.cpython_only
|
||||
def test_same_filename_used(self):
|
||||
s = """def f(): pass\ndef g(): pass"""
|
||||
co = compile(s, "myfile", "exec")
|
||||
co = marshal.loads(marshal.dumps(co))
|
||||
for obj in co.co_consts:
|
||||
if isinstance(obj, types.CodeType):
|
||||
self.assertIs(co.co_filename, obj.co_filename)
|
||||
|
||||
class ContainerTestCase(unittest.TestCase, HelperMixin):
|
||||
d = {'astring': 'foo@bar.baz.spam',
|
||||
'afloat': 7283.43,
|
||||
|
|
|
@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1?
|
|||
Core and Builtins
|
||||
-----------------
|
||||
|
||||
- Issue #12190: Try to use the same filename object when compiling unmarshalling
|
||||
a code objects in the same file.
|
||||
|
||||
- Issue #12166: Move implementations of dir() specialized for various types into
|
||||
the __dir__() methods of those types.
|
||||
|
||||
|
|
|
@ -135,6 +135,7 @@ managed by compiler_enter_scope() and compiler_exit_scope().
|
|||
|
||||
struct compiler {
|
||||
const char *c_filename;
|
||||
PyObject *c_filename_obj;
|
||||
struct symtable *c_st;
|
||||
PyFutureFeatures *c_future; /* pointer to module's __future__ */
|
||||
PyCompilerFlags *c_flags;
|
||||
|
@ -272,6 +273,9 @@ PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags,
|
|||
if (!compiler_init(&c))
|
||||
return NULL;
|
||||
c.c_filename = filename;
|
||||
c.c_filename_obj = PyUnicode_DecodeFSDefault(filename);
|
||||
if (!c.c_filename_obj)
|
||||
goto finally;
|
||||
c.c_arena = arena;
|
||||
c.c_future = PyFuture_FromAST(mod, filename);
|
||||
if (c.c_future == NULL)
|
||||
|
@ -324,6 +328,8 @@ compiler_free(struct compiler *c)
|
|||
PySymtable_Free(c->c_st);
|
||||
if (c->c_future)
|
||||
PyObject_Free(c->c_future);
|
||||
if (c->c_filename_obj)
|
||||
Py_DECREF(c->c_filename_obj);
|
||||
Py_DECREF(c->c_stack);
|
||||
}
|
||||
|
||||
|
@ -3361,7 +3367,7 @@ compiler_in_loop(struct compiler *c) {
|
|||
static int
|
||||
compiler_error(struct compiler *c, const char *errstr)
|
||||
{
|
||||
PyObject *loc, *filename;
|
||||
PyObject *loc;
|
||||
PyObject *u = NULL, *v = NULL;
|
||||
|
||||
loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno);
|
||||
|
@ -3369,16 +3375,7 @@ compiler_error(struct compiler *c, const char *errstr)
|
|||
Py_INCREF(Py_None);
|
||||
loc = Py_None;
|
||||
}
|
||||
if (c->c_filename != NULL) {
|
||||
filename = PyUnicode_DecodeFSDefault(c->c_filename);
|
||||
if (!filename)
|
||||
goto exit;
|
||||
}
|
||||
else {
|
||||
Py_INCREF(Py_None);
|
||||
filename = Py_None;
|
||||
}
|
||||
u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno,
|
||||
u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno,
|
||||
c->u->u_col_offset, loc);
|
||||
if (!u)
|
||||
goto exit;
|
||||
|
@ -3927,7 +3924,6 @@ makecode(struct compiler *c, struct assembler *a)
|
|||
PyObject *consts = NULL;
|
||||
PyObject *names = NULL;
|
||||
PyObject *varnames = NULL;
|
||||
PyObject *filename = NULL;
|
||||
PyObject *name = NULL;
|
||||
PyObject *freevars = NULL;
|
||||
PyObject *cellvars = NULL;
|
||||
|
@ -3951,10 +3947,6 @@ makecode(struct compiler *c, struct assembler *a)
|
|||
freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
|
||||
if (!freevars)
|
||||
goto error;
|
||||
filename = PyUnicode_DecodeFSDefault(c->c_filename);
|
||||
if (!filename)
|
||||
goto error;
|
||||
|
||||
nlocals = PyDict_Size(c->u->u_varnames);
|
||||
flags = compute_code_flags(c);
|
||||
if (flags < 0)
|
||||
|
@ -3974,14 +3966,13 @@ makecode(struct compiler *c, struct assembler *a)
|
|||
nlocals, stackdepth(c), flags,
|
||||
bytecode, consts, names, varnames,
|
||||
freevars, cellvars,
|
||||
filename, c->u->u_name,
|
||||
c->c_filename_obj, c->u->u_name,
|
||||
c->u->u_firstlineno,
|
||||
a->a_lnotab);
|
||||
error:
|
||||
Py_XDECREF(consts);
|
||||
Py_XDECREF(names);
|
||||
Py_XDECREF(varnames);
|
||||
Py_XDECREF(filename);
|
||||
Py_XDECREF(name);
|
||||
Py_XDECREF(freevars);
|
||||
Py_XDECREF(cellvars);
|
||||
|
|
|
@ -58,6 +58,7 @@ typedef struct {
|
|||
int depth;
|
||||
/* If fp == NULL, the following are valid: */
|
||||
PyObject *str;
|
||||
PyObject *current_filename;
|
||||
char *ptr;
|
||||
char *end;
|
||||
int version;
|
||||
|
@ -976,6 +977,18 @@ r_object(RFILE *p)
|
|||
filename = r_object(p);
|
||||
if (filename == NULL)
|
||||
goto code_error;
|
||||
if (PyUnicode_CheckExact(filename)) {
|
||||
if (p->current_filename != NULL) {
|
||||
if (!PyUnicode_Compare(filename, p->current_filename)) {
|
||||
Py_DECREF(filename);
|
||||
Py_INCREF(p->current_filename);
|
||||
filename = p->current_filename;
|
||||
}
|
||||
}
|
||||
else {
|
||||
p->current_filename = filename;
|
||||
}
|
||||
}
|
||||
name = r_object(p);
|
||||
if (name == NULL)
|
||||
goto code_error;
|
||||
|
@ -1037,6 +1050,7 @@ PyMarshal_ReadShortFromFile(FILE *fp)
|
|||
RFILE rf;
|
||||
assert(fp);
|
||||
rf.fp = fp;
|
||||
rf.current_filename = NULL;
|
||||
rf.end = rf.ptr = NULL;
|
||||
return r_short(&rf);
|
||||
}
|
||||
|
@ -1046,6 +1060,7 @@ PyMarshal_ReadLongFromFile(FILE *fp)
|
|||
{
|
||||
RFILE rf;
|
||||
rf.fp = fp;
|
||||
rf.current_filename = NULL;
|
||||
rf.ptr = rf.end = NULL;
|
||||
return r_long(&rf);
|
||||
}
|
||||
|
@ -1106,6 +1121,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp)
|
|||
RFILE rf;
|
||||
PyObject *result;
|
||||
rf.fp = fp;
|
||||
rf.current_filename = NULL;
|
||||
rf.depth = 0;
|
||||
rf.ptr = rf.end = NULL;
|
||||
result = r_object(&rf);
|
||||
|
@ -1118,6 +1134,7 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
|
|||
RFILE rf;
|
||||
PyObject *result;
|
||||
rf.fp = NULL;
|
||||
rf.current_filename = NULL;
|
||||
rf.ptr = str;
|
||||
rf.end = str + len;
|
||||
rf.depth = 0;
|
||||
|
@ -1214,6 +1231,7 @@ marshal_load(PyObject *self, PyObject *f)
|
|||
if (data == NULL)
|
||||
return NULL;
|
||||
rf.fp = NULL;
|
||||
rf.current_filename = NULL;
|
||||
if (PyBytes_Check(data)) {
|
||||
rf.ptr = PyBytes_AS_STRING(data);
|
||||
rf.end = rf.ptr + PyBytes_GET_SIZE(data);
|
||||
|
@ -1282,6 +1300,7 @@ marshal_loads(PyObject *self, PyObject *args)
|
|||
s = p.buf;
|
||||
n = p.len;
|
||||
rf.fp = NULL;
|
||||
rf.current_filename = NULL;
|
||||
rf.ptr = s;
|
||||
rf.end = s + n;
|
||||
rf.depth = 0;
|
||||
|
|
Loading…
Reference in New Issue