traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixes Issue #14331: Use significantly less stack space when importing modules by 

allocating path buffers on the heap instead of the stack.
This commit is contained in:
Gregory P. Smith 2012-03-18 16:07:10 -07:00
parent 28b22d0d15
commit fcdf04becc
2 changed files with 98 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Misc/NEWS
View File

@ -9,6 +9,9 @@ What's New in Python 2.7.4
Core and Builtins
-----------------
- Issue #14331: Use significantly less stack space when importing modules by
allocating path buffers on the heap instead of the stack.
- Issue #14334: Prevent in a segfault in type.__getattribute__ when it was not
passed strings. Also fix segfaults in the __getattribute__ and __setattr__
methods of old-style classes.

126
Python/import.c
View File

@ -996,7 +996,7 @@ load_source_module(char *name, char *pathname, FILE *fp)
{
struct stat st;
FILE *fpc;
char buf[MAXPATHLEN+1];
char *buf;
char *cpathname;
PyCodeObject *co;
PyObject *m;
@ -1015,6 +1015,10 @@ load_source_module(char *name, char *pathname, FILE *fp)
*/
st.st_mtime &= 0xFFFFFFFF;
}
buf = PyMem_MALLOC(MAXPATHLEN+1);
if (buf == NULL) {
return PyErr_NoMemory();
}
cpathname = make_compiled_pathname(pathname, buf,
(size_t)MAXPATHLEN + 1);
if (cpathname != NULL &&
@ -1022,9 +1026,9 @@ load_source_module(char *name, char *pathname, FILE *fp)
co = read_compiled_module(cpathname, fpc);
fclose(fpc);
if (co == NULL)
return NULL;
goto error_exit;
if (update_compiled_module(co, pathname) < 0)
return NULL;
goto error_exit;
if (Py_VerboseFlag)
PySys_WriteStderr("import %s # precompiled from %s\n",
name, cpathname);
@ -1033,7 +1037,7 @@ load_source_module(char *name, char *pathname, FILE *fp)
else {
co = parse_source_module(pathname, fp);
if (co == NULL)
return NULL;
goto error_exit;
if (Py_VerboseFlag)
PySys_WriteStderr("import %s # from %s\n",
name, pathname);
@ -1046,7 +1050,12 @@ load_source_module(char *name, char *pathname, FILE *fp)
m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, pathname);
Py_DECREF(co);
PyMem_FREE(buf);
return m;
error_exit:
PyMem_FREE(buf);
return NULL;
}
@ -1066,7 +1075,7 @@ load_package(char *name, char *pathname)
PyObject *file = NULL;
PyObject *path = NULL;
int err;
char buf[MAXPATHLEN+1];
char *buf = NULL;
FILE *fp = NULL;
struct filedescr *fdp;
@ -1088,8 +1097,13 @@ load_package(char *name, char *pathname)
err = PyDict_SetItemString(d, "__path__", path);
if (err != 0)
goto error;
buf = PyMem_MALLOC(MAXPATHLEN+1);
if (buf == NULL) {
PyErr_NoMemory();
goto error;
}
buf[0] = '\0';
fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL);
fdp = find_module(name, "__init__", path, buf, MAXPATHLEN+1, &fp, NULL);
if (fdp == NULL) {
if (PyErr_ExceptionMatches(PyExc_ImportError)) {
PyErr_Clear();
@ -1107,6 +1121,8 @@ load_package(char *name, char *pathname)
error:
m = NULL;
cleanup:
if (buf)
PyMem_FREE(buf);
Py_XDECREF(path);
Py_XDECREF(file);
return m;
@ -1235,7 +1251,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
static struct filedescr fd_frozen = {"", "", PY_FROZEN};
static struct filedescr fd_builtin = {"", "", C_BUILTIN};
static struct filedescr fd_package = {"", "", PKG_DIRECTORY};
char name[MAXPATHLEN+1];
char *name;
#if defined(PYOS_OS2)
size_t saved_len;
size_t saved_namelen;
@ -1249,6 +1265,10 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
"module name is too long");
return NULL;
}
name = PyMem_MALLOC(MAXPATHLEN+1);
if (name == NULL) {
return PyErr_NoMemory();
}
strcpy(name, subname);
/* sys.meta_path import hook */
@ -1260,7 +1280,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
PyErr_SetString(PyExc_RuntimeError,
"sys.meta_path must be a list of "
"import hooks");
return NULL;
goto error_exit;
}
Py_INCREF(meta_path); /* zap guard */
npath = PyList_Size(meta_path);
@ -1273,12 +1293,13 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
path : Py_None);
if (loader == NULL) {
Py_DECREF(meta_path);
return NULL; /* true error */
goto error_exit; /* true error */
}
if (loader != Py_None) {
/* a loader was found */
*p_loader = loader;
Py_DECREF(meta_path);
PyMem_FREE(name);
return &importhookdescr;
}
Py_DECREF(loader);
@ -1292,7 +1313,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
if (PyString_Size(path) + 1 + strlen(name) >= (size_t)buflen) {
PyErr_SetString(PyExc_ImportError,
"full frozen module name too long");
return NULL;
goto error_exit;
}
strcpy(buf, PyString_AsString(path));
strcat(buf, ".");
@ -1300,19 +1321,22 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
strcpy(name, buf);
if (find_frozen(name) != NULL) {
strcpy(buf, name);
PyMem_FREE(name);
return &fd_frozen;
}
PyErr_Format(PyExc_ImportError,
"No frozen submodule named %.200s", name);
return NULL;
goto error_exit;
}
if (path == NULL) {
if (is_builtin(name)) {
strcpy(buf, name);
PyMem_FREE(name);
return &fd_builtin;
}
if ((find_frozen(name)) != NULL) {
strcpy(buf, name);
PyMem_FREE(name);
return &fd_frozen;
}
@ -1320,6 +1344,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen);
if (fp != NULL) {
*p_fp = fp;
PyMem_FREE(name);
return fdp;
}
#endif
@ -1328,7 +1353,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
if (path == NULL || !PyList_Check(path)) {
PyErr_SetString(PyExc_RuntimeError,
"sys.path must be a list of directory names");
return NULL;
goto error_exit;
}
path_hooks = PySys_GetObject("path_hooks");
@ -1336,14 +1361,14 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
PyErr_SetString(PyExc_RuntimeError,
"sys.path_hooks must be a list of "
"import hooks");
return NULL;
goto error_exit;
}
path_importer_cache = PySys_GetObject("path_importer_cache");
if (path_importer_cache == NULL ||
!PyDict_Check(path_importer_cache)) {
PyErr_SetString(PyExc_RuntimeError,
"sys.path_importer_cache must be a dict");
return NULL;
goto error_exit;
}
npath = PyList_Size(path);
@ -1352,13 +1377,13 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
PyObject *copy = NULL;
PyObject *v = PyList_GetItem(path, i);
if (!v)
return NULL;
goto error_exit;
#ifdef Py_USING_UNICODE
if (PyUnicode_Check(v)) {
copy = PyUnicode_Encode(PyUnicode_AS_UNICODE(v),
PyUnicode_GET_SIZE(v), Py_FileSystemDefaultEncoding, NULL);
if (copy == NULL)
return NULL;
goto error_exit;
v = copy;
}
else
@ -1384,7 +1409,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
path_hooks, v);
if (importer == NULL) {
Py_XDECREF(copy);
return NULL;
goto error_exit;
}
/* Note: importer is a borrowed reference */
if (importer != Py_None) {
@ -1394,10 +1419,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
"s", fullname);
Py_XDECREF(copy);
if (loader == NULL)
return NULL; /* error */
goto error_exit; /* error */
if (loader != Py_None) {
/* a loader was found */
*p_loader = loader;
PyMem_FREE(name);
return &importhookdescr;
}
Py_DECREF(loader);
@ -1421,6 +1447,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
case_ok(buf, len, namelen, name)) { /* case matches */
if (find_init_module(buf)) { /* and has __init__.py */
Py_XDECREF(copy);
PyMem_FREE(name);
return &fd_package;
}
else {
@ -1431,7 +1458,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
if (PyErr_Warn(PyExc_ImportWarning,
warnstr)) {
Py_XDECREF(copy);
return NULL;
goto error_exit;
}
}
}
@ -1506,10 +1533,15 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
if (fp == NULL) {
PyErr_Format(PyExc_ImportError,
"No module named %.200s", name);
return NULL;
goto error_exit;
}
*p_fp = fp;
PyMem_FREE(name);
return fdp;
error_exit:
PyMem_FREE(name);
return NULL;
}
/* Helpers for main.c
@ -2116,7 +2148,7 @@ static PyObject *
import_module_level(char *name, PyObject *globals, PyObject *locals,
PyObject *fromlist, int level)
{
char buf[MAXPATHLEN+1];
char *buf;
Py_ssize_t buflen = 0;
PyObject *parent, *head, *next, *tail;
@ -2130,14 +2162,18 @@ import_module_level(char *name, PyObject *globals, PyObject *locals,
return NULL;
}
buf = PyMem_MALLOC(MAXPATHLEN+1);
if (buf == NULL) {
return PyErr_NoMemory();
}
parent = get_parent(globals, buf, &buflen, level);
if (parent == NULL)
return NULL;
goto error_exit;
head = load_next(parent, level < 0 ? Py_None : parent, &name, buf,
&buflen);
if (head == NULL)
return NULL;
goto error_exit;
tail = head;
Py_INCREF(tail);
@ -2146,7 +2182,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals,
Py_DECREF(tail);
if (next == NULL) {
Py_DECREF(head);
return NULL;
goto error_exit;
}
tail = next;
}
@ -2158,7 +2194,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals,
Py_DECREF(head);
PyErr_SetString(PyExc_ValueError,
"Empty module name");
return NULL;
goto error_exit;
}
if (fromlist != NULL) {
@ -2168,16 +2204,22 @@ import_module_level(char *name, PyObject *globals, PyObject *locals,
if (fromlist == NULL) {
Py_DECREF(tail);
PyMem_FREE(buf);
return head;
}
Py_DECREF(head);
if (!ensure_fromlist(tail, fromlist, buf, buflen, 0)) {
Py_DECREF(tail);
return NULL;
goto error_exit;
}
PyMem_FREE(buf);
return tail;
error_exit:
PyMem_FREE(buf);
return NULL;
}
PyObject *
@ -2567,7 +2609,7 @@ import_submodule(PyObject *mod, char *subname, char *fullname)
}
else {
PyObject *path, *loader = NULL;
char buf[MAXPATHLEN+1];
char *buf;
struct filedescr *fdp;
FILE *fp = NULL;
@ -2582,11 +2624,16 @@ import_submodule(PyObject *mod, char *subname, char *fullname)
}
}
buf = PyMem_MALLOC(MAXPATHLEN+1);
if (buf == NULL) {
return PyErr_NoMemory();
}
buf[0] = '\0';
fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1,
&fp, &loader);
Py_XDECREF(path);
if (fdp == NULL) {
PyMem_FREE(buf);
if (!PyErr_ExceptionMatches(PyExc_ImportError))
return NULL;
PyErr_Clear();
@ -2601,6 +2648,7 @@ import_submodule(PyObject *mod, char *subname, char *fullname)
Py_XDECREF(m);
m = NULL;
}
PyMem_FREE(buf);
}
return m;
@ -2618,7 +2666,7 @@ PyImport_ReloadModule(PyObject *m)
PyObject *modules = PyImport_GetModuleDict();
PyObject *path = NULL, *loader = NULL, *existing_m = NULL;
char *name, *subname;
char buf[MAXPATHLEN+1];
char *buf;
struct filedescr *fdp;
FILE *fp = NULL;
PyObject *newm;
@ -2678,6 +2726,11 @@ PyImport_ReloadModule(PyObject *m)
if (path == NULL)
PyErr_Clear();
}
buf = PyMem_MALLOC(MAXPATHLEN+1);
if (buf == NULL) {
Py_XDECREF(path);
return PyErr_NoMemory();
}
buf[0] = '\0';
fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader);
Py_XDECREF(path);
@ -2685,6 +2738,7 @@ PyImport_ReloadModule(PyObject *m)
if (fdp == NULL) {
Py_XDECREF(loader);
imp_modules_reloading_clear();
PyMem_FREE(buf);
return NULL;
}
@ -2702,6 +2756,7 @@ PyImport_ReloadModule(PyObject *m)
PyDict_SetItemString(modules, name, m);
}
imp_modules_reloading_clear();
PyMem_FREE(buf);
return newm;
}
@ -2832,20 +2887,28 @@ call_find_module(char *name, PyObject *path)
extern int fclose(FILE *);
PyObject *fob, *ret;
struct filedescr *fdp;
char pathname[MAXPATHLEN+1];
char *pathname;
FILE *fp = NULL;
pathname = PyMem_MALLOC(MAXPATHLEN+1);
if (pathname == NULL) {
return PyErr_NoMemory();
}
pathname[0] = '\0';
if (path == Py_None)
path = NULL;
fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL);
if (fdp == NULL)
if (fdp == NULL) {
PyMem_FREE(pathname);
return NULL;
}
if (fp != NULL) {
fob = PyFile_FromFile(fp, pathname, fdp->mode, fclose);
if (fob == NULL)
if (fob == NULL) {
PyMem_FREE(pathname);
return NULL;
}
}
else {
fob = Py_None;
Py_INCREF(fob);
@ -2853,6 +2916,7 @@ call_find_module(char *name, PyObject *path)
ret = Py_BuildValue("Os(ssi)",
fob, pathname, fdp->suffix, fdp->mode, fdp->type);
Py_DECREF(fob);
PyMem_FREE(pathname);
return ret;
}