Bug #1529871: The speed enhancement patch #921466 broke Python's compliance

with PEP 302.  This was fixed by adding an ``imp.NullImporter`` type that is
used in ``sys.path_importer_cache`` to cache non-directory paths and avoid
excessive filesystem operations during imports.
This commit is contained in:
Phillip J. Eby 2006-07-28 21:12:07 +00:00
parent 944f3b6ecb
commit f7575d0cb7
4 changed files with 161 additions and 38 deletions

View File

@ -232,6 +232,24 @@ properly matching byte-compiled file (with suffix \file{.pyc} or
source file.
\end{funcdesc}
\begin{classdesc}{NullImporter}{path_string}
The \class{NullImporter} type is a \pep{302} import hook that handles
non-directory path strings by failing to find any modules. Calling this
type with an existing directory or empty string raises
\exception{ImportError}. Otherwise, a \class{NullImporter} instance is
returned.
Python adds instances of this type to \code{sys.path_importer_cache} for
any path entries that are not directories and are not handled by any other
path hooks on \code{sys.path_hooks}. Instances have only one method:
\begin{methoddesc}{find_module}{fullname \optional{, path}}
This method always returns \code{None}, indicating that the requested
module could not be found.
\end{methoddesc}
\versionadded{2.5}
\end{classdesc}
\subsection{Examples}
\label{examples-imp}
@ -257,7 +275,7 @@ def __import__(name, globals=None, locals=None, fromlist=None):
# there's a problem we can't handle -- let the caller handle it.
fp, pathname, description = imp.find_module(name)
try:
return imp.load_module(name, fp, pathname, description)
finally:

View File

@ -381,9 +381,7 @@ def get_importer(path_item):
importer = None
sys.path_importer_cache.setdefault(path_item, importer)
# The boolean values are used for caching valid and invalid
# file paths for the built-in import machinery
if importer in (None, True, False):
if importer is None:
try:
importer = ImpImporter(path_item)
except ImportError:

View File

@ -12,6 +12,11 @@ What's New in Python 2.5 release candidate 1?
Core and builtins
-----------------
- Bug #1529871: The speed enhancement patch #921466 broke Python's compliance
with PEP 302. This was fixed by adding an ``imp.NullImporter`` type that is
used in ``sys.path_importer_cache`` to cache non-directory paths and avoid
excessive filesystem operations during imports.
- Bug #1521947: When checking for overflow, ``PyOS_strtol()`` used some
operations on signed longs that are formally undefined by C.
Unfortunately, at least one compiler now cares about that, so complicated
@ -106,10 +111,14 @@ Library
Extension Modules
-----------------
<<<<<<< .mine
- Bug #1471938: Fix curses module build problem on Solaris 8; patch by
=======
- The ``__reduce__()`` method of the new ``collections.defaultdict`` had
a memory leak, affecting pickles and deep copies.
- Bug #1471938: Fix curses module build problem on Solaris 8; patch by
>>>>>>> .r50915
Paul Eggert.
- Patch #1448199: Release interpreter lock in _winreg.ConnectRegistry.

View File

@ -98,6 +98,8 @@ static const struct filedescr _PyImport_StandardFiletab[] = {
};
#endif
static PyTypeObject NullImporterType; /* Forward reference */
/* Initialize things */
void
@ -155,6 +157,8 @@ _PyImportHooks_Init(void)
/* adding sys.path_hooks and sys.path_importer_cache, setting up
zipimport */
if (PyType_Ready(&NullImporterType) < 0)
goto error;
if (Py_VerboseFlag)
PySys_WriteStderr("# installing zipimport hook\n");
@ -180,9 +184,11 @@ _PyImportHooks_Init(void)
if (err) {
error:
PyErr_Print();
Py_FatalError("initializing sys.meta_path, sys.path_hooks or "
"path_importer_cache failed");
Py_FatalError("initializing sys.meta_path, sys.path_hooks, "
"path_importer_cache, or NullImporter failed"
);
}
zimpimport = PyImport_ImportModule("zipimport");
if (zimpimport == NULL) {
PyErr_Clear(); /* No zip import module -- okay */
@ -1058,9 +1064,18 @@ get_path_importer(PyObject *path_importer_cache, PyObject *path_hooks,
}
PyErr_Clear();
}
if (importer == NULL)
importer = Py_None;
else if (importer != Py_None) {
if (importer == NULL) {
importer = PyObject_CallFunctionObjArgs(
(PyObject *)&NullImporterType, p, NULL
);
if (importer == NULL) {
if (PyErr_ExceptionMatches(PyExc_ImportError)) {
PyErr_Clear();
return Py_None;
}
}
}
if (importer != NULL) {
int err = PyDict_SetItem(path_importer_cache, p, importer);
Py_DECREF(importer);
if (err != 0)
@ -1248,35 +1263,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
return NULL;
}
/* Note: importer is a borrowed reference */
if (importer == Py_False) {
/* Cached as not being a valid dir. */
Py_XDECREF(copy);
continue;
}
else if (importer == Py_True) {
/* Cached as being a valid dir, so just
* continue below. */
}
else if (importer == Py_None) {
/* No importer was found, so it has to be a file.
* Check if the directory is valid.
* Note that the empty string is a valid path, but
* not stat'able, hence the check for len. */
#ifdef HAVE_STAT
if (len && stat(buf, &statbuf) != 0) {
/* Directory does not exist. */
PyDict_SetItem(path_importer_cache,
v, Py_False);
Py_XDECREF(copy);
continue;
} else {
PyDict_SetItem(path_importer_cache,
v, Py_True);
}
#endif
}
else {
/* A real import hook importer was found. */
if (importer != Py_None) {
PyObject *loader;
loader = PyObject_CallMethod(importer,
"find_module",
@ -2935,11 +2922,120 @@ setint(PyObject *d, char *name, int value)
return err;
}
typedef struct {
PyObject_HEAD
} NullImporter;
static int
NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds)
{
char *path;
if (!_PyArg_NoKeywords("NullImporter()", kwds))
return -1;
if (!PyArg_ParseTuple(args, "s:NullImporter",
&path))
return -1;
if (strlen(path) == 0) {
PyErr_SetString(PyExc_ImportError, "empty pathname");
return -1;
} else {
#ifndef RISCOS
struct stat statbuf;
int rv;
rv = stat(path, &statbuf);
if (rv == 0) {
/* it exists */
if (S_ISDIR(statbuf.st_mode)) {
/* it's a directory */
PyErr_SetString(PyExc_ImportError,
"existing directory");
return -1;
}
}
#else
if (object_exists(path)) {
/* it exists */
if (isdir(path)) {
/* it's a directory */
PyErr_SetString(PyExc_ImportError,
"existing directory");
return -1;
}
}
#endif
}
return 0;
}
static PyObject *
NullImporter_find_module(NullImporter *self, PyObject *args)
{
Py_RETURN_NONE;
}
static PyMethodDef NullImporter_methods[] = {
{"find_module", (PyCFunction)NullImporter_find_module, METH_VARARGS,
"Always return None"
},
{NULL} /* Sentinel */
};
static PyTypeObject NullImporterType = {
PyObject_HEAD_INIT(NULL)
0, /*ob_size*/
"imp.NullImporter", /*tp_name*/
sizeof(NullImporter), /*tp_basicsize*/
0, /*tp_itemsize*/
0, /*tp_dealloc*/
0, /*tp_print*/
0, /*tp_getattr*/
0, /*tp_setattr*/
0, /*tp_compare*/
0, /*tp_repr*/
0, /*tp_as_number*/
0, /*tp_as_sequence*/
0, /*tp_as_mapping*/
0, /*tp_hash */
0, /*tp_call*/
0, /*tp_str*/
0, /*tp_getattro*/
0, /*tp_setattro*/
0, /*tp_as_buffer*/
Py_TPFLAGS_DEFAULT, /*tp_flags*/
"Null importer object", /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
NullImporter_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)NullImporter_init, /* tp_init */
0, /* tp_alloc */
PyType_GenericNew /* tp_new */
};
PyMODINIT_FUNC
initimp(void)
{
PyObject *m, *d;
if (PyType_Ready(&NullImporterType) < 0)
goto failure;
m = Py_InitModule4("imp", imp_methods, doc_imp,
NULL, PYTHON_API_VERSION);
if (m == NULL)
@ -2957,6 +3053,8 @@ initimp(void)
if (setint(d, "PY_CODERESOURCE", PY_CODERESOURCE) < 0) goto failure;
if (setint(d, "IMP_HOOK", IMP_HOOK) < 0) goto failure;
Py_INCREF(&NullImporterType);
PyModule_AddObject(m, "NullImporter", (PyObject *)&NullImporterType);
failure:
;
}