mirror of https://github.com/python/cpython
Issue #13959: Re-implement imp.load_source() in imp.py.
This commit is contained in:
Brett Cannon
parent
4132368d0a
commit
16475adcbb
29
Lib/imp.py
29
Lib/imp.py
|
|
@ -14,7 +14,7 @@ from _imp import (lock_held, acquire_lock, release_lock, reload,
|
|||
from _imp import (get_magic, get_tag, get_suffixes, cache_from_source,
|
||||
source_from_cache)
|
||||
# Should be re-implemented here (and mostly deprecated)
|
||||
from _imp import (find_module, load_compiled, load_source, NullImporter,
|
||||
from _imp import (find_module, load_compiled, NullImporter,
|
||||
SEARCH_ERROR, PY_SOURCE, PY_COMPILED, C_EXTENSION,
|
||||
PY_RESOURCE, PKG_DIRECTORY, C_BUILTIN, PY_FROZEN,
|
||||
PY_CODERESOURCE, IMP_HOOK)
|
||||
|
|
@ -25,6 +25,33 @@ from importlib import _bootstrap
|
|||
import os
|
||||
|
||||
|
||||
class _LoadSourceCompatibility(_bootstrap._SourceFileLoader):
|
||||
|
||||
"""Compatibility support for implementing load_source()."""
|
||||
|
||||
def __init__(self, fullname, path, file=None):
|
||||
super().__init__(fullname, path)
|
||||
self.file = file
|
||||
|
||||
def get_data(self, path):
|
||||
"""Gross hack to contort SourceFileLoader to deal w/ load_source()'s bad
|
||||
API."""
|
||||
if path == self._path:
|
||||
with self.file:
|
||||
# Technically should be returning bytes, but
|
||||
# SourceLoader.get_code() just passed what is returned to
|
||||
# compile() which can handle str. And converting to bytes would
|
||||
# require figuring out the encoding to decode to and
|
||||
# tokenize.detect_encoding() only accepts bytes.
|
||||
return self.file.read()
|
||||
else:
|
||||
return super().get_data(path)
|
||||
|
||||
|
||||
def load_source(name, pathname, file=None):
|
||||
return _LoadSourceCompatibility(name, pathname, file).load_module(name)
|
||||
|
||||
|
||||
def load_package(name, path):
|
||||
if os.path.isdir(path):
|
||||
extensions = _bootstrap._suffix_list(PY_SOURCE)
|
||||
|
|
|
|||
390
Python/import.c
390
Python/import.c
|
|
@ -904,26 +904,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
|
|||
}
|
||||
|
||||
|
||||
/* Like strrchr(string, '/') but searches for the rightmost of either SEP
|
||||
or ALTSEP, if the latter is defined.
|
||||
*/
|
||||
static Py_UCS4*
|
||||
rightmost_sep(Py_UCS4 *s)
|
||||
{
|
||||
Py_UCS4 *found, c;
|
||||
for (found = NULL; (c = *s); s++) {
|
||||
if (c == SEP
|
||||
#ifdef ALTSEP
|
||||
|| c == ALTSEP
|
||||
#endif
|
||||
)
|
||||
{
|
||||
found = s;
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/* Like rightmost_sep, but operate on unicode objects. */
|
||||
static Py_ssize_t
|
||||
rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end)
|
||||
|
|
@ -1081,50 +1061,6 @@ make_source_pathname(PyObject *path)
|
|||
return result;
|
||||
}
|
||||
|
||||
/* Given a pathname for a Python source file, its time of last
|
||||
modification, and a pathname for a compiled file, check whether the
|
||||
compiled file represents the same version of the source. If so,
|
||||
return a FILE pointer for the compiled file, positioned just after
|
||||
the header; if not, return NULL.
|
||||
Doesn't set an exception. */
|
||||
|
||||
static FILE *
|
||||
check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname)
|
||||
{
|
||||
FILE *fp;
|
||||
long magic;
|
||||
long pyc_mtime;
|
||||
long pyc_size;
|
||||
|
||||
fp = _Py_fopen(cpathname, "rb");
|
||||
if (fp == NULL)
|
||||
return NULL;
|
||||
magic = PyMarshal_ReadLongFromFile(fp);
|
||||
if (magic != pyc_magic) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# %R has bad magic\n", cpathname);
|
||||
fclose(fp);
|
||||
return NULL;
|
||||
}
|
||||
pyc_mtime = PyMarshal_ReadLongFromFile(fp);
|
||||
if (pyc_mtime != srcstat->st_mtime) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# %R has bad mtime\n", cpathname);
|
||||
fclose(fp);
|
||||
return NULL;
|
||||
}
|
||||
pyc_size = PyMarshal_ReadLongFromFile(fp);
|
||||
if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# %R has bad size\n", cpathname);
|
||||
fclose(fp);
|
||||
return NULL;
|
||||
}
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# %R matches %R\n", cpathname, pathname);
|
||||
return fp;
|
||||
}
|
||||
|
||||
|
||||
/* Read a code object from a file and check it for validity */
|
||||
|
||||
|
|
@ -1178,238 +1114,6 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp)
|
|||
return m;
|
||||
}
|
||||
|
||||
/* Parse a source file and return the corresponding code object */
|
||||
|
||||
static PyCodeObject *
|
||||
parse_source_module(PyObject *pathname, FILE *fp)
|
||||
{
|
||||
PyCodeObject *co;
|
||||
PyObject *pathbytes;
|
||||
mod_ty mod;
|
||||
PyCompilerFlags flags;
|
||||
PyArena *arena;
|
||||
|
||||
pathbytes = PyUnicode_EncodeFSDefault(pathname);
|
||||
if (pathbytes == NULL)
|
||||
return NULL;
|
||||
|
||||
arena = PyArena_New();
|
||||
if (arena == NULL) {
|
||||
Py_DECREF(pathbytes);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
flags.cf_flags = 0;
|
||||
mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL,
|
||||
Py_file_input, 0, 0, &flags,
|
||||
NULL, arena);
|
||||
if (mod != NULL)
|
||||
co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena);
|
||||
else
|
||||
co = NULL;
|
||||
Py_DECREF(pathbytes);
|
||||
PyArena_Free(arena);
|
||||
return co;
|
||||
}
|
||||
|
||||
/* Write a compiled module to a file, placing the time of last
|
||||
modification of its source into the header.
|
||||
Errors are ignored, if a write error occurs an attempt is made to
|
||||
remove the file. */
|
||||
|
||||
static void
|
||||
write_compiled_module(PyCodeObject *co, PyObject *cpathname,
|
||||
struct stat *srcstat)
|
||||
{
|
||||
Py_UCS4 *cpathname_ucs4;
|
||||
FILE *fp;
|
||||
time_t mtime = srcstat->st_mtime;
|
||||
long size = srcstat->st_size & 0xFFFFFFFF;
|
||||
PyObject *cpathname_tmp;
|
||||
#ifdef MS_WINDOWS /* since Windows uses different permissions */
|
||||
mode_t mode = srcstat->st_mode & ~S_IEXEC;
|
||||
wchar_t *wdirname, *wpathname, *wpathname_tmp;
|
||||
#else
|
||||
mode_t dirmode = (srcstat->st_mode |
|
||||
S_IXUSR | S_IXGRP | S_IXOTH |
|
||||
S_IWUSR | S_IWGRP | S_IWOTH);
|
||||
PyObject *dirbytes;
|
||||
PyObject *cpathbytes, *cpathbytes_tmp;
|
||||
#endif
|
||||
int fd;
|
||||
PyObject *dirname;
|
||||
Py_UCS4 *dirsep;
|
||||
int res, ok;
|
||||
|
||||
/* Ensure that the __pycache__ directory exists. */
|
||||
cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname);
|
||||
if (!cpathname_ucs4)
|
||||
return;
|
||||
dirsep = rightmost_sep(cpathname_ucs4);
|
||||
if (dirsep == NULL) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname);
|
||||
return;
|
||||
}
|
||||
dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
|
||||
cpathname_ucs4,
|
||||
dirsep - cpathname_ucs4);
|
||||
PyMem_Free(cpathname_ucs4);
|
||||
if (dirname == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef MS_WINDOWS
|
||||
wdirname = PyUnicode_AsUnicode(dirname);
|
||||
if (wdirname == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
res = CreateDirectoryW(wdirname, NULL);
|
||||
ok = (res != 0);
|
||||
if (!ok && GetLastError() == ERROR_ALREADY_EXISTS)
|
||||
ok = 1;
|
||||
#else
|
||||
dirbytes = PyUnicode_EncodeFSDefault(dirname);
|
||||
if (dirbytes == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode);
|
||||
Py_DECREF(dirbytes);
|
||||
if (0 <= res)
|
||||
ok = 1;
|
||||
else
|
||||
ok = (errno == EEXIST);
|
||||
#endif
|
||||
if (!ok) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# cannot create cache dir %R\n", dirname);
|
||||
Py_DECREF(dirname);
|
||||
return;
|
||||
}
|
||||
Py_DECREF(dirname);
|
||||
|
||||
/* We first write to a tmp file and then take advantage
|
||||
of atomic renaming (which *should* be true even under Windows).
|
||||
As in importlib, we use id(something) to generate a pseudo-random
|
||||
filename. mkstemp() can't be used since it doesn't allow specifying
|
||||
the file access permissions.
|
||||
*/
|
||||
cpathname_tmp = PyUnicode_FromFormat("%U.%zd",
|
||||
cpathname, (Py_ssize_t) co);
|
||||
if (cpathname_tmp == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
#ifdef MS_WINDOWS
|
||||
wpathname = PyUnicode_AsUnicode(cpathname);
|
||||
if (wpathname == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp);
|
||||
if (wpathname_tmp == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
|
||||
(void)DeleteFileW(wpathname_tmp);
|
||||
fd = _wopen(wpathname_tmp,
|
||||
O_EXCL | O_CREAT | O_WRONLY | O_BINARY,
|
||||
mode);
|
||||
if (0 <= fd)
|
||||
fp = fdopen(fd, "wb");
|
||||
else
|
||||
fp = NULL;
|
||||
#else
|
||||
cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp);
|
||||
Py_DECREF(cpathname_tmp);
|
||||
if (cpathbytes_tmp == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
cpathbytes = PyUnicode_EncodeFSDefault(cpathname);
|
||||
if (cpathbytes == NULL) {
|
||||
PyErr_Clear();
|
||||
return;
|
||||
}
|
||||
fd = open(PyBytes_AS_STRING(cpathbytes_tmp),
|
||||
O_CREAT | O_EXCL | O_WRONLY, 0666);
|
||||
if (0 <= fd)
|
||||
fp = fdopen(fd, "wb");
|
||||
else
|
||||
fp = NULL;
|
||||
#endif
|
||||
if (fp == NULL) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr(
|
||||
"# can't create %R\n", cpathname);
|
||||
#ifdef MS_WINDOWS
|
||||
Py_DECREF(cpathname_tmp);
|
||||
#else
|
||||
Py_DECREF(cpathbytes);
|
||||
Py_DECREF(cpathbytes_tmp);
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
|
||||
/* First write a 0 for mtime and size */
|
||||
PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
|
||||
PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
|
||||
PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
|
||||
fflush(fp);
|
||||
/* Now write the true mtime and size (as 32-bit fields) */
|
||||
fseek(fp, 4L, 0);
|
||||
assert(mtime <= 0xFFFFFFFF);
|
||||
PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
|
||||
PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION);
|
||||
if (fflush(fp) != 0 || ferror(fp)) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# can't write %R\n", cpathname);
|
||||
/* Don't keep partial file */
|
||||
fclose(fp);
|
||||
#ifdef MS_WINDOWS
|
||||
(void)DeleteFileW(wpathname_tmp);
|
||||
Py_DECREF(cpathname_tmp);
|
||||
#else
|
||||
(void) unlink(PyBytes_AS_STRING(cpathbytes_tmp));
|
||||
Py_DECREF(cpathbytes);
|
||||
Py_DECREF(cpathbytes_tmp);
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
fclose(fp);
|
||||
/* Do a (hopefully) atomic rename */
|
||||
#ifdef MS_WINDOWS
|
||||
if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# can't write %R\n", cpathname);
|
||||
/* Don't keep tmp file */
|
||||
(void) DeleteFileW(wpathname_tmp);
|
||||
Py_DECREF(cpathname_tmp);
|
||||
return;
|
||||
}
|
||||
Py_DECREF(cpathname_tmp);
|
||||
#else
|
||||
if (rename(PyBytes_AS_STRING(cpathbytes_tmp),
|
||||
PyBytes_AS_STRING(cpathbytes))) {
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# can't write %R\n", cpathname);
|
||||
/* Don't keep tmp file */
|
||||
unlink(PyBytes_AS_STRING(cpathbytes_tmp));
|
||||
Py_DECREF(cpathbytes);
|
||||
Py_DECREF(cpathbytes_tmp);
|
||||
return;
|
||||
}
|
||||
Py_DECREF(cpathbytes);
|
||||
Py_DECREF(cpathbytes_tmp);
|
||||
#endif
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("# wrote %R\n", cpathname);
|
||||
}
|
||||
|
||||
static void
|
||||
update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname)
|
||||
{
|
||||
|
|
@ -1474,76 +1178,6 @@ imp_fix_co_filename(PyObject *self, PyObject *args)
|
|||
Py_RETURN_NONE;
|
||||
}
|
||||
|
||||
/* Load a source module from a given file and return its module
|
||||
object WITH INCREMENTED REFERENCE COUNT. If there's a matching
|
||||
byte-compiled file, use that instead. */
|
||||
|
||||
static PyObject *
|
||||
load_source_module(PyObject *name, PyObject *pathname, FILE *fp)
|
||||
{
|
||||
struct stat st;
|
||||
FILE *fpc;
|
||||
PyObject *cpathname = NULL, *cpathbytes = NULL;
|
||||
PyCodeObject *co;
|
||||
PyObject *m = NULL;
|
||||
|
||||
if (fstat(fileno(fp), &st) != 0) {
|
||||
PyErr_Format(PyExc_RuntimeError,
|
||||
"unable to get file status from %R",
|
||||
pathname);
|
||||
goto error;
|
||||
}
|
||||
if (sizeof st.st_mtime > 4) {
|
||||
/* Python's .pyc timestamp handling presumes that the timestamp fits
|
||||
in 4 bytes. Since the code only does an equality comparison,
|
||||
ordering is not important and we can safely ignore the higher bits
|
||||
(collisions are extremely unlikely).
|
||||
*/
|
||||
st.st_mtime &= 0xFFFFFFFF;
|
||||
}
|
||||
if (PyUnicode_READY(pathname) < 0)
|
||||
return NULL;
|
||||
cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag);
|
||||
|
||||
if (cpathname != NULL)
|
||||
fpc = check_compiled_module(pathname, &st, cpathname);
|
||||
else
|
||||
fpc = NULL;
|
||||
|
||||
if (fpc) {
|
||||
co = read_compiled_module(cpathname, fpc);
|
||||
fclose(fpc);
|
||||
if (co == NULL)
|
||||
goto error;
|
||||
update_compiled_module(co, pathname);
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("import %U # precompiled from %R\n",
|
||||
name, cpathname);
|
||||
m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
|
||||
cpathname, cpathname);
|
||||
}
|
||||
else {
|
||||
co = parse_source_module(pathname, fp);
|
||||
if (co == NULL)
|
||||
goto error;
|
||||
if (Py_VerboseFlag)
|
||||
PySys_FormatStderr("import %U # from %R\n",
|
||||
name, pathname);
|
||||
if (cpathname != NULL) {
|
||||
PyObject *ro = PySys_GetObject("dont_write_bytecode");
|
||||
if (ro == NULL || !PyObject_IsTrue(ro))
|
||||
write_compiled_module(co, cpathname, &st);
|
||||
}
|
||||
m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
|
||||
pathname, cpathname);
|
||||
}
|
||||
Py_DECREF(co);
|
||||
|
||||
error:
|
||||
Py_XDECREF(cpathbytes);
|
||||
Py_XDECREF(cpathname);
|
||||
return m;
|
||||
}
|
||||
|
||||
/* Get source file -> unicode or None
|
||||
* Returns the path to the py file if available, else the given path
|
||||
|
|
@ -3426,29 +3060,6 @@ imp_load_dynamic(PyObject *self, PyObject *args)
|
|||
|
||||
#endif /* HAVE_DYNAMIC_LOADING */
|
||||
|
||||
static PyObject *
|
||||
imp_load_source(PyObject *self, PyObject *args)
|
||||
{
|
||||
PyObject *name, *pathname;
|
||||
PyObject *fob = NULL;
|
||||
PyObject *m;
|
||||
FILE *fp;
|
||||
if (!PyArg_ParseTuple(args, "UO&|O:load_source",
|
||||
&name,
|
||||
PyUnicode_FSDecoder, &pathname,
|
||||
&fob))
|
||||
return NULL;
|
||||
fp = get_file(pathname, fob, "r");
|
||||
if (fp == NULL) {
|
||||
Py_DECREF(pathname);
|
||||
return NULL;
|
||||
}
|
||||
m = load_source_module(name, pathname, fp);
|
||||
Py_DECREF(pathname);
|
||||
fclose(fp);
|
||||
return m;
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
imp_reload(PyObject *self, PyObject *v)
|
||||
{
|
||||
|
|
@ -3600,7 +3211,6 @@ static PyMethodDef imp_methods[] = {
|
|||
#ifdef HAVE_DYNAMIC_LOADING
|
||||
{"load_dynamic", imp_load_dynamic, METH_VARARGS},
|
||||
#endif
|
||||
{"load_source", imp_load_source, METH_VARARGS},
|
||||
{"_fix_co_filename", imp_fix_co_filename, METH_VARARGS},
|
||||
{NULL, NULL} /* sentinel */
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue