Issue #13959: Re-implement imp.load_source() in imp.py.

Lib/imp.py

@@ -14,7 +14,7 @@ from _imp import (lock_held, acquire_lock, release_lock, reload,
 from _imp import (get_magic, get_tag, get_suffixes, cache_from_source,
                   source_from_cache)
 # Should be re-implemented here (and mostly deprecated)
-from _imp import (find_module, load_compiled, load_source, NullImporter,
+from _imp import (find_module, load_compiled, NullImporter,
                   SEARCH_ERROR, PY_SOURCE, PY_COMPILED, C_EXTENSION,
                   PY_RESOURCE, PKG_DIRECTORY, C_BUILTIN, PY_FROZEN,
                   PY_CODERESOURCE, IMP_HOOK)
@@ -25,6 +25,33 @@ from importlib import _bootstrap
 import os
 
 
+class _LoadSourceCompatibility(_bootstrap._SourceFileLoader):
+
+    """Compatibility support for implementing load_source()."""
+
+    def __init__(self, fullname, path, file=None):
+        super().__init__(fullname, path)
+        self.file = file
+
+    def get_data(self, path):
+        """Gross hack to contort SourceFileLoader to deal w/ load_source()'s bad
+        API."""
+        if path == self._path:
+            with self.file:
+                # Technically should be returning bytes, but
+                # SourceLoader.get_code() just passed what is returned to
+                # compile() which can handle str. And converting to bytes would
+                # require figuring out the encoding to decode to and
+                # tokenize.detect_encoding() only accepts bytes.
+                return self.file.read()
+        else:
+            return super().get_data(path)
+
+
+def load_source(name, pathname, file=None):
+    return _LoadSourceCompatibility(name, pathname, file).load_module(name)
+
+
 def load_package(name, path):
     if os.path.isdir(path):
         extensions = _bootstrap._suffix_list(PY_SOURCE)

Python/import.c

@@ -904,26 +904,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname,
 }
 
 
-/* Like strrchr(string, '/') but searches for the rightmost of either SEP
-   or ALTSEP, if the latter is defined.
-*/
-static Py_UCS4*
-rightmost_sep(Py_UCS4 *s)
-{
-    Py_UCS4 *found, c;
-    for (found = NULL; (c = *s); s++) {
-        if (c == SEP
-#ifdef ALTSEP
-            || c == ALTSEP
-#endif
-            )
-        {
-            found = s;
-        }
-    }
-    return found;
-}
-
 /* Like rightmost_sep, but operate on unicode objects. */
 static Py_ssize_t
 rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end)
@@ -1081,50 +1061,6 @@ make_source_pathname(PyObject *path)
     return result;
 }
 
-/* Given a pathname for a Python source file, its time of last
-   modification, and a pathname for a compiled file, check whether the
-   compiled file represents the same version of the source.  If so,
-   return a FILE pointer for the compiled file, positioned just after
-   the header; if not, return NULL.
-   Doesn't set an exception. */
-
-static FILE *
-check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname)
-{
-    FILE *fp;
-    long magic;
-    long pyc_mtime;
-    long pyc_size;
-
-    fp = _Py_fopen(cpathname, "rb");
-    if (fp == NULL)
-        return NULL;
-    magic = PyMarshal_ReadLongFromFile(fp);
-    if (magic != pyc_magic) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# %R has bad magic\n", cpathname);
-        fclose(fp);
-        return NULL;
-    }
-    pyc_mtime = PyMarshal_ReadLongFromFile(fp);
-    if (pyc_mtime != srcstat->st_mtime) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# %R has bad mtime\n", cpathname);
-        fclose(fp);
-        return NULL;
-    }
-    pyc_size = PyMarshal_ReadLongFromFile(fp);
-    if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# %R has bad size\n", cpathname);
-        fclose(fp);
-        return NULL;
-    }
-    if (Py_VerboseFlag)
-        PySys_FormatStderr("# %R matches %R\n", cpathname, pathname);
-    return fp;
-}
-
 
 /* Read a code object from a file and check it for validity */
 
@@ -1178,238 +1114,6 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp)
     return m;
 }
 
-/* Parse a source file and return the corresponding code object */
-
-static PyCodeObject *
-parse_source_module(PyObject *pathname, FILE *fp)
-{
-    PyCodeObject *co;
-    PyObject *pathbytes;
-    mod_ty mod;
-    PyCompilerFlags flags;
-    PyArena *arena;
-
-    pathbytes = PyUnicode_EncodeFSDefault(pathname);
-    if (pathbytes == NULL)
-        return NULL;
-
-    arena = PyArena_New();
-    if (arena == NULL) {
-        Py_DECREF(pathbytes);
-        return NULL;
-    }
-
-    flags.cf_flags = 0;
-    mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL,
-                               Py_file_input, 0, 0, &flags,
-                               NULL, arena);
-    if (mod != NULL)
-        co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena);
-    else
-        co = NULL;
-    Py_DECREF(pathbytes);
-    PyArena_Free(arena);
-    return co;
-}
-
-/* Write a compiled module to a file, placing the time of last
-   modification of its source into the header.
-   Errors are ignored, if a write error occurs an attempt is made to
-   remove the file. */
-
-static void
-write_compiled_module(PyCodeObject *co, PyObject *cpathname,
-                      struct stat *srcstat)
-{
-    Py_UCS4 *cpathname_ucs4;
-    FILE *fp;
-    time_t mtime = srcstat->st_mtime;
-    long size = srcstat->st_size & 0xFFFFFFFF;
-    PyObject *cpathname_tmp;
-#ifdef MS_WINDOWS   /* since Windows uses different permissions  */
-    mode_t mode = srcstat->st_mode & ~S_IEXEC;
-    wchar_t *wdirname, *wpathname, *wpathname_tmp;
-#else
-    mode_t dirmode = (srcstat->st_mode |
-                      S_IXUSR | S_IXGRP | S_IXOTH |
-                      S_IWUSR | S_IWGRP | S_IWOTH);
-    PyObject *dirbytes;
-    PyObject *cpathbytes, *cpathbytes_tmp;
-#endif
-    int fd;
-    PyObject *dirname;
-    Py_UCS4 *dirsep;
-    int res, ok;
-
-    /* Ensure that the __pycache__ directory exists. */
-    cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname);
-    if (!cpathname_ucs4)
-        return;
-    dirsep = rightmost_sep(cpathname_ucs4);
-    if (dirsep == NULL) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname);
-        return;
-    }
-    dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
-                                        cpathname_ucs4,
-                                        dirsep - cpathname_ucs4);
-    PyMem_Free(cpathname_ucs4);
-    if (dirname == NULL) {
-        PyErr_Clear();
-        return;
-    }
-
-#ifdef MS_WINDOWS
-    wdirname = PyUnicode_AsUnicode(dirname);
-    if (wdirname == NULL) {
-        PyErr_Clear();
-        return;
-    }
-    res = CreateDirectoryW(wdirname, NULL);
-    ok = (res != 0);
-    if (!ok && GetLastError() == ERROR_ALREADY_EXISTS)
-        ok = 1;
-#else
-    dirbytes = PyUnicode_EncodeFSDefault(dirname);
-    if (dirbytes == NULL) {
-        PyErr_Clear();
-        return;
-    }
-    res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode);
-    Py_DECREF(dirbytes);
-    if (0 <= res)
-        ok = 1;
-    else
-        ok = (errno == EEXIST);
-#endif
-    if (!ok) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# cannot create cache dir %R\n", dirname);
-        Py_DECREF(dirname);
-        return;
-    }
-    Py_DECREF(dirname);
-
-    /* We first write to a tmp file and then take advantage
-       of atomic renaming (which *should* be true even under Windows).
-       As in importlib, we use id(something) to generate a pseudo-random
-       filename.  mkstemp() can't be used since it doesn't allow specifying
-       the file access permissions.
-    */
-    cpathname_tmp = PyUnicode_FromFormat("%U.%zd",
-                                         cpathname, (Py_ssize_t) co);
-    if (cpathname_tmp == NULL) {
-        PyErr_Clear();
-        return;
-    }
-#ifdef MS_WINDOWS
-    wpathname = PyUnicode_AsUnicode(cpathname);
-    if (wpathname == NULL) {
-        PyErr_Clear();
-        return;
-    }
-    wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp);
-    if (wpathname_tmp == NULL) {
-        PyErr_Clear();
-        return;
-    }
-
-    (void)DeleteFileW(wpathname_tmp);
-    fd = _wopen(wpathname_tmp,
-                O_EXCL | O_CREAT | O_WRONLY | O_BINARY,
-                mode);
-    if (0 <= fd)
-        fp = fdopen(fd, "wb");
-    else
-        fp = NULL;
-#else
-    cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp);
-    Py_DECREF(cpathname_tmp);
-    if (cpathbytes_tmp == NULL) {
-        PyErr_Clear();
-        return;
-    }
-    cpathbytes = PyUnicode_EncodeFSDefault(cpathname);
-    if (cpathbytes == NULL) {
-        PyErr_Clear();
-        return;
-    }
-    fd = open(PyBytes_AS_STRING(cpathbytes_tmp),
-              O_CREAT | O_EXCL | O_WRONLY, 0666);
-    if (0 <= fd)
-        fp = fdopen(fd, "wb");
-    else
-        fp = NULL;
-#endif
-    if (fp == NULL) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr(
-                "# can't create %R\n", cpathname);
-#ifdef MS_WINDOWS
-        Py_DECREF(cpathname_tmp);
-#else
-        Py_DECREF(cpathbytes);
-        Py_DECREF(cpathbytes_tmp);
-#endif
-        return;
-    }
-    PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION);
-    /* First write a 0 for mtime and size */
-    PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
-    PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION);
-    PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION);
-    fflush(fp);
-    /* Now write the true mtime and size (as 32-bit fields) */
-    fseek(fp, 4L, 0);
-    assert(mtime <= 0xFFFFFFFF);
-    PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION);
-    PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION);
-    if (fflush(fp) != 0 || ferror(fp)) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# can't write %R\n", cpathname);
-        /* Don't keep partial file */
-        fclose(fp);
-#ifdef MS_WINDOWS
-        (void)DeleteFileW(wpathname_tmp);
-        Py_DECREF(cpathname_tmp);
-#else
-        (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp));
-        Py_DECREF(cpathbytes);
-        Py_DECREF(cpathbytes_tmp);
-#endif
-        return;
-    }
-    fclose(fp);
-    /* Do a (hopefully) atomic rename */
-#ifdef MS_WINDOWS
-    if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# can't write %R\n", cpathname);
-        /* Don't keep tmp file */
-        (void) DeleteFileW(wpathname_tmp);
-        Py_DECREF(cpathname_tmp);
-        return;
-    }
-    Py_DECREF(cpathname_tmp);
-#else
-    if (rename(PyBytes_AS_STRING(cpathbytes_tmp),
-               PyBytes_AS_STRING(cpathbytes))) {
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("# can't write %R\n", cpathname);
-        /* Don't keep tmp file */
-        unlink(PyBytes_AS_STRING(cpathbytes_tmp));
-        Py_DECREF(cpathbytes);
-        Py_DECREF(cpathbytes_tmp);
-        return;
-    }
-    Py_DECREF(cpathbytes);
-    Py_DECREF(cpathbytes_tmp);
-#endif
-    if (Py_VerboseFlag)
-        PySys_FormatStderr("# wrote %R\n", cpathname);
-}
-
 static void
 update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname)
 {
@@ -1474,76 +1178,6 @@ imp_fix_co_filename(PyObject *self, PyObject *args)
     Py_RETURN_NONE;
 }
 
-/* Load a source module from a given file and return its module
-   object WITH INCREMENTED REFERENCE COUNT.  If there's a matching
-   byte-compiled file, use that instead. */
-
-static PyObject *
-load_source_module(PyObject *name, PyObject *pathname, FILE *fp)
-{
-    struct stat st;
-    FILE *fpc;
-    PyObject *cpathname = NULL, *cpathbytes = NULL;
-    PyCodeObject *co;
-    PyObject *m = NULL;
-
-    if (fstat(fileno(fp), &st) != 0) {
-        PyErr_Format(PyExc_RuntimeError,
-                     "unable to get file status from %R",
-                     pathname);
-        goto error;
-    }
-    if (sizeof st.st_mtime > 4) {
-        /* Python's .pyc timestamp handling presumes that the timestamp fits
-           in 4 bytes. Since the code only does an equality comparison,
-           ordering is not important and we can safely ignore the higher bits
-           (collisions are extremely unlikely).
-         */
-        st.st_mtime &= 0xFFFFFFFF;
-    }
-    if (PyUnicode_READY(pathname) < 0)
-        return NULL;
-    cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag);
-
-    if (cpathname != NULL)
-        fpc = check_compiled_module(pathname, &st, cpathname);
-    else
-        fpc = NULL;
-
-    if (fpc) {
-        co = read_compiled_module(cpathname, fpc);
-        fclose(fpc);
-        if (co == NULL)
-            goto error;
-        update_compiled_module(co, pathname);
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("import %U # precompiled from %R\n",
-                               name, cpathname);
-        m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
-                                          cpathname, cpathname);
-    }
-    else {
-        co = parse_source_module(pathname, fp);
-        if (co == NULL)
-            goto error;
-        if (Py_VerboseFlag)
-            PySys_FormatStderr("import %U # from %R\n",
-                               name, pathname);
-        if (cpathname != NULL) {
-            PyObject *ro = PySys_GetObject("dont_write_bytecode");
-            if (ro == NULL || !PyObject_IsTrue(ro))
-                write_compiled_module(co, cpathname, &st);
-        }
-        m = PyImport_ExecCodeModuleObject(name, (PyObject *)co,
-                                          pathname, cpathname);
-    }
-    Py_DECREF(co);
-
-error:
-    Py_XDECREF(cpathbytes);
-    Py_XDECREF(cpathname);
-    return m;
-}
 
 /* Get source file -> unicode or None
  * Returns the path to the py file if available, else the given path
@@ -3426,29 +3060,6 @@ imp_load_dynamic(PyObject *self, PyObject *args)
 
 #endif /* HAVE_DYNAMIC_LOADING */
 
-static PyObject *
-imp_load_source(PyObject *self, PyObject *args)
-{
-    PyObject *name, *pathname;
-    PyObject *fob = NULL;
-    PyObject *m;
-    FILE *fp;
-    if (!PyArg_ParseTuple(args, "UO&|O:load_source",
-                          &name,
-                          PyUnicode_FSDecoder, &pathname,
-                          &fob))
-        return NULL;
-    fp = get_file(pathname, fob, "r");
-    if (fp == NULL) {
-        Py_DECREF(pathname);
-        return NULL;
-    }
-    m = load_source_module(name, pathname, fp);
-    Py_DECREF(pathname);
-    fclose(fp);
-    return m;
-}
-
 static PyObject *
 imp_reload(PyObject *self, PyObject *v)
 {
@@ -3600,7 +3211,6 @@ static PyMethodDef imp_methods[] = {
 #ifdef HAVE_DYNAMIC_LOADING
     {"load_dynamic",            imp_load_dynamic,       METH_VARARGS},
 #endif
-    {"load_source",             imp_load_source,        METH_VARARGS},
     {"_fix_co_filename",        imp_fix_co_filename,    METH_VARARGS},
     {NULL,                      NULL}           /* sentinel */
 };