Merge with 3.6

Doc/using/windows.rst

@@ -720,15 +720,24 @@ installation directory.  So, if you had installed Python to
 :file:`C:\\Python\\Lib\\` and third-party modules should be stored in
 :file:`C:\\Python\\Lib\\site-packages\\`.
 
-To completely override :data:`sys.path`, create a text file named ``'sys.path'``
+To completely override :data:`sys.path`, create a ``._pth`` file with the same
-containing a list of paths alongside the Python executable. This will ignore all
+name as the DLL (``python36._pth``) or the executable (``python._pth``) and
-registry settings and environment variables, enable isolated mode, disable
+specify one line for each path to add to :data:`sys.path`. The file based on the
-importing :mod:`site`, and fill :data:`sys.path` with exactly the paths listed
+DLL name overrides the one based on the executable, which allows paths to be
-in the file. Paths may be absolute or relative to the directory containing the
+restricted for any program loading the runtime if desired.
-file.
 
-When the ``'sys.path'`` file is missing, this is how :data:`sys.path` is
+When the file exists, all registry and environment variables are ignored,
-populated on Windows:
+isolated mode is enabled, and :mod:`site` is not imported unless one line in the
+file specifies ``import site``. Blank paths and lines starting with ``#`` are
+ignored. Each path may be absolute or relative to the location of the file.
+Import statements other than to ``site`` are not permitted, and arbitrary code
+cannot be specified.
+
+Note that ``.pth`` files (without leading underscore) will be processed normally
+by the :mod:`site` module.
+
+When no ``._pth`` file is found, this is how :data:`sys.path` is populated on
+Windows:
 
 * An empty entry is added at the start, which corresponds to the current
   directory.
@@ -782,9 +791,10 @@ The end result of all this is:
 For those who want to bundle Python into their application or distribution, the
 following advice will prevent conflicts with other installations:
 
-* Include a ``sys.path`` file alongside your executable containing the
+* Include a ``._pth`` file alongside your executable containing the
-  directories to include. This will ignore user site-packages and other paths
+  directories to include. This will ignore paths listed in the registry and
-  listed in the registry or in environment variables.
+  environment variables, and also ignore :mod:`site` unless ``import site`` is
+  listed.
 
 * If you are loading :file:`python3.dll` or :file:`python36.dll` in your own
   executable, explicitly call :c:func:`Py_SetPath` or (at least)

Doc/whatsnew/3.6.rst

@@ -108,7 +108,7 @@ Windows improvements:
   which means that when the 260 character path limit may no longer apply.
   See :ref:`removing the MAX_PATH limitation <max-path>` for details.
 
-* A ``sys.path`` file can be added to force isolated mode and fully specify
+* A ``._pth`` file can be added to force isolated mode and fully specify
   all search paths to avoid registry and environment lookup. See
   :ref:`the documentation <finding_modules>` for more information.
 

Lib/site.py

@@ -60,7 +60,8 @@ omitted because it is not mentioned in either path configuration file.
 
 The readline module is also automatically configured to enable
 completion for systems that support it.  This can be overridden in
-sitecustomize, usercustomize or PYTHONSTARTUP.
+sitecustomize, usercustomize or PYTHONSTARTUP.  Starting Python in
+isolated mode (-I) disables automatic readline configuration.
 
 After these operations, an attempt is made to import a module
 named sitecustomize, which can perform arbitrary additional
@@ -491,7 +492,7 @@ def execsitecustomize():
             else:
                 raise
     except Exception as err:
-        if os.environ.get("PYTHONVERBOSE"):
+        if sys.flags.verbose:
             sys.excepthook(*sys.exc_info())
         else:
             sys.stderr.write(
@@ -511,7 +512,7 @@ def execusercustomize():
             else:
                 raise
     except Exception as err:
-        if os.environ.get("PYTHONVERBOSE"):
+        if sys.flags.verbose:
             sys.excepthook(*sys.exc_info())
         else:
             sys.stderr.write(
@@ -538,12 +539,13 @@ def main():
     setquit()
     setcopyright()
     sethelper()
-    enablerlcompleter()
+    if not sys.flags.isolated:
+        enablerlcompleter()
     execsitecustomize()
     if ENABLE_USER_SITE:
         execusercustomize()
 
-# Prevent edition of sys.path when python was started with -S and
+# Prevent extending of sys.path when python was started with -S and
 # site is imported later.
 if not sys.flags.no_site:
     main()

Lib/test/test_site.py

@@ -140,8 +140,6 @@ class HelperFunctionsTests(unittest.TestCase):
         self.assertRegex(err_out.getvalue(), 'Traceback')
         self.assertRegex(err_out.getvalue(), 'ModuleNotFoundError')
 
-    @unittest.skipIf(sys.platform == "win32", "Windows does not raise an "
-                      "error for file paths containing null characters")
     def test_addpackage_import_bad_pth_file(self):
         # Issue 5258
         pth_dir, pth_fn = self.make_pth("abc\x00def\n")
@@ -447,10 +445,9 @@ class StartupImportTests(unittest.TestCase):
         popen = subprocess.Popen([sys.executable, '-I', '-v', '-c',
                                   'import sys; print(set(sys.modules))'],
                                  stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE)
+                                 stderr=subprocess.PIPE,
+                                 encoding='utf-8')
         stdout, stderr = popen.communicate()
-        stdout = stdout.decode('utf-8')
-        stderr = stderr.decode('utf-8')
         modules = eval(stdout)
 
         self.assertIn('site', modules)
@@ -474,6 +471,5 @@ class StartupImportTests(unittest.TestCase):
         if sys.platform != 'darwin':
             self.assertFalse(modules.intersection(collection_mods), stderr)
 
-
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS

@@ -10,6 +10,8 @@ What's New in Python 3.7.0 alpha 1
 Core and Builtins
 -----------------
 
+- Issue #28192: Don't import readline in isolated mode.
+
 - Issue #27441: Remove some redundant assignments to ob_size in longobject.c.
   Thanks Oren Milman.
 
@@ -70,6 +72,13 @@ Library
 - Issue #27759: Fix selectors incorrectly retain invalid file descriptors.
   Patch by Mark Williams.
 
+Windows
+-------
+
+- Issue #28137: Renames Windows path file to ._pth
+
+- Issue #28138: Windows ._pth file should allow import site
+
 Build
 -----
 

Modules/main.c

@@ -703,7 +703,8 @@ Py_Main(int argc, wchar_t **argv)
     PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind);
 
     if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) &&
-        isatty(fileno(stdin))) {
+        isatty(fileno(stdin)) &&
+        !Py_IsolatedFlag) {
         PyObject *v;
         v = PyImport_ImportModule("readline");
         if (v == NULL)

PC/getpathp.c

@@ -6,8 +6,9 @@
    PATH RULES FOR WINDOWS:
    This describes how sys.path is formed on Windows.  It describes the
    functionality, not the implementation (ie, the order in which these
-   are actually fetched is different). The presence of a sys.path file
+   are actually fetched is different). The presence of a python._pth or
-   alongside the program overrides these rules - see below.
+   pythonXY._pth file alongside the program overrides these rules - see
+   below.
 
    * Python always adds an empty entry at the start, which corresponds
      to the current directory.
@@ -37,11 +38,21 @@
      used (eg. .\Lib;.\DLLs, etc)
 
 
-   If a sys.path file exists adjacent to python.exe, it must contain a
+   If a '._pth' file exists adjacent to the executable with the same base name
-   list of paths to add to sys.path, one per line (like a .pth file but without
+   (e.g. python._pth adjacent to python.exe) or adjacent to the shared library
-   the ability to execute arbitrary code). Each path is relative to the
+   (e.g. python36._pth adjacent to python36.dll), it is used in preference to
-   directory containing the file. No other paths are added to the search path,
+   the above process. The shared library file takes precedence over the
-   and the registry finder is not enabled.
+   executable. The path file must contain a list of paths to add to sys.path,
+   one per line. Each path is relative to the directory containing the file.
+   Blank lines and comments beginning with '#' are permitted.
+
+   In the presence of this ._pth file, no other paths are added to the search
+   path, the registry finder is not enabled, site.py is not imported and
+   isolated mode is enabled. The site package can be enabled by including a
+   line reading "import site"; no other imports are recognized. Any invalid
+   entry (other than directories that do not exist) will result in immediate
+   termination of the program.
+
 
   The end result of all this is:
   * When running python.exe, or any other .exe in the main Python directory
@@ -61,8 +72,9 @@
   * An embedding application can use Py_SetPath() to override all of
     these automatic path computations.
 
-  * An isolation install of Python can disable all implicit paths by
+  * An install of Python can fully specify the contents of sys.path using
-    providing a sys.path file.
+    either a 'EXENAME._pth' or 'DLLNAME._pth' file, optionally including
+    "import site" to enable the site module.
 
    ---------------------------------------------------------------- */
 
@@ -135,6 +147,33 @@ reduce(wchar_t *dir)
     dir[i] = '\0';
 }
 
+static int
+change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext)
+{
+    size_t src_len = wcsnlen_s(src, MAXPATHLEN+1);
+    size_t i = src_len;
+    if (i >= MAXPATHLEN+1)
+        Py_FatalError("buffer overflow in getpathp.c's reduce()");
+
+    while (i > 0 && src[i] != '.' && !is_sep(src[i]))
+        --i;
+
+    if (i == 0) {
+        dest[0] = '\0';
+        return -1;
+    }
+
+    if (is_sep(src[i]))
+        i = src_len;
+
+    if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) ||
+        wcscat_s(dest, MAXPATHLEN+1, ext)) {
+        dest[0] = '\0';
+        return -1;
+    }
+
+    return 0;
+}
 
 static int
 exists(wchar_t *filename)
@@ -499,12 +538,17 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value)
 }
 
 static int
-read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
+read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite)
 {
     FILE *sp_file = _Py_wfopen(path, L"r");
     if (sp_file == NULL)
         return -1;
 
+    wcscpy_s(prefix, MAXPATHLEN+1, path);
+    reduce(prefix);
+    *isolated = 1;
+    *nosite = 1;
+
     size_t bufsiz = MAXPATHLEN;
     size_t prefixlen = wcslen(prefix);
 
@@ -516,16 +560,25 @@ read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
         char *p = fgets(line, MAXPATHLEN + 1, sp_file);
         if (!p)
             break;
+        if (*p == '\0' || *p == '#')
+            continue;
+        while (*++p) {
+            if (*p == '\r' || *p == '\n') {
+                *p = '\0';
+                break;
+            }
+        }
 
-        DWORD n = strlen(line);
+        if (strcmp(line, "import site") == 0) {
-        if (n == 0 || p[n - 1] != '\n')
+            *nosite = 0;
-            break;
+            continue;
-        if (n > 2 && p[n - 1] == '\r')
+        } else if (strncmp(line, "import ", 7) == 0) {
-            --n;
+            Py_FatalError("only 'import site' is supported in ._pth file");
+        }
 
-        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, n - 1, NULL, 0);
+        DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0);
         wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t));
-        wn = MultiByteToWideChar(CP_UTF8, 0, line, n - 1, wline, wn);
+        wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1);
         wline[wn] = '\0';
 
         while (wn + prefixlen + 4 > bufsiz) {
@@ -539,8 +592,8 @@ read_sys_path_file(const wchar_t *path, const wchar_t *prefix)
 
         if (buf[0])
             wcscat_s(buf, bufsiz, L";");
+
         wchar_t *b = &buf[wcslen(buf)];
-        
         wcscat_s(buf, bufsiz, prefix);
         join(b, wline);
 
@@ -586,13 +639,12 @@ calculate_path(void)
     {
         wchar_t spbuffer[MAXPATHLEN+1];
 
-        wcscpy_s(spbuffer, MAXPATHLEN+1, argv0_path);
+        if ((dllpath[0] && !change_ext(spbuffer, dllpath, L"._pth") && exists(spbuffer)) ||
-        join(spbuffer, L"sys.path");
+            (progpath[0] && !change_ext(spbuffer, progpath, L"._pth") && exists(spbuffer))) {
-        if (exists(spbuffer) && read_sys_path_file(spbuffer, argv0_path) == 0) {
+
-            wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path);
+            if (!read_pth_file(spbuffer, prefix, &Py_IsolatedFlag, &Py_NoSiteFlag)) {
-            Py_IsolatedFlag = 1;
+                return;
-            Py_NoSiteFlag = 1;
+            }
-            return;
         }
     }
 
@@ -631,16 +683,7 @@ calculate_path(void)
     }
 
     /* Calculate zip archive path from DLL or exe path */
-    if (wcscpy_s(zip_path, MAXPATHLEN + 1, dllpath[0] ? dllpath : progpath)) {
+    change_ext(zip_path, dllpath[0] ? dllpath : progpath, L".zip");
-        /* exceeded buffer length - ignore zip_path */
-        zip_path[0] = '\0';
-    } else {
-        wchar_t *dot = wcsrchr(zip_path, '.');
-        if (!dot || wcscpy_s(dot, MAXPATHLEN + 1 - (dot - zip_path), L".zip")) {
-            /* exceeded buffer length - ignore zip_path */
-            zip_path[0] = L'\0';
-        }
-    }
 
     if (pythonhome == NULL || *pythonhome == '\0') {
         if (zip_path[0] && exists(zip_path)) {

Tools/msi/make_zip.py

@@ -91,11 +91,13 @@ def include_in_tools(p):
 
     return p.suffix.lower() in {'.py', '.pyw', '.txt'}
 
+BASE_NAME = 'python{0.major}{0.minor}'.format(sys.version_info)
+
 FULL_LAYOUT = [
     ('/', 'PCBuild/$arch', 'python.exe', is_not_debug),
     ('/', 'PCBuild/$arch', 'pythonw.exe', is_not_debug),
-    ('/', 'PCBuild/$arch', 'python{0.major}.dll'.format(sys.version_info), is_not_debug),
+    ('/', 'PCBuild/$arch', 'python{}.dll'.format(sys.version_info.major), is_not_debug),
-    ('/', 'PCBuild/$arch', 'python{0.major}{0.minor}.dll'.format(sys.version_info), is_not_debug),
+    ('/', 'PCBuild/$arch', '{}.dll'.format(BASE_NAME), is_not_debug),
     ('DLLs/', 'PCBuild/$arch', '*.pyd', is_not_debug),
     ('DLLs/', 'PCBuild/$arch', '*.dll', is_not_debug_or_python),
     ('include/', 'include', '*.h', None),
@@ -109,7 +111,7 @@ EMBED_LAYOUT = [
     ('/', 'PCBuild/$arch', 'python*.exe', is_not_debug),
     ('/', 'PCBuild/$arch', '*.pyd', is_not_debug),
     ('/', 'PCBuild/$arch', '*.dll', is_not_debug),
-    ('python{0.major}{0.minor}.zip'.format(sys.version_info), 'Lib', '**/*', include_in_lib),
+    ('{}.zip'.format(BASE_NAME), 'Lib', '**/*', include_in_lib),
 ]
 
 if os.getenv('DOC_FILENAME'):
@@ -209,9 +211,12 @@ def main():
             print('Copied {} files'.format(copied))
 
         if ns.embed:
-            with open(str(temp / 'sys.path'), 'w') as f:
+            with open(str(temp / (BASE_NAME + '._pth')), 'w') as f:
-                print('python{0.major}{0.minor}.zip'.format(sys.version_info), file=f)
+                print(BASE_NAME + '.zip', file=f)
                 print('.', file=f)
+                print('', file=f)
+                print('# Uncomment to run site.main() automatically', file=f)
+                print('#import site', file=f)
 
         if out:
             total = copy_to_layout(out, rglob(temp, '**/*', None))