bpo-35854: Fix EnvBuilder and --symlinks in venv on Windows (GH-11700)

(cherry picked from commit a1f9a3332b)

Co-authored-by: Steve Dower <steve.dower@microsoft.com>

Doc/library/venv.rst

@@ -114,8 +114,7 @@ creation according to their needs, the :class:`EnvBuilder` class.
       any existing target directory, before creating the environment.
 
     * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the
-      Python binary (and any necessary DLLs or other binaries,
-      e.g. ``pythonw.exe``), rather than copying.
+      Python binary rather than copying.
 
     * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing
       environment with the running Python - for use when that Python has been
@@ -181,15 +180,15 @@ creation according to their needs, the :class:`EnvBuilder` class.
 
     .. method:: setup_python(context)
 
-        Creates a copy of the Python executable in the environment on POSIX
-        systems. If a specific executable ``python3.x`` was used, symlinks to
-        ``python`` and ``python3`` will be created pointing to that executable,
-        unless files with those names already exist.
+        Creates a copy or symlink to the Python executable in the environment.
+        On POSIX systems, if a specific executable ``python3.x`` was used,
+        symlinks to ``python`` and ``python3`` will be created pointing to that
+        executable, unless files with those names already exist.
 
     .. method:: setup_scripts(context)
 
         Installs activation scripts appropriate to the platform into the virtual
-        environment. On Windows, also installs the ``python[w].exe`` scripts.
+        environment.
 
     .. method:: post_setup(context)
 
@@ -199,8 +198,13 @@ creation according to their needs, the :class:`EnvBuilder` class.
 
     .. versionchanged:: 3.7.2
        Windows now uses redirector scripts for ``python[w].exe`` instead of
-       copying the actual binaries, and so :meth:`setup_python` does nothing
-       unless running from a build in the source tree.
+       copying the actual binaries. In 3.7.2 only :meth:`setup_python` does
+       nothing unless running from a build in the source tree.
+
+    .. versionchanged:: 3.7.3
+       Windows copies the redirector scripts as part of :meth:`setup_python`
+       instead of :meth:`setup_scripts`. This was not the case in 3.7.2.
+       When using symlinks, the original executables will be linked.
 
     In addition, :class:`EnvBuilder` provides this utility method that can be
     called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to

Doc/using/venv-create.inc

@@ -70,6 +70,11 @@ The command, if run with ``-h``, will show the available options::
    In earlier versions, if the target directory already existed, an error was
    raised, unless the ``--clear`` or ``--upgrade`` option was provided.
 
+.. note::
+   While symlinks are supported on Windows, they are not recommended. Of
+   particular note is that double-clicking ``python.exe`` in File Explorer
+   will resolve the symlink eagerly and ignore the virtual environment.
+
 The created ``pyvenv.cfg`` file also includes the
 ``include-system-site-packages`` key, set to ``true`` if ``venv`` is
 run with the ``--system-site-packages`` option, ``false`` otherwise.

Lib/test/test_venv.py

@@ -243,7 +243,6 @@ class BasicTest(BaseTest):
             self.assertIn('include-system-site-packages = %s\n' % s, data)
 
     @unittest.skipUnless(can_symlink(), 'Needs symlinks')
-    @unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows')
     def test_symlinking(self):
         """
         Test symlinking works as expected

Lib/venv/__init__.py

@@ -64,11 +64,10 @@ class EnvBuilder:
         self.system_site_packages = False
         self.create_configuration(context)
         self.setup_python(context)
-        if not self.upgrade:
-            self.setup_scripts(context)
         if self.with_pip:
             self._setup_pip(context)
         if not self.upgrade:
+            self.setup_scripts(context)
             self.post_setup(context)
         if true_system_site_packages:
             # We had set it to False before, now
@@ -176,6 +175,23 @@ class EnvBuilder:
                 logger.warning('Unable to symlink %r to %r', src, dst)
                 force_copy = True
         if force_copy:
+            if os.name == 'nt':
+                # On Windows, we rewrite symlinks to our base python.exe into
+                # copies of venvlauncher.exe
+                basename, ext = os.path.splitext(os.path.basename(src))
+                if basename.endswith('_d'):
+                    ext = '_d' + ext
+                    basename = basename[:-2]
+                if sysconfig.is_python_build(True):
+                    if basename == 'python':
+                        basename = 'venvlauncher'
+                    elif basename == 'pythonw':
+                        basename = 'venvwlauncher'
+                    scripts = os.path.dirname(src)
+                else:
+                    scripts = os.path.join(os.path.dirname(__file__), "scripts", "nt")
+                src = os.path.join(scripts, basename + ext)
+
             shutil.copyfile(src, dst)
 
     def setup_python(self, context):
@@ -202,23 +218,31 @@ class EnvBuilder:
                     if not os.path.islink(path):
                         os.chmod(path, 0o755)
         else:
-            # For normal cases, the venvlauncher will be copied from
-            # our scripts folder. For builds, we need to copy it
-            # manually.
+            if self.symlinks:
+                # For symlinking, we need a complete copy of the root directory
+                # If symlinks fail, you'll get unnecessary copies of files, but
+                # we assume that if you've opted into symlinks on Windows then
+                # you know what you're doing.
+                suffixes = [
+                    f for f in os.listdir(dirname) if
+                    os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll')
+                ]
+                if sysconfig.is_python_build(True):
+                    suffixes = [
+                        f for f in suffixes if
+                        os.path.normcase(f).startswith(('python', 'vcruntime'))
+                    ]
+            else:
+                suffixes = ['python.exe', 'python_d.exe', 'pythonw.exe',
+                            'pythonw_d.exe']
+
+            for suffix in suffixes:
+                src = os.path.join(dirname, suffix)
+                if os.path.exists(src):
+                    copier(src, os.path.join(binpath, suffix))
+
             if sysconfig.is_python_build(True):
-                suffix = '.exe'
-                if context.python_exe.lower().endswith('_d.exe'):
-                    suffix = '_d.exe'
-
-                src = os.path.join(dirname, "venvlauncher" + suffix)
-                dst = os.path.join(binpath, context.python_exe)
-                copier(src, dst)
-
-                src = os.path.join(dirname, "venvwlauncher" + suffix)
-                dst = os.path.join(binpath, "pythonw" + suffix)
-                copier(src, dst)
-
-                # copy init.tcl over
+                # copy init.tcl
                 for root, dirs, files in os.walk(context.python_dir):
                     if 'init.tcl' in files:
                         tcldir = os.path.basename(root)
@@ -304,6 +328,9 @@ class EnvBuilder:
                         dirs.remove(d)
                 continue # ignore files in top level
             for f in files:
+                if (os.name == 'nt' and f.startswith('python')
+                        and f.endswith(('.exe', '.pdb'))):
+                    continue
                 srcfile = os.path.join(root, f)
                 suffix = root[plen:].split(os.sep)[2:]
                 if not suffix:

Misc/NEWS.d/next/Windows/2019-01-29-15-44-46.bpo-35854.Ww3z19.rst

@@ -0,0 +1 @@
+Fix EnvBuilder and --symlinks in venv on Windows