bpo-34011: Fixes missing venv files and other tests (GH-9458)

Lib/distutils/tests/test_bdist.py

@@ -39,6 +39,9 @@ class BuildTestCase(support.TempdirManager,
 
         for name in names:
             subcmd = cmd.get_finalized_command(name)
+            if getattr(subcmd, '_unsupported', False):
+                # command is not supported on this build
+                continue
             self.assertTrue(subcmd.skip_build,
                             '%s should take --skip-build from bdist' % name)
 

Lib/distutils/tests/test_bdist_wininst.py

@@ -5,6 +5,8 @@ from test.support import run_unittest
 from distutils.command.bdist_wininst import bdist_wininst
 from distutils.tests import support
 
+@unittest.skipIf(getattr(bdist_wininst, '_unsupported', False),
+    'bdist_wininst is not supported in this install')
 class BuildWinInstTestCase(support.TempdirManager,
                            support.LoggingSilencer,
                            unittest.TestCase):

Lib/test/test_platform.py

@@ -3,6 +3,7 @@ import os
 import platform
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import unittest
 import warnings
@@ -16,29 +17,34 @@ class PlatformTest(unittest.TestCase):
     @support.skip_unless_symlink
     def test_architecture_via_symlink(self): # issue3762
         # On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
-        # so we add the directory to the path and PYTHONPATH.
+        # so we add the directory to the path, PYTHONHOME and PYTHONPATH.
+        env = None
         if sys.platform == "win32":
-            def restore_environ(old_env):
+            env = {k.upper(): os.environ[k] for k in os.environ}
-                os.environ.clear()
+            env["PATH"] = "{};{}".format(
-                os.environ.update(old_env)
+                os.path.dirname(sys.executable), env.get("PATH", ""))
+            env["PYTHONHOME"] = os.path.dirname(sys.executable)
+            if sysconfig.is_python_build(True):
+                env["PYTHONPATH"] = os.path.dirname(os.__file__)
 
-            self.addCleanup(restore_environ, dict(os.environ))
+        def get(python, env=None):
-
-            os.environ["Path"] = "{};{}".format(
-                os.path.dirname(sys.executable), os.environ["Path"])
-            os.environ["PYTHONPATH"] = os.path.dirname(sys.executable)
-
-        def get(python):
             cmd = [python, '-c',
                 'import platform; print(platform.architecture())']
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-            return p.communicate()
+                                 stderr=subprocess.PIPE, env=env)
+            r = p.communicate()
+            if p.returncode:
+                print(repr(r[0]))
+                print(repr(r[1]), file=sys.stderr)
+                self.fail('unexpected return code: {0} (0x{0:08X})'
+                          .format(p.returncode))
+            return r
 
         real = os.path.realpath(sys.executable)
         link = os.path.abspath(support.TESTFN)
         os.symlink(real, link)
         try:
-            self.assertEqual(get(real), get(link))
+            self.assertEqual(get(real), get(link, env=env))
         finally:
             os.remove(link)
 

Lib/test/test_site.py

@@ -6,7 +6,8 @@ executing have not been removed.
 """
 import unittest
 import test.support
-from test.support import captured_stderr, TESTFN, EnvironmentVarGuard
+from test.support import (captured_stderr, TESTFN, EnvironmentVarGuard,
+                          change_cwd)
 import builtins
 import os
 import sys
@@ -346,40 +347,47 @@ class ImportSideEffectTests(unittest.TestCase):
         # __file__ if abs_paths() does not get run.  sys and builtins (the
         # only other modules imported before site.py runs) do not have
         # __file__ or __cached__ because they are built-in.
-        parent = os.path.relpath(os.path.dirname(os.__file__))
+        try:
-        env = os.environ.copy()
+            parent = os.path.relpath(os.path.dirname(os.__file__))
-        env['PYTHONPATH'] = parent
+            cwd = os.getcwd()
-        code = ('import os, sys',
+        except ValueError:
-            # use ASCII to avoid locale issues with non-ASCII directories
+            # Failure to get relpath probably means we need to chdir
-            'os_file = os.__file__.encode("ascii", "backslashreplace")',
+            # to the same drive.
-            r'sys.stdout.buffer.write(os_file + b"\n")',
+            cwd, parent = os.path.split(os.path.dirname(os.__file__))
-            'os_cached = os.__cached__.encode("ascii", "backslashreplace")',
+        with change_cwd(cwd):
-            r'sys.stdout.buffer.write(os_cached + b"\n")')
+            env = os.environ.copy()
-        command = '\n'.join(code)
+            env['PYTHONPATH'] = parent
-        # First, prove that with -S (no 'import site'), the paths are
+            code = ('import os, sys',
-        # relative.
+                # use ASCII to avoid locale issues with non-ASCII directories
-        proc = subprocess.Popen([sys.executable, '-S', '-c', command],
+                'os_file = os.__file__.encode("ascii", "backslashreplace")',
-                                env=env,
+                r'sys.stdout.buffer.write(os_file + b"\n")',
-                                stdout=subprocess.PIPE)
+                'os_cached = os.__cached__.encode("ascii", "backslashreplace")',
-        stdout, stderr = proc.communicate()
+                r'sys.stdout.buffer.write(os_cached + b"\n")')
+            command = '\n'.join(code)
+            # First, prove that with -S (no 'import site'), the paths are
+            # relative.
+            proc = subprocess.Popen([sys.executable, '-S', '-c', command],
+                                    env=env,
+                                    stdout=subprocess.PIPE)
+            stdout, stderr = proc.communicate()
 
-        self.assertEqual(proc.returncode, 0)
+            self.assertEqual(proc.returncode, 0)
-        os__file__, os__cached__ = stdout.splitlines()[:2]
+            os__file__, os__cached__ = stdout.splitlines()[:2]
-        self.assertFalse(os.path.isabs(os__file__))
+            self.assertFalse(os.path.isabs(os__file__))
-        self.assertFalse(os.path.isabs(os__cached__))
+            self.assertFalse(os.path.isabs(os__cached__))
-        # Now, with 'import site', it works.
+            # Now, with 'import site', it works.
-        proc = subprocess.Popen([sys.executable, '-c', command],
+            proc = subprocess.Popen([sys.executable, '-c', command],
-                                env=env,
+                                    env=env,
-                                stdout=subprocess.PIPE)
+                                    stdout=subprocess.PIPE)
-        stdout, stderr = proc.communicate()
+            stdout, stderr = proc.communicate()
-        self.assertEqual(proc.returncode, 0)
+            self.assertEqual(proc.returncode, 0)
-        os__file__, os__cached__ = stdout.splitlines()[:2]
+            os__file__, os__cached__ = stdout.splitlines()[:2]
-        self.assertTrue(os.path.isabs(os__file__),
+            self.assertTrue(os.path.isabs(os__file__),
-                        "expected absolute path, got {}"
+                            "expected absolute path, got {}"
-                        .format(os__file__.decode('ascii')))
+                            .format(os__file__.decode('ascii')))
-        self.assertTrue(os.path.isabs(os__cached__),
+            self.assertTrue(os.path.isabs(os__cached__),
-                        "expected absolute path, got {}"
+                            "expected absolute path, got {}"
-                        .format(os__cached__.decode('ascii')))
+                            .format(os__cached__.decode('ascii')))
 
     def test_no_duplicate_paths(self):
         # No duplicate paths should exist in sys.path

Lib/test/test_sysconfig.py

@@ -235,21 +235,34 @@ class TestSysConfig(unittest.TestCase):
     def test_symlink(self):
         # On Windows, the EXE needs to know where pythonXY.dll is at so we have
         # to add the directory to the path.
+        env = None
         if sys.platform == "win32":
-            os.environ["PATH"] = "{};{}".format(
+            env = {k.upper(): os.environ[k] for k in os.environ}
-                os.path.dirname(sys.executable), os.environ["PATH"])
+            env["PATH"] = "{};{}".format(
+                os.path.dirname(sys.executable), env.get("PATH", ""))
+            # Requires PYTHONHOME as well since we locate stdlib from the
+            # EXE path and not the DLL path (which should be fixed)
+            env["PYTHONHOME"] = os.path.dirname(sys.executable)
+            if sysconfig.is_python_build(True):
+                env["PYTHONPATH"] = os.path.dirname(os.__file__)
 
         # Issue 7880
-        def get(python):
+        def get(python, env=None):
             cmd = [python, '-c',
                    'import sysconfig; print(sysconfig.get_platform())']
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, env=os.environ)
+            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-            return p.communicate()
+                                 stderr=subprocess.PIPE, env=env)
+            out, err = p.communicate()
+            if p.returncode:
+                print((out, err))
+                self.fail('Non-zero return code {0} (0x{0:08X})'
+                            .format(p.returncode))
+            return out, err
         real = os.path.realpath(sys.executable)
         link = os.path.abspath(TESTFN)
         os.symlink(real, link)
         try:
-            self.assertEqual(get(real), get(link))
+            self.assertEqual(get(real), get(link, env))
         finally:
             unlink(link)
 

Lib/test/test_venv.py

@@ -27,6 +27,17 @@ except ImportError:
 skipInVenv = unittest.skipIf(sys.prefix != sys.base_prefix,
                              'Test not appropriate in a venv')
 
+def check_output(cmd, encoding=None):
+    p = subprocess.Popen(cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding=encoding)
+    out, err = p.communicate()
+    if p.returncode:
+        raise subprocess.CalledProcessError(
+            p.returncode, cmd, None, out, err)
+    return out, err
+
 class BaseTest(unittest.TestCase):
     """Base class for venv tests."""
     maxDiff = 80 * 50
@@ -134,9 +145,7 @@ class BasicTest(BaseTest):
             ('base_prefix', sys.prefix),
             ('base_exec_prefix', sys.exec_prefix)):
             cmd[2] = 'import sys; print(sys.%s)' % prefix
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+            out, err = check_output(cmd)
-                                 stderr=subprocess.PIPE)
-            out, err = p.communicate()
             self.assertEqual(out.strip(), expected.encode())
 
     if sys.platform == 'win32':
@@ -259,11 +268,10 @@ class BasicTest(BaseTest):
         """
         rmtree(self.env_dir)
         self.run_with_capture(venv.create, self.env_dir)
-        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
+        envpy = os.path.join(os.path.realpath(self.env_dir),
-        cmd = [envpy, '-c', 'import sys; print(sys.executable)']
+                             self.bindir, self.exe)
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+        out, err = check_output([envpy, '-c',
-                             stderr=subprocess.PIPE)
+            'import sys; print(sys.executable)'])
-        out, err = p.communicate()
         self.assertEqual(out.strip(), envpy.encode())
 
     @unittest.skipUnless(can_symlink(), 'Needs symlinks')
@@ -274,17 +282,16 @@ class BasicTest(BaseTest):
         rmtree(self.env_dir)
         builder = venv.EnvBuilder(clear=True, symlinks=True)
         builder.create(self.env_dir)
-        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
+        envpy = os.path.join(os.path.realpath(self.env_dir),
-        cmd = [envpy, '-c', 'import sys; print(sys.executable)']
+                             self.bindir, self.exe)
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+        out, err = check_output([envpy, '-c',
-                             stderr=subprocess.PIPE)
+            'import sys; print(sys.executable)'])
-        out, err = p.communicate()
         self.assertEqual(out.strip(), envpy.encode())
 
     @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
     def test_unicode_in_batch_file(self):
         """
-        Test isolation from system site-packages
+        Test handling of Unicode paths
         """
         rmtree(self.env_dir)
         env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
@@ -292,12 +299,10 @@ class BasicTest(BaseTest):
         builder.create(env_dir)
         activate = os.path.join(env_dir, self.bindir, 'activate.bat')
         envpy = os.path.join(env_dir, self.bindir, self.exe)
-        cmd = [activate, '&', self.exe, '-c', 'print(0)']
+        out, err = check_output(
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+            [activate, '&', self.exe, '-c', 'print(0)'],
-                             stderr=subprocess.PIPE, encoding='oem',
+            encoding='oem',
-                             shell=True)
+        )
-        out, err = p.communicate()
-        print(err)
         self.assertEqual(out.strip(), '0')
 
 @skipInVenv
@@ -306,11 +311,8 @@ class EnsurePipTest(BaseTest):
     def assert_pip_not_installed(self):
         envpy = os.path.join(os.path.realpath(self.env_dir),
                              self.bindir, self.exe)
-        try_import = 'try:\n import pip\nexcept ImportError:\n print("OK")'
+        out, err = check_output([envpy, '-c',
-        cmd = [envpy, '-c', try_import]
+            'try:\n import pip\nexcept ImportError:\n print("OK")'])
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-        out, err = p.communicate()
         # We force everything to text, so unittest gives the detailed diff
         # if we get unexpected results
         err = err.decode("latin-1") # Force to text, prevent decoding errors
@@ -388,11 +390,8 @@ class EnsurePipTest(BaseTest):
         # Ensure pip is available in the virtual environment
         envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
         # Ignore DeprecationWarning since pip code is not part of Python
-        cmd = [envpy, '-W', 'ignore::DeprecationWarning', '-I',
+        out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I',
-               '-m', 'pip', '--version']
+               '-m', 'pip', '--version'])
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                  stderr=subprocess.PIPE)
-        out, err = p.communicate()
         # We force everything to text, so unittest gives the detailed diff
         # if we get unexpected results
         err = err.decode("latin-1") # Force to text, prevent decoding errors
@@ -406,12 +405,10 @@ class EnsurePipTest(BaseTest):
         # http://bugs.python.org/issue19728
         # Check the private uninstall command provided for the Windows
         # installers works (at least in a virtual environment)
-        cmd = [envpy, '-W', 'ignore::DeprecationWarning', '-I',
-               '-m', 'ensurepip._uninstall']
         with EnvironmentVarGuard() as envvars:
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+            out, err = check_output([envpy,
-                                      stderr=subprocess.PIPE)
+                '-W', 'ignore::DeprecationWarning', '-I',
-            out, err = p.communicate()
+                '-m', 'ensurepip._uninstall'])
         # We force everything to text, so unittest gives the detailed diff
         # if we get unexpected results
         err = err.decode("latin-1") # Force to text, prevent decoding errors

Lib/venv/__init__.py

@@ -208,11 +208,9 @@ class EnvBuilder:
                     copier(context.env_exe, path, relative_symlinks_ok=True)
                     if not os.path.islink(path):
                         os.chmod(path, 0o755)
-        elif sysconfig.is_python_build(True):
+        else:
-            # See bpo-34011. This copying code should only be needed when a
+            # See bpo-34011. When using a proper install, we should only need to
-            # venv is created from a source Python build (i.e. not an installed
+            # copy the top-level of DLLs.
-            # Python)
-            subdir = 'DLLs'
             include = self.include_binary
             files = [f for f in os.listdir(dirname) if include(f)]
             for f in files:
@@ -220,24 +218,28 @@ class EnvBuilder:
                 dst = os.path.join(binpath, f)
                 if dst != context.env_exe:  # already done, above
                     copier(src, dst)
-            dirname = os.path.join(dirname, subdir)
+
-            if os.path.isdir(dirname):
+            # When creating from a build directory, we continue to copy all files.
-                files = [f for f in os.listdir(dirname) if include(f)]
+            if sysconfig.is_python_build(True):
-                for f in files:
+                subdir = 'DLLs'
-                    src = os.path.join(dirname, f)
+                dirname = os.path.join(dirname, subdir)
-                    dst = os.path.join(binpath, f)
+                if os.path.isdir(dirname):
-                    copier(src, dst)
+                    files = [f for f in os.listdir(dirname) if include(f)]
-            # copy init.tcl over
+                    for f in files:
-            for root, dirs, files in os.walk(context.python_dir):
+                        src = os.path.join(dirname, f)
-                if 'init.tcl' in files:
+                        dst = os.path.join(binpath, f)
-                    tcldir = os.path.basename(root)
+                        copier(src, dst)
-                    tcldir = os.path.join(context.env_dir, 'Lib', tcldir)
+                # copy init.tcl over
-                    if not os.path.exists(tcldir):
+                for root, dirs, files in os.walk(context.python_dir):
-                        os.makedirs(tcldir)
+                    if 'init.tcl' in files:
-                    src = os.path.join(root, 'init.tcl')
+                        tcldir = os.path.basename(root)
-                    dst = os.path.join(tcldir, 'init.tcl')
+                        tcldir = os.path.join(context.env_dir, 'Lib', tcldir)
-                    shutil.copyfile(src, dst)
+                        if not os.path.exists(tcldir):
-                    break
+                            os.makedirs(tcldir)
+                        src = os.path.join(root, 'init.tcl')
+                        dst = os.path.join(tcldir, 'init.tcl')
+                        shutil.copyfile(src, dst)
+                        break
 
     def _setup_pip(self, context):
         """Installs or upgrades pip in a virtual environment"""

Tools/msi/distutils.command.bdist_wininst.py

@@ -9,6 +9,9 @@ from distutils.errors import DistutilsPlatformError
 class bdist_wininst(Command):
     description = "create an executable installer for MS Windows"
 
+    # Marker for tests that we have the unsupported bdist_wininst
+    _unsupported = True
+
     def initialize_options(self):
         pass