From e3d4fed07429670af631e5662086b76c1ec098c4 Mon Sep 17 00:00:00 2001 From: Kai Zhang Date: Mon, 14 Nov 2022 23:05:14 +0800 Subject: [PATCH] gh-99370: Calculate zip path from prefix when in a venv (GH-99371) Before python3.11, when in a venv the zip path is calculated from prefix on POSIX platforms. In python3.11 the behavior is accidentally changed to calculating from default prefix. This change will break venv created from a non-installed python with a stdlib zip file. This commit restores the behavior back to before python3.11. --- Lib/test/test_getpath.py | 33 ++++++++++ Lib/test/test_venv.py | 63 +++++++++++++++++++ ...2-11-12-01-39-57.gh-issue-99370._cu32j.rst | 2 + Modules/getpath.py | 3 +- 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 7a22a9a8fb6..bdcf4a37191 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -382,6 +382,39 @@ class MockGetPathTests(unittest.TestCase): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_non_installed_zip_path_posix(self): + "Test a venv created from non-installed python has correct zip path.""" + ns = MockPosixNamespace( + argv0="/venv/bin/python", + PREFIX="/usr", + ENV_PATH="/venv/bin:/usr/bin", + ) + ns.add_known_xfile("/path/to/non-installed/bin/python") + ns.add_known_xfile("/venv/bin/python") + ns.add_known_link("/venv/bin/python", + "/path/to/non-installed/bin/python") + ns.add_known_file("/path/to/non-installed/lib/python9.8/os.py") + ns.add_known_dir("/path/to/non-installed/lib/python9.8/lib-dynload") + ns.add_known_file("/venv/pyvenv.cfg", [ + r"home = /path/to/non-installed" + ]) + expected = dict( + executable="/venv/bin/python", + prefix="/path/to/non-installed", + exec_prefix="/path/to/non-installed", + base_executable="/path/to/non-installed/bin/python", + base_prefix="/path/to/non-installed", + base_exec_prefix="/path/to/non-installed", + module_search_paths_set=1, + module_search_paths=[ + "/path/to/non-installed/lib/python98.zip", + "/path/to/non-installed/lib/python9.8", + "/path/to/non-installed/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_changed_name_copy_posix(self): "Test a venv --copies layout on *nix that lacks a distributed 'python'" ns = MockPosixNamespace( diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 5ce86cffd24..3e306d39feb 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -537,6 +537,69 @@ class BasicTest(BaseTest): self.assertRaises(ValueError, venv.create, bad_itempath) self.assertRaises(ValueError, venv.create, pathlib.Path(bad_itempath)) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + @requireVenvCreate + def test_zippath_from_non_installed_posix(self): + """ + Test that when create venv from non-installed python, the zip path + value is as expected. + """ + rmtree(self.env_dir) + # First try to create a non-installed python. It's not a real full + # functional non-installed python, but enough for this test. + non_installed_dir = os.path.realpath(tempfile.mkdtemp()) + try: + bindir = os.path.join(non_installed_dir, self.bindir) + os.mkdir(bindir) + shutil.copy2(sys.executable, bindir) + libdir = os.path.join(non_installed_dir, *self.lib) + os.makedirs(libdir) + landmark = os.path.join(libdir, "os.py") + stdlib_zip = "python%d%d.zip" % sys.version_info[:2] + zip_landmark = os.path.join(non_installed_dir, + self.lib[0], + stdlib_zip) + additional_pythonpath_for_non_installed = [] + # Copy stdlib files to the non-installed python so venv can + # correctly calculate the prefix. + for eachpath in sys.path: + if eachpath.endswith(".zip"): + if os.path.isfile(eachpath): + shutil.copyfile( + eachpath, + os.path.join(non_installed_dir, self.lib[0])) + elif os.path.isfile(os.path.join(eachpath, "os.py")): + for name in os.listdir(eachpath): + if name == "site-packages": + continue + fn = os.path.join(eachpath, name) + if os.path.isfile(fn): + shutil.copy(fn, libdir) + elif os.path.isdir(fn): + shutil.copytree(fn, os.path.join(libdir, name)) + else: + additional_pythonpath_for_non_installed.append( + eachpath) + cmd = [os.path.join(non_installed_dir, self.bindir, self.exe), + "-m", + "venv", + "--without-pip", + self.env_dir] + # Our fake non-installed python is not fully functional because + # it cannot find the extensions. Set PYTHONPATH so it can run the + # venv module correctly. + pythonpath = os.pathsep.join( + additional_pythonpath_for_non_installed) + subprocess.check_call(cmd, env={"PYTHONPATH": pythonpath}) + envpy = os.path.join(self.env_dir, self.bindir, self.exe) + # Now check the venv created from the non-installed python has + # correct zip path in pythonpath. + cmd = [envpy, '-S', '-c', 'import sys; print(sys.path)'] + out, err = check_output(cmd) + self.assertTrue(zip_landmark.encode() in out) + finally: + rmtree(non_installed_dir) + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst new file mode 100644 index 00000000000..142f91ccd92 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-12-01-39-57.gh-issue-99370._cu32j.rst @@ -0,0 +1,2 @@ +Fix zip path for venv created from a non-installed python on POSIX +platforms. diff --git a/Modules/getpath.py b/Modules/getpath.py index d24b15259a9..ab0d2dc0636 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -679,9 +679,8 @@ elif not pythonpath_was_set: else: library_dir = executable_dir pythonpath.append(joinpath(library_dir, ZIP_LANDMARK)) - elif build_prefix or venv_prefix: + elif build_prefix: # QUIRK: POSIX uses the default prefix when in the build directory - # or a venv pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) else: pythonpath.append(joinpath(prefix, ZIP_LANDMARK))