gh-83180: Made launcher treat shebang 'python' tags as low priority so that active virtual environments are preferred (GH-108101)

Doc/using/windows.rst

@@ -867,17 +867,18 @@ For example, if the first line of your script starts with
 
   #! /usr/bin/python
 
-The default Python will be located and used.  As many Python scripts written
+The default Python or an active virtual environment will be located and used.
-to work on Unix will already have this line, you should find these scripts can
+As many Python scripts written to work on Unix will already have this line,
-be used by the launcher without modification.  If you are writing a new script
+you should find these scripts can be used by the launcher without modification.
-on Windows which you hope will be useful on Unix, you should use one of the
+If you are writing a new script on Windows which you hope will be useful on
-shebang lines starting with ``/usr``.
+Unix, you should use one of the shebang lines starting with ``/usr``.
 
 Any of the above virtual commands can be suffixed with an explicit version
 (either just the major version, or the major and minor version).
 Furthermore the 32-bit version can be requested by adding "-32" after the
 minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the
-32-bit python 3.7.
+32-bit Python 3.7. If a virtual environment is active, the version will be
+ignored and the environment will be used.
 
 .. versionadded:: 3.7
 
@@ -891,6 +892,13 @@ minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the
    not provably i386/32-bit". To request a specific environment, use the new
    :samp:`-V:{TAG}` argument with the complete tag.
 
+.. versionchanged:: 3.13
+
+   Virtual commands referencing ``python`` now prefer an active virtual
+   environment rather than searching :envvar:`PATH`. This handles cases where
+   the shebang specifies ``/usr/bin/env python3`` but :file:`python3.exe` is
+   not present in the active environment.
+
 The ``/usr/bin/env`` form of shebang line has one further special property.
 Before looking for installed Python interpreters, this form will search the
 executable :envvar:`PATH` for a Python executable matching the name provided

Lib/test/test_launcher.py

@@ -717,3 +717,25 @@ class TestLauncher(unittest.TestCase, RunPyMixin):
             f"{expect} arg1 {script}",
             data["stdout"].strip(),
         )
+
+    def test_shebang_command_in_venv(self):
+        stem = "python-that-is-not-on-path"
+
+        # First ensure that our test name doesn't exist, and the launcher does
+        # not match any installed env
+        with self.script(f'#! /usr/bin/env {stem} arg1') as script:
+            data = self.run_py([script], expect_returncode=103)
+
+        with self.fake_venv() as (venv_exe, env):
+            # Put a real Python (ourselves) on PATH as a distraction.
+            # The active VIRTUAL_ENV should be preferred when the name isn't an
+            # exact match.
+            env["PATH"] = f"{Path(sys.executable).parent};{os.environ['PATH']}"
+
+            with self.script(f'#! /usr/bin/env {stem} arg1') as script:
+                data = self.run_py([script], env=env)
+            self.assertEqual(data["stdout"].strip(), f"{venv_exe} arg1 {script}")
+
+            with self.script(f'#! /usr/bin/env {Path(sys.executable).stem} arg1') as script:
+                data = self.run_py([script], env=env)
+            self.assertEqual(data["stdout"].strip(), f"{sys.executable} arg1 {script}")

Misc/NEWS.d/next/Windows/2023-08-18-00-01-21.gh-issue-83180.DdLffv.rst

@@ -0,0 +1,3 @@
+Changes the :ref:`launcher` to prefer an active virtual environment when the
+launched script has a shebang line using a Unix-like virtual command, even
+if the command requests a specific version of Python.

PC/launcher2.c

@@ -195,6 +195,13 @@ join(wchar_t *buffer, size_t bufferLength, const wchar_t *fragment)
 }
 
 
+bool
+split_parent(wchar_t *buffer, size_t bufferLength)
+{
+    return SUCCEEDED(PathCchRemoveFileSpec(buffer, bufferLength));
+}
+
+
 int
 _compare(const wchar_t *x, int xLen, const wchar_t *y, int yLen)
 {
@@ -414,8 +421,8 @@ typedef struct {
     // if true, treats 'tag' as a non-PEP 514 filter
     bool oldStyleTag;
     // if true, ignores 'tag' when a high priority environment is found
-    // gh-92817: This is currently set when a tag is read from configuration or
+    // gh-92817: This is currently set when a tag is read from configuration,
-    // the environment, rather than the command line or a shebang line, and the
+    // the environment, or a shebang, rather than the command line, and the
     // only currently possible high priority environment is an active virtual
     // environment
     bool lowPriorityTag;
@@ -794,6 +801,8 @@ searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength)
         }
     }
 
+    debug(L"# Search PATH for %s\n", filename);
+
     wchar_t pathVariable[MAXLEN];
     int n = GetEnvironmentVariableW(L"PATH", pathVariable, MAXLEN);
     if (!n) {
@@ -1031,8 +1040,11 @@ checkShebang(SearchInfo *search)
     debug(L"Shebang: %s\n", shebang);
 
     // Handle shebangs that we should search PATH for
+    int executablePathWasSetByUsrBinEnv = 0;
     exitCode = searchPath(search, shebang, shebangLength);
-    if (exitCode != RC_NO_SHEBANG) {
+    if (exitCode == 0) {
+        executablePathWasSetByUsrBinEnv = 1;
+    } else if (exitCode != RC_NO_SHEBANG) {
         return exitCode;
     }
 
@@ -1067,7 +1079,7 @@ checkShebang(SearchInfo *search)
             search->tagLength = commandLength;
             // If we had 'python3.12.exe' then we want to strip the suffix
             // off of the tag
-            if (search->tagLength > 4) {
+            if (search->tagLength >= 4) {
                 const wchar_t *suffix = &search->tag[search->tagLength - 4];
                 if (0 == _comparePath(suffix, 4, L".exe", -1)) {
                     search->tagLength -= 4;
@@ -1075,13 +1087,14 @@ checkShebang(SearchInfo *search)
             }
             // If we had 'python3_d' then we want to strip the '_d' (any
             // '.exe' is already gone)
-            if (search->tagLength > 2) {
+            if (search->tagLength >= 2) {
                 const wchar_t *suffix = &search->tag[search->tagLength - 2];
                 if (0 == _comparePath(suffix, 2, L"_d", -1)) {
                     search->tagLength -= 2;
                 }
             }
             search->oldStyleTag = true;
+            search->lowPriorityTag = true;
             search->executableArgs = &command[commandLength];
             search->executableArgsLength = shebangLength - commandLength;
             if (search->tag && search->tagLength) {
@@ -1095,6 +1108,11 @@ checkShebang(SearchInfo *search)
         }
     }
 
+    // Didn't match a template, but we found it on PATH
+    if (executablePathWasSetByUsrBinEnv) {
+        return 0;
+    }
+
     // Unrecognised executables are first tried as command aliases
     commandLength = 0;
     while (commandLength < shebangLength && !isspace(shebang[commandLength])) {
@@ -1765,7 +1783,15 @@ virtualenvSearch(const SearchInfo *search, EnvironmentInfo **result)
         return 0;
     }
 
-    if (INVALID_FILE_ATTRIBUTES == GetFileAttributesW(buffer)) {
+    DWORD attr = GetFileAttributesW(buffer);
+    if (INVALID_FILE_ATTRIBUTES == attr && search->lowPriorityTag) {
+        if (!split_parent(buffer, MAXLEN) || !join(buffer, MAXLEN, L"python.exe")) {
+            return 0;
+        }
+        attr = GetFileAttributesW(buffer);
+    }
+
+    if (INVALID_FILE_ATTRIBUTES == attr) {
         debug(L"Python executable %s missing from virtual env\n", buffer);
         return 0;
     }