bpo-33185: Fix regression in pydoc CLI sys.path handling (GH-6419)

The pydoc CLI assumed -m pydoc would add the empty string
to sys.path, and hence got confused when it switched to
adding the full initial working directory instead.

This refactors the pydoc CLI path manipulation to be
more testable, and ensures it won't accidentally
remove the standard library directory containing
pydoc itself from sys.path.

Doc/whatsnew/3.7.rst

@@ -1145,9 +1145,10 @@ Changes in Python behavior
   (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.)
 
 * When using the ``-m`` switch, the starting directory is now added to sys.path,
-  rather than the current working directory. Any programs that are found to be
-  relying on the previous behaviour will need to be updated to manipulate
-  :data:`sys.path` appropriately.
+  rather than the current working directory. Any programs that are checking for
+  the empty string in :data:`sys.path`, or otherwise relying on the previous
+  behaviour, will need to be updated accordingly (e.g. by checking for
+  ``os.getcwd()`` in addition to checking for the empty string).
 
 
 Changes in the Python API

Lib/pydoc.py

@@ -2614,18 +2614,50 @@ def browse(port=0, *, open_browser=True, hostname='localhost'):
 def ispath(x):
     return isinstance(x, str) and x.find(os.sep) >= 0
 
+def _get_revised_path(given_path, argv0):
+    """Ensures current directory is on returned path, and argv0 directory is not
+
+    Exception: argv0 dir is left alone if it's also pydoc's directory.
+
+    Returns a new path entry list, or None if no adjustment is needed.
+    """
+    # Scripts may get the current directory in their path by default if they're
+    # run with the -m switch, or directly from the current directory.
+    # The interactive prompt also allows imports from the current directory.
+
+    # Accordingly, if the current directory is already present, don't make
+    # any changes to the given_path
+    if '' in given_path or os.curdir in given_path or os.getcwd() in given_path:
+        return None
+
+    # Otherwise, add the current directory to the given path, and remove the
+    # script directory (as long as the latter isn't also pydoc's directory.
+    stdlib_dir = os.path.dirname(__file__)
+    script_dir = os.path.dirname(argv0)
+    revised_path = given_path.copy()
+    if script_dir in given_path and not os.path.samefile(script_dir, stdlib_dir):
+        revised_path.remove(script_dir)
+    revised_path.insert(0, os.getcwd())
+    return revised_path
+
+
+# Note: the tests only cover _get_revised_path, not _adjust_cli_path itself
+def _adjust_cli_sys_path():
+    """Ensures current directory is on sys.path, and __main__ directory is not
+
+    Exception: __main__ dir is left alone if it's also pydoc's directory.
+    """
+    revised_path = _get_revised_path(sys.path, sys.argv[0])
+    if revised_path is not None:
+        sys.path[:] = revised_path
+
+
 def cli():
     """Command-line interface (looks at sys.argv to decide what to do)."""
     import getopt
     class BadUsage(Exception): pass
 
-    # Scripts don't get the current directory in their path by default
-    # unless they are run with the '-m' switch
-    if '' not in sys.path:
-        scriptdir = os.path.dirname(sys.argv[0])
-        if scriptdir in sys.path:
-            sys.path.remove(scriptdir)
-        sys.path.insert(0, '.')
+    _adjust_cli_sys_path()
 
     try:
         opts, args = getopt.getopt(sys.argv[1:], 'bk:n:p:w')

Lib/test/test_pydoc.py

@@ -11,6 +11,7 @@ import pkgutil
 import re
 import stat
 import string
+import tempfile
 import test.support
 import time
 import types
@@ -1084,6 +1085,71 @@ class PydocWithMetaClasses(unittest.TestCase):
         self.assertIn('class Enum', helptext)
 
 
+class TestInternalUtilities(unittest.TestCase):
+
+    def setUp(self):
+        tmpdir = tempfile.TemporaryDirectory()
+        self.argv0dir = tmpdir.name
+        self.argv0 = os.path.join(tmpdir.name, "nonexistent")
+        self.addCleanup(tmpdir.cleanup)
+        self.abs_curdir = abs_curdir = os.getcwd()
+        self.curdir_spellings = ["", os.curdir, abs_curdir]
+
+    def _get_revised_path(self, given_path, argv0=None):
+        # Checking that pydoc.cli() actually calls pydoc._get_revised_path()
+        # is handled via code review (at least for now).
+        if argv0 is None:
+            argv0 = self.argv0
+        return pydoc._get_revised_path(given_path, argv0)
+
+    def _get_starting_path(self):
+        # Get a copy of sys.path without the current directory
+        clean_path = sys.path.copy()
+        for spelling in self.curdir_spellings:
+            for __ in range(clean_path.count(spelling)):
+                clean_path.remove(spelling)
+        return clean_path
+
+    def test_sys_path_adjustment_adds_missing_curdir(self):
+        clean_path = self._get_starting_path()
+        expected_path = [self.abs_curdir] + clean_path
+        self.assertEqual(self._get_revised_path(clean_path), expected_path)
+
+    def test_sys_path_adjustment_removes_argv0_dir(self):
+        clean_path = self._get_starting_path()
+        expected_path = [self.abs_curdir] + clean_path
+        leading_argv0dir = [self.argv0dir] + clean_path
+        self.assertEqual(self._get_revised_path(leading_argv0dir), expected_path)
+        trailing_argv0dir = clean_path + [self.argv0dir]
+        self.assertEqual(self._get_revised_path(trailing_argv0dir), expected_path)
+
+
+    def test_sys_path_adjustment_protects_pydoc_dir(self):
+        def _get_revised_path(given_path):
+            return self._get_revised_path(given_path, argv0=pydoc.__file__)
+        clean_path = self._get_starting_path()
+        leading_argv0dir = [self.argv0dir] + clean_path
+        expected_path = [self.abs_curdir] + leading_argv0dir
+        self.assertEqual(_get_revised_path(leading_argv0dir), expected_path)
+        trailing_argv0dir = clean_path + [self.argv0dir]
+        expected_path = [self.abs_curdir] + trailing_argv0dir
+        self.assertEqual(_get_revised_path(trailing_argv0dir), expected_path)
+
+    def test_sys_path_adjustment_when_curdir_already_included(self):
+        clean_path = self._get_starting_path()
+        for spelling in self.curdir_spellings:
+            with self.subTest(curdir_spelling=spelling):
+                # If curdir is already present, no alterations are made at all
+                leading_curdir = [spelling] + clean_path
+                self.assertIsNone(self._get_revised_path(leading_curdir))
+                trailing_curdir = clean_path + [spelling]
+                self.assertIsNone(self._get_revised_path(trailing_curdir))
+                leading_argv0dir = [self.argv0dir] + leading_curdir
+                self.assertIsNone(self._get_revised_path(leading_argv0dir))
+                trailing_argv0dir = trailing_curdir + [self.argv0dir]
+                self.assertIsNone(self._get_revised_path(trailing_argv0dir))
+
+
 @reap_threads
 def test_main():
     try:
@@ -1094,6 +1160,7 @@ def test_main():
                                   PydocUrlHandlerTest,
                                   TestHelper,
                                   PydocWithMetaClasses,
+                                  TestInternalUtilities,
                                   )
     finally:
         reap_children()

Misc/NEWS.d/next/Library/2018-04-08-22-54-07.bpo-33185.Id-Ba9.rst

@@ -0,0 +1,5 @@
+Fixed regression when running pydoc with the ``-m`` switch. (The regression
+was introduced in 3.7.0b3 by the resolution of bpo-33053)
+
+This fix also changed pydoc to add ``os.getcwd()`` to ``sys.path`` when
+necessary, rather than adding ``"."``.