#7712: add a temp_cwd context manager to test_support and use it in regrtest to run all the tests in a temporary directory, saving the original CWD in test_support.SAVEDCWD. Thanks to Florent Xicluna who helped with the patch.

Lib/test/regrtest.py

@@ -158,9 +158,27 @@ import time
 import traceback
 import warnings
 import unittest
+import tempfile
 import imp
 
 
+# Some times __path__ and __file__ are not absolute (e.g. while running from
+# Lib/) and, if we change the CWD to run the tests in a temporary dir, some
+# imports might fail.  This affects only the modules imported before os.chdir().
+# These modules are searched first in sys.path[0] (so '' -- the CWD) and if
+# they are found in the CWD their __file__ and __path__ will be relative (this
+# happens before the chdir).  All the modules imported after the chdir, are
+# not found in the CWD, and since the other paths in sys.path[1:] are absolute
+# (site.py absolutize them), the __file__ and __path__ will be absolute too.
+# Therefore it is necessary to absolutize manually the __file__ and __path__ of
+# the packages to prevent later imports to fail when the CWD is different.
+for module in sys.modules.itervalues():
+    if hasattr(module, '__path__'):
+        module.__path__ = [os.path.abspath(path) for path in module.__path__]
+    if hasattr(module, '__file__'):
+        module.__file__ = os.path.abspath(module.__file__)
+
+
 # Ignore ImportWarnings that only occur in the source tree,
 # (because of modules with the same name as source-directories in Modules/)
 for mod in ("ctypes", "gzip", "zipfile", "tarfile", "encodings.zlib_codec",
@@ -350,6 +368,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
     resource_denieds = []
     environment_changed = []
 
+    if verbose:
+        print 'The CWD is now', os.getcwd()
+
     if findleaks:
         try:
             import gc
@@ -364,8 +385,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
             found_garbage = []
 
     if single:
-        from tempfile import gettempdir
+        filename = 'pynexttest'
-        filename = os.path.join(gettempdir(), 'pynexttest')
         try:
             fp = open(filename, 'r')
             next_test = fp.read().strip()
@@ -376,7 +396,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
 
     if fromfile:
         tests = []
-        fp = open(fromfile)
+        fp = open(os.path.join(test_support.SAVEDCWD, fromfile))
         for line in fp:
             guts = line.split() # assuming no test has whitespace in its name
             if guts and not guts[0].startswith('#'):
@@ -959,6 +979,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
 
     deltas = []
     nwarmup, ntracked, fname = huntrleaks
+    fname = os.path.join(test_support.SAVEDCWD, fname)
     repcount = nwarmup + ntracked
     print >> sys.stderr, "beginning", repcount, "repetitions"
     print >> sys.stderr, ("1234567890"*(repcount//10 + 1))[:repcount]
@@ -1501,4 +1522,23 @@ if __name__ == '__main__':
         i -= 1
         if os.path.abspath(os.path.normpath(sys.path[i])) == mydir:
             del sys.path[i]
-    main()
+
+    # findtestdir() gets the dirname out of sys.argv[0], so we have to make it
+    # absolute before changing the CWD.
+    if sys.argv[0]:
+        sys.argv[0] = os.path.abspath(sys.argv[0])
+
+
+    # Define a writable temp dir that will be used as cwd while running
+    # the tests. The name of the dir includes the pid to allow parallel
+    # testing (see the -j option).
+    TESTCWD = 'test_python_{}'.format(os.getpid())
+
+    TESTCWD = os.path.abspath(os.path.join(tempfile.gettempdir(), TESTCWD))
+
+    # Run the tests in a context manager that temporary changes the CWD to a
+    # temporary and writable directory. If it's not possible to create or
+    # change the CWD, the original CWD will be used. The original CWD is
+    # available from test_support.SAVEDCWD.
+    with test_support.temp_cwd(TESTCWD, quiet=True):
+        main()

Lib/test/test_subprocess.py

@@ -7,6 +7,7 @@ import os
 import tempfile
 import time
 import re
+import sysconfig
 
 mswindows = (sys.platform == "win32")
 
@@ -140,9 +141,21 @@ class ProcessTestCase(unittest.TestCase):
         p.wait()
         self.assertEqual(p.stderr, None)
 
-    def test_executable(self):
+    def test_executable_with_cwd(self):
-        p = subprocess.Popen(["somethingyoudonthave",
+        python_dir = os.path.dirname(os.path.realpath(sys.executable))
-                              "-c", "import sys; sys.exit(47)"],
+        p = subprocess.Popen(["somethingyoudonthave", "-c",
+                              "import sys; sys.exit(47)"],
+                             executable=sys.executable, cwd=python_dir)
+        p.wait()
+        self.assertEqual(p.returncode, 47)
+
+    @unittest.skipIf(sysconfig.is_python_build(),
+                     "need an installed Python. See #7774")
+    def test_executable_without_cwd(self):
+        # For a normal installation, it should work without 'cwd'
+        # argument.  For test runs in the build directory, see #7774.
+        p = subprocess.Popen(["somethingyoudonthave", "-c",
+                              "import sys; sys.exit(47)"],
                              executable=sys.executable)
         p.wait()
         self.assertEqual(p.returncode, 47)

Lib/test/test_support.py

@@ -22,7 +22,7 @@ __all__ = ["Error", "TestFailed", "ResourceDenied", "import_module",
            "get_original_stdout", "unload", "unlink", "rmtree", "forget",
            "is_resource_enabled", "requires", "find_unused_port", "bind_port",
            "fcmp", "have_unicode", "is_jython", "TESTFN", "HOST", "FUZZ",
-           "findfile", "sortdict", "check_syntax_error",
+           "SAVEDCWD", "temp_cwd", "findfile", "sortdict", "check_syntax_error",
            "open_urlresource", "check_warnings", "CleanImport",
            "EnvironmentVarGuard", "captured_output",
            "captured_stdout", "TransientResource", "transient_internet",
@@ -379,27 +379,42 @@ else:
                 'Unicode filename tests may not be effective' \
                 % TESTFN_UNICODE_UNENCODEABLE
 
+
 # Disambiguate TESTFN for parallel testing, while letting it remain a valid
 # module name.
-TESTFN = "{0}_{1}_tmp".format(TESTFN, os.getpid())
+TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
 
-# Make sure we can write to TESTFN, try in /tmp if we can't
+# Save the initial cwd
-fp = None
+SAVEDCWD = os.getcwd()
-try:
+
-    fp = open(TESTFN, 'w+')
+@contextlib.contextmanager
-except IOError:
+def temp_cwd(name='tempcwd', quiet=False):
-    TMP_TESTFN = os.path.join('/tmp', TESTFN)
+    """
+    Context manager that creates a temporary directory and set it as CWD.
+
+    The new CWD is created in the current directory and it's named *name*.
+    If *quiet* is False (default) and it's not possible to create or change
+    the CWD, an error is raised.  If it's True, only a warning is raised
+    and the original CWD is used.
+    """
+    saved_dir = os.getcwd()
+    is_temporary = False
     try:
-        fp = open(TMP_TESTFN, 'w+')
+        os.mkdir(name)
-        TESTFN = TMP_TESTFN
+        os.chdir(name)
-        del TMP_TESTFN
+        is_temporary = True
-    except IOError:
+    except OSError:
-        print ('WARNING: tests will fail, unable to write to: %s or %s' %
+        if not quiet:
-                (TESTFN, TMP_TESTFN))
+            raise
-if fp is not None:
+        warnings.warn('tests may fail, unable to change the CWD to ' + name,
-    fp.close()
+                      RuntimeWarning, stacklevel=3)
-    unlink(TESTFN)
+    try:
-del fp
+        yield os.getcwd()
+    finally:
+        os.chdir(saved_dir)
+        if is_temporary:
+            rmtree(name)
+
 
 def findfile(file, here=__file__):
     """Try to find a file on sys.path and the working directory.  If it is not

Misc/NEWS

@@ -17,13 +17,24 @@ Library
 
 - Issue #6003: add an argument to ``zipfile.Zipfile.writestr`` to
   specify the compression type.
+
 - Issue #7893: ``unittest.TextTestResult`` is made public and a ``resultclass``
   argument added to the TextTestRunner constructor allowing a different result
   class to be used without having to subclass.
+
 - Issue 7588: ``unittest.TextTestResult.getDescription`` now includes the test
   name in failure reports even if the test has a docstring.
 
 
+Tests
+-----
+
+- Issue #7712: test_support gained a new `temp_cwd` context manager which is
+  now also used by regrtest to run all the tests in a temporary directory.
+  The original CWD is saved in `test_support.SAVEDCWD`.
+  Thanks to Florent Xicluna who helped with the patch.
+
+
 What's New in Python 2.7 alpha 3?
 =================================