From 700cfa8c90a90016638bac13c4efd03786b2b2a0 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka <storchaka@gmail.com>
Date: Thu, 25 Jun 2020 17:56:31 +0300
Subject: [PATCH] bpo-41069: Make TESTFN and the CWD for tests containing
 non-ascii characters. (GH-21035)

---
 Lib/test/libregrtest/main.py                  |  1 +
 Lib/test/support/__init__.py                  |  2 +-
 Lib/test/support/os_helper.py                 | 21 +++++------
 Lib/test/test_binhex.py                       |  7 ++--
 Lib/test/test_cgitb.py                        | 10 +++---
 Lib/test/test_compileall.py                   |  6 ++--
 Lib/test/test_embed.py                        |  4 +--
 Lib/test/test_fstring.py                      |  5 +--
 Lib/test/test_genericpath.py                  |  2 +-
 Lib/test/test_gzip.py                         | 18 +++++++---
 Lib/test/test_msilib.py                       |  4 +--
 Lib/test/test_ntpath.py                       |  2 +-
 Lib/test/test_os.py                           |  6 ++--
 Lib/test/test_pdb.py                          | 11 +++---
 Lib/test/test_posixpath.py                    |  2 +-
 Lib/test/test_tarfile.py                      |  9 +++--
 Lib/test/test_tools/test_pathfix.py           |  8 +++--
 Lib/test/test_trace.py                        |  9 +++--
 Lib/test/test_urllib.py                       |  7 ++--
 Lib/test/test_uu.py                           |  5 +--
 Lib/test/test_venv.py                         |  4 +--
 Lib/test/test_warnings/__init__.py            |  6 ++--
 .../2020-06-22-00-21-12.bpo-41069.bLZkX-.rst  |  2 ++
 Modules/_testcapimodule.c                     | 36 +++++++++----------
 24 files changed, 110 insertions(+), 77 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Tests/2020-06-22-00-21-12.bpo-41069.bLZkX-.rst

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 3f9771b9308..7675a97b5b4 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -597,6 +597,7 @@ class Regrtest:
             test_cwd = 'test_python_worker_{}'.format(pid)
         else:
             test_cwd = 'test_python_{}'.format(pid)
+        test_cwd += support.FS_NONASCII
         test_cwd = os.path.join(self.tmp_dir, test_cwd)
         return test_cwd
 
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 5707d8eeaa2..f8f60fb6c27 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -20,7 +20,7 @@ from .import_helper import (
     forget, import_fresh_module, import_module, make_legacy_pyc,
     modules_cleanup, modules_setup, unload)
 from .os_helper import (
-    FS_NONASCII, SAVEDCWD, TESTFN, TESTFN_NONASCII,
+    FS_NONASCII, SAVEDCWD, TESTFN, TESTFN_ASCII, TESTFN_NONASCII,
     TESTFN_UNENCODABLE, TESTFN_UNDECODABLE,
     TESTFN_UNICODE, can_symlink, can_xattr,
     change_cwd, create_empty_file, fd_count,
diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py
index d3347027cf2..d9807a1e114 100644
--- a/Lib/test/support/os_helper.py
+++ b/Lib/test/support/os_helper.py
@@ -13,16 +13,16 @@ import warnings
 # Filename used for testing
 if os.name == 'java':
     # Jython disallows @ in module names
-    TESTFN = '$test'
+    TESTFN_ASCII = '$test'
 else:
-    TESTFN = '@test'
+    TESTFN_ASCII = '@test'
 
 # Disambiguate TESTFN for parallel testing, while letting it remain a valid
 # module name.
-TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
+TESTFN_ASCII = "{}_{}_tmp".format(TESTFN_ASCII, os.getpid())
 
 # TESTFN_UNICODE is a non-ascii filename
-TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
+TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f"
 if sys.platform == 'darwin':
     # In Mac OS X's VFS API file names are, by definition, canonically
     # decomposed Unicode, encoded using UTF-8. See QA1173:
@@ -39,7 +39,7 @@ if os.name == 'nt':
     if sys.getwindowsversion().platform >= 2:
         # Different kinds of characters from various languages to minimize the
         # probability that the whole name is encodable to MBCS (issue #9819)
-        TESTFN_UNENCODABLE = TESTFN + "-\u5171\u0141\u2661\u0363\uDC80"
+        TESTFN_UNENCODABLE = TESTFN_ASCII + "-\u5171\u0141\u2661\u0363\uDC80"
         try:
             TESTFN_UNENCODABLE.encode(sys.getfilesystemencoding())
         except UnicodeEncodeError:
@@ -56,7 +56,7 @@ elif sys.platform != 'darwin':
         b'\xff'.decode(sys.getfilesystemencoding())
     except UnicodeDecodeError:
         # 0xff will be encoded using the surrogate character u+DCFF
-        TESTFN_UNENCODABLE = TESTFN \
+        TESTFN_UNENCODABLE = TESTFN_ASCII \
             + b'-\xff'.decode(sys.getfilesystemencoding(), 'surrogateescape')
     else:
         # File system encoding (eg. ISO-8859-* encodings) can encode
@@ -64,8 +64,8 @@ elif sys.platform != 'darwin':
         pass
 
 # FS_NONASCII: non-ASCII character encodable by os.fsencode(),
-# or None if there is no such character.
-FS_NONASCII = None
+# or an empty string if there is no such character.
+FS_NONASCII = ''
 for character in (
     # First try printable and common characters to have a readable filename.
     # For each character, the encoding list are just example of encodings able
@@ -141,13 +141,14 @@ for name in (
     try:
         name.decode(sys.getfilesystemencoding())
     except UnicodeDecodeError:
-        TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
+        TESTFN_UNDECODABLE = os.fsencode(TESTFN_ASCII) + name
         break
 
 if FS_NONASCII:
-    TESTFN_NONASCII = TESTFN + '-' + FS_NONASCII
+    TESTFN_NONASCII = TESTFN_ASCII + FS_NONASCII
 else:
     TESTFN_NONASCII = None
+TESTFN = TESTFN_NONASCII or TESTFN_ASCII
 
 
 def make_bad_fd():
diff --git a/Lib/test/test_binhex.py b/Lib/test/test_binhex.py
index 859553222a3..591f32a4f0f 100644
--- a/Lib/test/test_binhex.py
+++ b/Lib/test/test_binhex.py
@@ -13,9 +13,10 @@ with support.check_warnings(('', DeprecationWarning)):
 class BinHexTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.fname1 = support.TESTFN + "1"
-        self.fname2 = support.TESTFN + "2"
-        self.fname3 = support.TESTFN + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
+        # binhex supports only file names encodable to Latin1
+        self.fname1 = support.TESTFN_ASCII + "1"
+        self.fname2 = support.TESTFN_ASCII + "2"
+        self.fname3 = support.TESTFN_ASCII + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
 
     def tearDown(self):
         support.unlink(self.fname1)
diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py
index 8991bc1ff34..bab152d8554 100644
--- a/Lib/test/test_cgitb.py
+++ b/Lib/test/test_cgitb.py
@@ -41,8 +41,9 @@ class TestCgitb(unittest.TestCase):
             rc, out, err = assert_python_failure(
                   '-c',
                   ('import cgitb; cgitb.enable(logdir=%s); '
-                   'raise ValueError("Hello World")') % repr(tracedir))
-        out = out.decode(sys.getfilesystemencoding())
+                   'raise ValueError("Hello World")') % repr(tracedir),
+                  PYTHONIOENCODING='utf-8')
+        out = out.decode()
         self.assertIn("ValueError", out)
         self.assertIn("Hello World", out)
         self.assertIn("<strong>&lt;module&gt;</strong>", out)
@@ -56,8 +57,9 @@ class TestCgitb(unittest.TestCase):
             rc, out, err = assert_python_failure(
                   '-c',
                   ('import cgitb; cgitb.enable(format="text", logdir=%s); '
-                   'raise ValueError("Hello World")') % repr(tracedir))
-        out = out.decode(sys.getfilesystemencoding())
+                   'raise ValueError("Hello World")') % repr(tracedir),
+                  PYTHONIOENCODING='utf-8')
+        out = out.decode()
         self.assertIn("ValueError", out)
         self.assertIn("Hello World", out)
         self.assertNotIn('<p>', out)
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index b4061b79357..3bbc6817f8d 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -456,13 +456,15 @@ class CommandLineTestsBase:
 
     def assertRunOK(self, *args, **env_vars):
         rc, out, err = script_helper.assert_python_ok(
-                         *self._get_run_args(args), **env_vars)
+                         *self._get_run_args(args), **env_vars,
+                         PYTHONIOENCODING='utf-8')
         self.assertEqual(b'', err)
         return out
 
     def assertRunNotOK(self, *args, **env_vars):
         rc, out, err = script_helper.assert_python_failure(
-                        *self._get_run_args(args), **env_vars)
+                        *self._get_run_args(args), **env_vars,
+                        PYTHONIOENCODING='utf-8')
         return rc, out, err
 
     def assertCompiled(self, fn):
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index e740fe89529..fe47289777a 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -1349,7 +1349,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
                                       returncode=1)
 
     def test_audit_run_interactivehook(self):
-        startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
+        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
         with open(startup, "w", encoding="utf-8") as f:
             print("import sys", file=f)
             print("sys.__interactivehook__ = lambda: None", file=f)
@@ -1362,7 +1362,7 @@ class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
             os.unlink(startup)
 
     def test_audit_run_startup(self):
-        startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
+        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
         with open(startup, "w", encoding="utf-8") as f:
             print("pass", file=f)
         try:
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 9eb7ebe1055..7ffe01d2d8c 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -1055,8 +1055,9 @@ non-important content
             file_path = os.path.join(cwd, 't.py')
             with open(file_path, 'w') as f:
                 f.write('f"{a b}"') # This generates a SyntaxError
-            _, _, stderr = assert_python_failure(file_path)
-        self.assertIn(file_path, stderr.decode('utf-8'))
+            _, _, stderr = assert_python_failure(file_path,
+                                                 PYTHONIOENCODING='ascii')
+        self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr)
 
     def test_loop(self):
         for i in range(1000):
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
index 9d5ac44b6d0..e7acbcd2908 100644
--- a/Lib/test/test_genericpath.py
+++ b/Lib/test/test_genericpath.py
@@ -534,7 +534,7 @@ class CommonTest(GenericTest):
 class PathLikeTests(unittest.TestCase):
 
     def setUp(self):
-        self.file_name = support.TESTFN.lower()
+        self.file_name = support.TESTFN
         self.file_path = FakePath(support.TESTFN)
         self.addCleanup(support.unlink, self.file_name)
         create_file(self.file_name, b"test_genericpath.PathLikeTests")
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
index 78334213f24..0f235d1805e 100644
--- a/Lib/test/test_gzip.py
+++ b/Lib/test/test_gzip.py
@@ -328,8 +328,15 @@ class TestGzip(BaseTest):
             cmByte = fRead.read(1)
             self.assertEqual(cmByte, b'\x08') # deflate
 
+            try:
+                expectedname = self.filename.encode('Latin-1') + b'\x00'
+                expectedflags = b'\x08' # only the FNAME flag is set
+            except UnicodeEncodeError:
+                expectedname = b''
+                expectedflags = b'\x00'
+
             flagsByte = fRead.read(1)
-            self.assertEqual(flagsByte, b'\x08') # only the FNAME flag is set
+            self.assertEqual(flagsByte, expectedflags)
 
             mtimeBytes = fRead.read(4)
             self.assertEqual(mtimeBytes, struct.pack('<i', mtime)) # little-endian
@@ -344,9 +351,8 @@ class TestGzip(BaseTest):
             # RFC 1952 specifies that this is the name of the input file, if any.
             # However, the gzip module defaults to storing the name of the output
             # file in this field.
-            expected = self.filename.encode('Latin-1') + b'\x00'
-            nameBytes = fRead.read(len(expected))
-            self.assertEqual(nameBytes, expected)
+            nameBytes = fRead.read(len(expectedname))
+            self.assertEqual(nameBytes, expectedname)
 
             # Since no other flags were set, the header ends here.
             # Rather than process the compressed data, let's seek to the trailer.
@@ -358,6 +364,10 @@ class TestGzip(BaseTest):
             isizeBytes = fRead.read(4)
             self.assertEqual(isizeBytes, struct.pack('<i', len(data1)))
 
+    def test_metadata_ascii_name(self):
+        self.filename = support.TESTFN_ASCII
+        self.test_metadata()
+
     def test_compresslevel_metadata(self):
         # see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
         # specifically, discussion of XFL in section 2.3.1
diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py
index 153a8ac05e5..4a233c3784e 100644
--- a/Lib/test/test_msilib.py
+++ b/Lib/test/test_msilib.py
@@ -1,13 +1,13 @@
 """ Test suite for the code in msilib """
 import os
 import unittest
-from test.support import TESTFN, FS_NONASCII, import_module, unlink
+from test.support import TESTFN, import_module, unlink
 msilib = import_module('msilib')
 import msilib.schema
 
 
 def init_database():
-    path = TESTFN + (FS_NONASCII or '') + '.msi'
+    path = TESTFN + '.msi'
     db = msilib.init_database(
         path,
         msilib.schema,
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 4a02db28172..6f881f197c4 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -725,7 +725,7 @@ class PathLikeTests(NtpathTestCase):
     path = ntpath
 
     def setUp(self):
-        self.file_name = support.TESTFN.lower()
+        self.file_name = support.TESTFN
         self.file_path = FakePath(support.TESTFN)
         self.addCleanup(support.unlink, self.file_name)
         with open(self.file_name, 'xb', 0) as file:
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 7d4376aed89..ef2395d87a6 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1174,7 +1174,7 @@ class WalkTests(unittest.TestCase):
         os.makedirs(t2_path)
 
         for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
-            with open(path, "x") as f:
+            with open(path, "x", encoding='utf-8') as f:
                 f.write("I'm " + path + " and proud of it.  Blame test_os.\n")
 
         if support.can_symlink():
@@ -2360,7 +2360,7 @@ class Win32ListdirTests(unittest.TestCase):
             file_name = 'FILE%d' % i
             file_path = os.path.join(support.TESTFN, file_name)
             os.makedirs(dir_path)
-            with open(file_path, 'w') as f:
+            with open(file_path, 'w', encoding='utf-8') as f:
                 f.write("I'm %s and proud of it. Blame test_os.\n" % file_path)
             self.created_paths.extend([dir_name, file_name])
         self.created_paths.sort()
@@ -3738,7 +3738,7 @@ class PathTConverterTests(unittest.TestCase):
         if os.name == 'nt':
             bytes_fspath = bytes_filename = None
         else:
-            bytes_filename = support.TESTFN.encode('ascii')
+            bytes_filename = os.fsencode(support.TESTFN)
             bytes_fspath = FakePath(bytes_filename)
         fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT)
         self.addCleanup(support.unlink, support.TESTFN)
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 1e8b12a9af0..65bca291d96 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -1198,6 +1198,7 @@ class PdbTestCase(unittest.TestCase):
                 stdout=subprocess.PIPE,
                 stdin=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
+                env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
         ) as proc:
             stdout, stderr = proc.communicate(str.encode(commands))
         stdout = stdout and bytes.decode(stdout)
@@ -1353,10 +1354,11 @@ def bœr():
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE,
             stderr=subprocess.STDOUT,
+            env={**os.environ, 'PYTHONIOENCODING': 'utf-8'}
             )
         self.addCleanup(proc.stdout.close)
         stdout, stderr = proc.communicate(b'cont\n')
-        self.assertNotIn('Error', stdout.decode(),
+        self.assertNotIn(b'Error', stdout,
                          "Got an error running test script under PDB")
 
     def test_issue36250(self):
@@ -1382,10 +1384,11 @@ def bœr():
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE,
             stderr=subprocess.STDOUT,
+            env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
             )
         self.addCleanup(proc.stdout.close)
         stdout, stderr = proc.communicate(b'cont\ncont\n')
-        self.assertNotIn('Error', stdout.decode(),
+        self.assertNotIn(b'Error', stdout,
                          "Got an error running test script under PDB")
 
     def test_issue16180(self):
@@ -1425,8 +1428,8 @@ def bœr():
                 )
                 with proc:
                     stdout, stderr = proc.communicate(b'q\n')
-                    self.assertNotIn("NameError: name 'invalid' is not defined",
-                                  stdout.decode())
+                    self.assertNotIn(b"NameError: name 'invalid' is not defined",
+                                  stdout)
 
         finally:
             if save_home is not None:
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
index 4d3d8976d60..18819a5dc1c 100644
--- a/Lib/test/test_posixpath.py
+++ b/Lib/test/test_posixpath.py
@@ -627,7 +627,7 @@ class PathLikeTests(unittest.TestCase):
     path = posixpath
 
     def setUp(self):
-        self.file_name = support.TESTFN.lower()
+        self.file_name = support.TESTFN
         self.file_path = FakePath(support.TESTFN)
         self.addCleanup(support.unlink, self.file_name)
         with open(self.file_name, 'xb', 0) as file:
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index be66f1f89e6..d60d35b5be0 100644
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -2305,7 +2305,8 @@ class CommandLineTest(unittest.TestCase):
     def test_test_command_verbose(self):
         for tar_name in testtarnames:
             for opt in '-v', '--verbose':
-                out = self.tarfilecmd(opt, '-t', tar_name)
+                out = self.tarfilecmd(opt, '-t', tar_name,
+                                      PYTHONIOENCODING='utf-8')
                 self.assertIn(b'is a tar archive.\n', out)
 
     def test_test_command_invalid_file(self):
@@ -2376,7 +2377,8 @@ class CommandLineTest(unittest.TestCase):
                                   'and-utf8-bom-sig-only.txt')]
         for opt in '-v', '--verbose':
             try:
-                out = self.tarfilecmd(opt, '-c', tmpname, *files)
+                out = self.tarfilecmd(opt, '-c', tmpname, *files,
+                                      PYTHONIOENCODING='utf-8')
                 self.assertIn(b' file created.', out)
                 with tarfile.open(tmpname) as tar:
                     tar.getmembers()
@@ -2434,7 +2436,8 @@ class CommandLineTest(unittest.TestCase):
         for opt in '-v', '--verbose':
             try:
                 with support.temp_cwd(tarextdir):
-                    out = self.tarfilecmd(opt, '-e', tmpname)
+                    out = self.tarfilecmd(opt, '-e', tmpname,
+                                          PYTHONIOENCODING='utf-8')
                 self.assertIn(b' file is extracted.', out)
             finally:
                 support.rmtree(tarextdir)
diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
index 8b419307340..03ed29d3f97 100644
--- a/Lib/test/test_tools/test_pathfix.py
+++ b/Lib/test/test_tools/test_pathfix.py
@@ -30,16 +30,18 @@ class TestPathfixFunctional(unittest.TestCase):
         with open(filename, 'w', encoding='utf8') as f:
             f.write(f'{shebang}\n' + 'print("Hello world")\n')
 
+        encoding = sys.getfilesystemencoding()
         proc = subprocess.run(
             [sys.executable, self.script,
              *pathfix_flags, '-n', pathfix_arg],
-            capture_output=True, text=1)
+            env={**os.environ, 'PYTHONIOENCODING': encoding},
+            capture_output=True)
 
         if stdout == '' and proc.returncode == 0:
             stdout = f'{filename}: updating\n'
         self.assertEqual(proc.returncode, exitcode, proc)
-        self.assertEqual(proc.stdout, stdout, proc)
-        self.assertEqual(proc.stderr, stderr, proc)
+        self.assertEqual(proc.stdout.decode(encoding), stdout.replace('\n', os.linesep), proc)
+        self.assertEqual(proc.stderr.decode(encoding), stderr.replace('\n', os.linesep), proc)
 
         with open(filename, 'r', encoding='utf8') as f:
             output = f.read()
diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py
index 7cda546b8b9..89d46376bff 100644
--- a/Lib/test/test_trace.py
+++ b/Lib/test/test_trace.py
@@ -488,7 +488,8 @@ class TestCommandLine(unittest.TestCase):
         with open(TESTFN, 'w') as fd:
             self.addCleanup(unlink, TESTFN)
             fd.write("a = 1\n")
-            status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN)
+            status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN,
+                                                      PYTHONIOENCODING='utf-8')
             self.assertIn(b'functions called:', stdout)
 
     def test_sys_argv_list(self):
@@ -498,7 +499,8 @@ class TestCommandLine(unittest.TestCase):
             fd.write("print(type(sys.argv))\n")
 
         status, direct_stdout, stderr = assert_python_ok(TESTFN)
-        status, trace_stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN)
+        status, trace_stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN,
+                                                        PYTHONIOENCODING='utf-8')
         self.assertIn(direct_stdout.strip(), trace_stdout)
 
     def test_count_and_summary(self):
@@ -517,7 +519,8 @@ class TestCommandLine(unittest.TestCase):
                 for i in range(10):
                     f()
             """))
-        status, stdout, _ = assert_python_ok('-m', 'trace', '-cs', filename)
+        status, stdout, _ = assert_python_ok('-m', 'trace', '-cs', filename,
+                                             PYTHONIOENCODING='utf-8')
         stdout = stdout.decode()
         self.assertEqual(status, 0)
         self.assertIn('lines   cov%   module   (path)', stdout)
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
index ebeb9a00145..68bb49efb28 100644
--- a/Lib/test/test_urllib.py
+++ b/Lib/test/test_urllib.py
@@ -151,7 +151,8 @@ class urlopen_FileTests(unittest.TestCase):
         finally:
             f.close()
         self.pathname = support.TESTFN
-        self.returned_obj = urlopen("file:%s" % self.pathname)
+        self.quoted_pathname = urllib.parse.quote(self.pathname)
+        self.returned_obj = urlopen("file:%s" % self.quoted_pathname)
 
     def tearDown(self):
         """Shut down the open object"""
@@ -198,7 +199,7 @@ class urlopen_FileTests(unittest.TestCase):
         self.assertIsInstance(self.returned_obj.headers, email.message.Message)
 
     def test_url(self):
-        self.assertEqual(self.returned_obj.url, self.pathname)
+        self.assertEqual(self.returned_obj.url, self.quoted_pathname)
 
     def test_status(self):
         self.assertIsNone(self.returned_obj.status)
@@ -207,7 +208,7 @@ class urlopen_FileTests(unittest.TestCase):
         self.assertIsInstance(self.returned_obj.info(), email.message.Message)
 
     def test_geturl(self):
-        self.assertEqual(self.returned_obj.geturl(), self.pathname)
+        self.assertEqual(self.returned_obj.geturl(), self.quoted_pathname)
 
     def test_getcode(self):
         self.assertIsNone(self.returned_obj.getcode())
diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py
index e229e92085c..753b31eef0d 100644
--- a/Lib/test/test_uu.py
+++ b/Lib/test/test_uu.py
@@ -174,8 +174,9 @@ class UUStdIOTest(unittest.TestCase):
 class UUFileTest(unittest.TestCase):
 
     def setUp(self):
-        self.tmpin  = os_helper.TESTFN + "i"
-        self.tmpout = os_helper.TESTFN + "o"
+        # uu.encode() supports only ASCII file names
+        self.tmpin  = os_helper.TESTFN_ASCII + "i"
+        self.tmpout = os_helper.TESTFN_ASCII + "o"
         self.addCleanup(os_helper.unlink, self.tmpin)
         self.addCleanup(os_helper.unlink, self.tmpout)
 
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index d3191ed7b99..ca003d55d7c 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -80,8 +80,8 @@ class BaseTest(unittest.TestCase):
     def get_env_file(self, *args):
         return os.path.join(self.env_dir, *args)
 
-    def get_text_file_contents(self, *args):
-        with open(self.get_env_file(*args), 'r') as f:
+    def get_text_file_contents(self, *args, encoding='utf-8'):
+        with open(self.get_env_file(*args), 'r', encoding=encoding) as f:
             result = f.read()
         return result
 
diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py
index 65d0b557104..dcc0ea89b5a 100644
--- a/Lib/test/test_warnings/__init__.py
+++ b/Lib/test/test_warnings/__init__.py
@@ -940,8 +940,8 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
             """))
 
         def run(*args):
-            res = assert_python_ok(*args)
-            stderr = res.err.decode('ascii', 'replace')
+            res = assert_python_ok(*args, PYTHONIOENCODING='utf-8')
+            stderr = res.err.decode('utf-8', 'replace')
             stderr = '\n'.join(stderr.splitlines())
 
             # normalize newlines
@@ -1198,7 +1198,7 @@ class EnvironmentVariableTests(BaseTest):
     @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii',
                          'requires non-ascii filesystemencoding')
     def test_nonascii(self):
-        PYTHONWARNINGS="ignore:DeprecationWarning" + (support.FS_NONASCII or '')
+        PYTHONWARNINGS="ignore:DeprecationWarning" + support.FS_NONASCII
         rc, stdout, stderr = assert_python_ok("-c",
             "import sys; sys.stdout.write(str(sys.warnoptions))",
             PYTHONIOENCODING="utf-8",
diff --git a/Misc/NEWS.d/next/Tests/2020-06-22-00-21-12.bpo-41069.bLZkX-.rst b/Misc/NEWS.d/next/Tests/2020-06-22-00-21-12.bpo-41069.bLZkX-.rst
new file mode 100644
index 00000000000..14bbd1a39a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2020-06-22-00-21-12.bpo-41069.bLZkX-.rst
@@ -0,0 +1,2 @@
+:data:`test.support.TESTFN` and the current directory for tests when run via
+``test.regrtest`` contain now non-ascii characters if possible.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 808483ebd7b..adc5877c48a 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -4235,15 +4235,15 @@ static PyObject*
 pymarshal_write_long_to_file(PyObject* self, PyObject *args)
 {
     long value;
-    char *filename;
+    PyObject *filename;
     int version;
     FILE *fp;
 
-    if (!PyArg_ParseTuple(args, "lsi:pymarshal_write_long_to_file",
+    if (!PyArg_ParseTuple(args, "lOi:pymarshal_write_long_to_file",
                           &value, &filename, &version))
         return NULL;
 
-    fp = fopen(filename, "wb");
+    fp = _Py_fopen_obj(filename, "wb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -4261,15 +4261,15 @@ static PyObject*
 pymarshal_write_object_to_file(PyObject* self, PyObject *args)
 {
     PyObject *obj;
-    char *filename;
+    PyObject *filename;
     int version;
     FILE *fp;
 
-    if (!PyArg_ParseTuple(args, "Osi:pymarshal_write_object_to_file",
+    if (!PyArg_ParseTuple(args, "OOi:pymarshal_write_object_to_file",
                           &obj, &filename, &version))
         return NULL;
 
-    fp = fopen(filename, "wb");
+    fp = _Py_fopen_obj(filename, "wb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -4288,13 +4288,13 @@ pymarshal_read_short_from_file(PyObject* self, PyObject *args)
 {
     int value;
     long pos;
-    char *filename;
+    PyObject *filename;
     FILE *fp;
 
-    if (!PyArg_ParseTuple(args, "s:pymarshal_read_short_from_file", &filename))
+    if (!PyArg_ParseTuple(args, "O:pymarshal_read_short_from_file", &filename))
         return NULL;
 
-    fp = fopen(filename, "rb");
+    fp = _Py_fopen_obj(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -4313,13 +4313,13 @@ static PyObject*
 pymarshal_read_long_from_file(PyObject* self, PyObject *args)
 {
     long value, pos;
-    char *filename;
+    PyObject *filename;
     FILE *fp;
 
-    if (!PyArg_ParseTuple(args, "s:pymarshal_read_long_from_file", &filename))
+    if (!PyArg_ParseTuple(args, "O:pymarshal_read_long_from_file", &filename))
         return NULL;
 
-    fp = fopen(filename, "rb");
+    fp = _Py_fopen_obj(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -4339,13 +4339,13 @@ pymarshal_read_last_object_from_file(PyObject* self, PyObject *args)
 {
     PyObject *obj;
     long pos;
-    char *filename;
+    PyObject *filename;
     FILE *fp;
 
-    if (!PyArg_ParseTuple(args, "s:pymarshal_read_last_object_from_file", &filename))
+    if (!PyArg_ParseTuple(args, "O:pymarshal_read_last_object_from_file", &filename))
         return NULL;
 
-    fp = fopen(filename, "rb");
+    fp = _Py_fopen_obj(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -4363,13 +4363,13 @@ pymarshal_read_object_from_file(PyObject* self, PyObject *args)
 {
     PyObject *obj;
     long pos;
-    char *filename;
+    PyObject *filename;
     FILE *fp;
 
-    if (!PyArg_ParseTuple(args, "s:pymarshal_read_object_from_file", &filename))
+    if (!PyArg_ParseTuple(args, "O:pymarshal_read_object_from_file", &filename))
         return NULL;
 
-    fp = fopen(filename, "rb");
+    fp = _Py_fopen_obj(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;