traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-36698: IDLE no longer fails when write non-encodable characters to stderr. (GH-16583) 

It now escapes them with a backslash, as the regular Python interpreter.
Added the "errors" field to the standard streams.
(cherry picked from commit b690a2759e)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>
This commit is contained in:
Miss Islington (bot) 2019-10-08 04:51:16 -07:00 committed by GitHub
parent aa9d5b8ec3
commit a1f45008f1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 87 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
Lib/idlelib/idle_test/test_run.py
View File

@ -36,7 +36,7 @@ class RunTest(unittest.TestCase):
self.assertIn('UnhashableException: ex1', tb[10]) self.assertIn('UnhashableException: ex1', tb[10])
# PseudoFile tests. # StdioFile tests.
class S(str): class S(str):
def __str__(self): def __str__(self):
@ -68,14 +68,14 @@ class MockShell:
self.lines = list(lines)[::-1] self.lines = list(lines)[::-1]
class PseudeInputFilesTest(unittest.TestCase): class StdInputFilesTest(unittest.TestCase):
def test_misc(self): def test_misc(self):
shell = MockShell() shell = MockShell()
f = run.PseudoInputFile(shell, 'stdin', 'utf-8') f = run.StdInputFile(shell, 'stdin')
self.assertIsInstance(f, io.TextIOBase) self.assertIsInstance(f, io.TextIOBase)
self.assertEqual(f.encoding, 'utf-8') self.assertEqual(f.encoding, 'utf-8')
self.assertIsNone(f.errors) self.assertEqual(f.errors, 'strict')
self.assertIsNone(f.newlines) self.assertIsNone(f.newlines)
self.assertEqual(f.name, '<stdin>') self.assertEqual(f.name, '<stdin>')
self.assertFalse(f.closed) self.assertFalse(f.closed)
@ -86,7 +86,7 @@ class PseudeInputFilesTest(unittest.TestCase):
def test_unsupported(self): def test_unsupported(self):
shell = MockShell() shell = MockShell()
f = run.PseudoInputFile(shell, 'stdin', 'utf-8') f = run.StdInputFile(shell, 'stdin')
self.assertRaises(OSError, f.fileno) self.assertRaises(OSError, f.fileno)
self.assertRaises(OSError, f.tell) self.assertRaises(OSError, f.tell)
self.assertRaises(OSError, f.seek, 0) self.assertRaises(OSError, f.seek, 0)
@ -95,7 +95,7 @@ class PseudeInputFilesTest(unittest.TestCase):
def test_read(self): def test_read(self):
shell = MockShell() shell = MockShell()
f = run.PseudoInputFile(shell, 'stdin', 'utf-8') f = run.StdInputFile(shell, 'stdin')
shell.push(['one\n', 'two\n', '']) shell.push(['one\n', 'two\n', ''])
self.assertEqual(f.read(), 'one\ntwo\n') self.assertEqual(f.read(), 'one\ntwo\n')
shell.push(['one\n', 'two\n', '']) shell.push(['one\n', 'two\n', ''])
@ -115,7 +115,7 @@ class PseudeInputFilesTest(unittest.TestCase):
def test_readline(self): def test_readline(self):
shell = MockShell() shell = MockShell()
f = run.PseudoInputFile(shell, 'stdin', 'utf-8') f = run.StdInputFile(shell, 'stdin')
shell.push(['one\n', 'two\n', 'three\n', 'four\n']) shell.push(['one\n', 'two\n', 'three\n', 'four\n'])
self.assertEqual(f.readline(), 'one\n') self.assertEqual(f.readline(), 'one\n')
self.assertEqual(f.readline(-1), 'two\n') self.assertEqual(f.readline(-1), 'two\n')
@ -140,7 +140,7 @@ class PseudeInputFilesTest(unittest.TestCase):
def test_readlines(self): def test_readlines(self):
shell = MockShell() shell = MockShell()
f = run.PseudoInputFile(shell, 'stdin', 'utf-8') f = run.StdInputFile(shell, 'stdin')
shell.push(['one\n', 'two\n', '']) shell.push(['one\n', 'two\n', ''])
self.assertEqual(f.readlines(), ['one\n', 'two\n']) self.assertEqual(f.readlines(), ['one\n', 'two\n'])
shell.push(['one\n', 'two\n', '']) shell.push(['one\n', 'two\n', ''])
@ -161,7 +161,7 @@ class PseudeInputFilesTest(unittest.TestCase):
def test_close(self): def test_close(self):
shell = MockShell() shell = MockShell()
f = run.PseudoInputFile(shell, 'stdin', 'utf-8') f = run.StdInputFile(shell, 'stdin')
shell.push(['one\n', 'two\n', '']) shell.push(['one\n', 'two\n', ''])
self.assertFalse(f.closed) self.assertFalse(f.closed)
self.assertEqual(f.readline(), 'one\n') self.assertEqual(f.readline(), 'one\n')
@ -171,14 +171,14 @@ class PseudeInputFilesTest(unittest.TestCase):
self.assertRaises(TypeError, f.close, 1) self.assertRaises(TypeError, f.close, 1)
class PseudeOutputFilesTest(unittest.TestCase): class StdOutputFilesTest(unittest.TestCase):
def test_misc(self): def test_misc(self):
shell = MockShell() shell = MockShell()
f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') f = run.StdOutputFile(shell, 'stdout')
self.assertIsInstance(f, io.TextIOBase) self.assertIsInstance(f, io.TextIOBase)
self.assertEqual(f.encoding, 'utf-8') self.assertEqual(f.encoding, 'utf-8')
self.assertIsNone(f.errors) self.assertEqual(f.errors, 'strict')
self.assertIsNone(f.newlines) self.assertIsNone(f.newlines)
self.assertEqual(f.name, '<stdout>') self.assertEqual(f.name, '<stdout>')
self.assertFalse(f.closed) self.assertFalse(f.closed)
@ -189,7 +189,7 @@ class PseudeOutputFilesTest(unittest.TestCase):
def test_unsupported(self): def test_unsupported(self):
shell = MockShell() shell = MockShell()
f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') f = run.StdOutputFile(shell, 'stdout')
self.assertRaises(OSError, f.fileno) self.assertRaises(OSError, f.fileno)
self.assertRaises(OSError, f.tell) self.assertRaises(OSError, f.tell)
self.assertRaises(OSError, f.seek, 0) self.assertRaises(OSError, f.seek, 0)
@ -198,16 +198,36 @@ class PseudeOutputFilesTest(unittest.TestCase):
def test_write(self): def test_write(self):
shell = MockShell() shell = MockShell()
f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') f = run.StdOutputFile(shell, 'stdout')
f.write('test') f.write('test')
self.assertEqual(shell.written, [('test', 'stdout')]) self.assertEqual(shell.written, [('test', 'stdout')])
shell.reset() shell.reset()
f.write('t\xe8st') f.write('t\xe8\u015b\U0001d599')
self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) self.assertEqual(shell.written, [('t\xe8\u015b\U0001d599', 'stdout')])
shell.reset() shell.reset()
f.write(S('t\xe8st')) f.write(S('t\xe8\u015b\U0001d599'))
self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) self.assertEqual(shell.written, [('t\xe8\u015b\U0001d599', 'stdout')])
self.assertEqual(type(shell.written[0][0]), str)
shell.reset()
self.assertRaises(TypeError, f.write)
self.assertEqual(shell.written, [])
self.assertRaises(TypeError, f.write, b'test')
self.assertRaises(TypeError, f.write, 123)
self.assertEqual(shell.written, [])
self.assertRaises(TypeError, f.write, 'test', 'spam')
self.assertEqual(shell.written, [])
def test_write_stderr_nonencodable(self):
shell = MockShell()
f = run.StdOutputFile(shell, 'stderr', 'iso-8859-15', 'backslashreplace')
f.write('t\xe8\u015b\U0001d599\xa4')
self.assertEqual(shell.written, [('t\xe8\\u015b\\U0001d599\\xa4', 'stderr')])
shell.reset()
f.write(S('t\xe8\u015b\U0001d599\xa4'))
self.assertEqual(shell.written, [('t\xe8\\u015b\\U0001d599\\xa4', 'stderr')])
self.assertEqual(type(shell.written[0][0]), str) self.assertEqual(type(shell.written[0][0]), str)
shell.reset() shell.reset()
@ -221,7 +241,7 @@ class PseudeOutputFilesTest(unittest.TestCase):
def test_writelines(self): def test_writelines(self):
shell = MockShell() shell = MockShell()
f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') f = run.StdOutputFile(shell, 'stdout')
f.writelines([]) f.writelines([])
self.assertEqual(shell.written, []) self.assertEqual(shell.written, [])
shell.reset() shell.reset()
@ -251,7 +271,7 @@ class PseudeOutputFilesTest(unittest.TestCase):
def test_close(self): def test_close(self):
shell = MockShell() shell = MockShell()
f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') f = run.StdOutputFile(shell, 'stdout')
self.assertFalse(f.closed) self.assertFalse(f.closed)
f.write('test') f.write('test')
f.close() f.close()

41
Lib/idlelib/iomenu.py
View File

@ -15,6 +15,7 @@ from idlelib.config import idleConf
if idlelib.testing: # Set True by test.test_idle to avoid setlocale. if idlelib.testing: # Set True by test.test_idle to avoid setlocale.
encoding = 'utf-8' encoding = 'utf-8'
errors = 'surrogateescape'
else: else:
# Try setting the locale, so that we can find out # Try setting the locale, so that we can find out
# what encoding to use # what encoding to use
@ -24,15 +25,9 @@ else:
except (ImportError, locale.Error): except (ImportError, locale.Error):
pass pass
locale_decode = 'ascii'
if sys.platform == 'win32': if sys.platform == 'win32':
# On Windows, we could use "mbcs". However, to give the user encoding = 'utf-8'
# a portable encoding name, we need to find the code page errors = 'surrogateescape'
try:
locale_encoding = locale.getdefaultlocale()[1]
codecs.lookup(locale_encoding)
except LookupError:
pass
else: else:
try: try:
# Different things can fail here: the locale module may not be # Different things can fail here: the locale module may not be
@ -40,30 +35,30 @@ else:
# resulting codeset may be unknown to Python. We ignore all # resulting codeset may be unknown to Python. We ignore all
# these problems, falling back to ASCII # these problems, falling back to ASCII
locale_encoding = locale.nl_langinfo(locale.CODESET) locale_encoding = locale.nl_langinfo(locale.CODESET)
if locale_encoding is None or locale_encoding == '': if locale_encoding:
# situation occurs on macOS codecs.lookup(locale_encoding)
locale_encoding = 'ascii'
codecs.lookup(locale_encoding)
except (NameError, AttributeError, LookupError): except (NameError, AttributeError, LookupError):
# Try getdefaultlocale: it parses environment variables, # Try getdefaultlocale: it parses environment variables,
# which may give a clue. Unfortunately, getdefaultlocale has # which may give a clue. Unfortunately, getdefaultlocale has
# bugs that can cause ValueError. # bugs that can cause ValueError.
try: try:
locale_encoding = locale.getdefaultlocale()[1] locale_encoding = locale.getdefaultlocale()[1]
if locale_encoding is None or locale_encoding == '': if locale_encoding:
# situation occurs on macOS codecs.lookup(locale_encoding)
locale_encoding = 'ascii'
codecs.lookup(locale_encoding)
except (ValueError, LookupError): except (ValueError, LookupError):
pass pass
locale_encoding = locale_encoding.lower() if locale_encoding:
encoding = locale_encoding.lower()
encoding = locale_encoding errors = 'strict'
# Encoding is used in multiple files; locale_encoding nowhere. else:
# The only use of 'encoding' below is in _decode as initial value # POSIX locale or macOS
# of deprecated block asking user for encoding. encoding = 'ascii'
# Perhaps use elsewhere should be reviewed. errors = 'surrogateescape'
# Encoding is used in multiple files; locale_encoding nowhere.
# The only use of 'encoding' below is in _decode as initial value
# of deprecated block asking user for encoding.
# Perhaps use elsewhere should be reviewed.
coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)

14
Lib/idlelib/pyshell.py
View File

@ -54,7 +54,7 @@ from idlelib.editor import EditorWindow, fixwordbreaks
from idlelib.filelist import FileList from idlelib.filelist import FileList
from idlelib.outwin import OutputWindow from idlelib.outwin import OutputWindow
from idlelib import rpc from idlelib import rpc
from idlelib.run import idle_formatwarning, PseudoInputFile, PseudoOutputFile from idlelib.run import idle_formatwarning, StdInputFile, StdOutputFile
from idlelib.undo import UndoDelegator from idlelib.undo import UndoDelegator
HOST = '127.0.0.1' # python execution server on localhost loopback HOST = '127.0.0.1' # python execution server on localhost loopback
@ -902,10 +902,14 @@ class PyShell(OutputWindow):
self.save_stderr = sys.stderr self.save_stderr = sys.stderr
self.save_stdin = sys.stdin self.save_stdin = sys.stdin
from idlelib import iomenu from idlelib import iomenu
self.stdin = PseudoInputFile(self, "stdin", iomenu.encoding) self.stdin = StdInputFile(self, "stdin",
self.stdout = PseudoOutputFile(self, "stdout", iomenu.encoding) iomenu.encoding, iomenu.errors)
self.stderr = PseudoOutputFile(self, "stderr", iomenu.encoding) self.stdout = StdOutputFile(self, "stdout",
self.console = PseudoOutputFile(self, "console", iomenu.encoding) iomenu.encoding, iomenu.errors)
self.stderr = StdOutputFile(self, "stderr",
iomenu.encoding, "backslashreplace")
self.console = StdOutputFile(self, "console",
iomenu.encoding, iomenu.errors)
if not use_subprocess: if not use_subprocess:
sys.stdout = self.stdout sys.stdout = self.stdout
sys.stderr = self.stderr sys.stderr = self.stderr

36
Lib/idlelib/run.py
View File

@ -401,17 +401,22 @@ class MyRPCServer(rpc.RPCServer):
# Pseudofiles for shell-remote communication (also used in pyshell) # Pseudofiles for shell-remote communication (also used in pyshell)
class PseudoFile(io.TextIOBase): class StdioFile(io.TextIOBase):
def __init__(self, shell, tags, encoding=None): def __init__(self, shell, tags, encoding='utf-8', errors='strict'):
self.shell = shell self.shell = shell
self.tags = tags self.tags = tags
self._encoding = encoding self._encoding = encoding
self._errors = errors
@property @property
def encoding(self): def encoding(self):
return self._encoding return self._encoding
@property
def errors(self):
return self._errors
@property @property
def name(self): def name(self):
return '<%s>' % self.tags return '<%s>' % self.tags
@ -420,7 +425,7 @@ class PseudoFile(io.TextIOBase):
return True return True
class PseudoOutputFile(PseudoFile): class StdOutputFile(StdioFile):
def writable(self): def writable(self):
return True return True
@ -428,19 +433,12 @@ class PseudoOutputFile(PseudoFile):
def write(self, s): def write(self, s):
if self.closed: if self.closed:
raise ValueError("write to closed file") raise ValueError("write to closed file")
if type(s) is not str: s = str.encode(s, self.encoding, self.errors).decode(self.encoding, self.errors)
if not isinstance(s, str):
raise TypeError('must be str, not ' + type(s).__name__)
# See issue #19481
s = str.__str__(s)
return self.shell.write(s, self.tags) return self.shell.write(s, self.tags)
class PseudoInputFile(PseudoFile): class StdInputFile(StdioFile):
_line_buffer = ''
def __init__(self, shell, tags, encoding=None):
PseudoFile.__init__(self, shell, tags, encoding)
self._line_buffer = ''
def readable(self): def readable(self):
return True return True
@ -495,12 +493,12 @@ class MyHandler(rpc.RPCHandler):
executive = Executive(self) executive = Executive(self)
self.register("exec", executive) self.register("exec", executive)
self.console = self.get_remote_proxy("console") self.console = self.get_remote_proxy("console")
sys.stdin = PseudoInputFile(self.console, "stdin", sys.stdin = StdInputFile(self.console, "stdin",
iomenu.encoding) iomenu.encoding, iomenu.errors)
sys.stdout = PseudoOutputFile(self.console, "stdout", sys.stdout = StdOutputFile(self.console, "stdout",
iomenu.encoding) iomenu.encoding, iomenu.errors)
sys.stderr = PseudoOutputFile(self.console, "stderr", sys.stderr = StdOutputFile(self.console, "stderr",
iomenu.encoding) iomenu.encoding, "backslashreplace")
sys.displayhook = rpc.displayhook sys.displayhook = rpc.displayhook
# page help() text to shell. # page help() text to shell.

3
Misc/NEWS.d/next/IDLE/2019-10-04-18-03-09.bpo-36698.BKcmom.rst Normal file
View File

@ -0,0 +1,3 @@
IDLE no longer fails when write non-encodable characters to stderr. It now
escapes them with a backslash, as the regular Python interpreter. Added the
``errors`` field to the standard streams.