diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 24d55451271..f22be2c52c1 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -263,7 +263,7 @@ class Charset: Returns "quoted-printable" if self.body_encoding is QP. Returns "base64" if self.body_encoding is BASE64. - Returns "7bit" otherwise. + Returns conversion function otherwise. """ assert self.body_encoding != SHORTEST if self.body_encoding == QP: @@ -381,7 +381,10 @@ class Charset: """Body-encode a string by converting it first to bytes. The type of encoding (base64 or quoted-printable) will be based on - self.body_encoding. + self.body_encoding. If body_encoding is None, we assume the + output charset is a 7bit encoding, so re-encoding the decoded + string using the ascii codec produces the correct string version + of the content. """ # 7bit/8bit encodings return the string unchanged (module conversions) if self.body_encoding is BASE64: @@ -391,4 +394,6 @@ class Charset: elif self.body_encoding is QP: return email.quoprimime.body_encode(string) else: + if isinstance(string, str): + string = string.encode(self.output_charset).decode('ascii') return string diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0ea441d9637..c66f4ccf382 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -54,10 +54,13 @@ def encode_7or8bit(msg): # There's no payload. For backwards compatibility we use 7bit msg['Content-Transfer-Encoding'] = '7bit' return - # We play a trick to make this go fast. If encoding to ASCII succeeds, we - # know the data must be 7bit, otherwise treat it as 8bit. + # We play a trick to make this go fast. If encoding/decode to ASCII + # succeeds, we know the data must be 7bit, otherwise treat it as 8bit. try: - orig.encode('ascii') + if isinstance(orig, str): + orig.encode('ascii') + else: + orig.decode('ascii') except UnicodeError: # iso-2022-* is non-ASCII but still 7-bit charset = msg.get_charset() diff --git a/Lib/email/message.py b/Lib/email/message.py index b821bfd6225..28835d09d95 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -242,7 +242,7 @@ class Message: raise TypeError('Expected list, got %s' % type(self._payload)) payload = self._payload cte = self.get('content-transfer-encoding', '').lower() - # payload can be bytes here, (I wonder if that is actually a bug?) + # payload may be bytes here. if isinstance(payload, str): if _has_surrogates(payload): bpayload = payload.encode('ascii', 'surrogateescape') diff --git a/Lib/email/test/test_email.py b/Lib/email/test/test_email.py index f2d5210727f..3fa8f9326ea 100644 --- a/Lib/email/test/test_email.py +++ b/Lib/email/test/test_email.py @@ -3372,9 +3372,9 @@ class TestCharset(unittest.TestCase): # built-in encodings where the header encoding is QP but the body # encoding is not. from email import charset as CharsetModule - CharsetModule.add_charset('fake', CharsetModule.QP, None) + CharsetModule.add_charset('fake', CharsetModule.QP, None, 'utf-8') c = Charset('fake') - eq('hello w\xf6rld', c.body_encode('hello w\xf6rld')) + eq('hello world', c.body_encode('hello world')) def test_unicode_charset_name(self): charset = Charset('us-ascii') diff --git a/Lib/email/test/test_email_codecs.py b/Lib/email/test/test_email_codecs.py index acc19c30f96..ca85f5731e0 100644 --- a/Lib/email/test/test_email_codecs.py +++ b/Lib/email/test/test_email_codecs.py @@ -13,7 +13,7 @@ from email.message import Message # We're compatible with Python 2.3, but it doesn't have the built-in Asian # codecs, so we have to skip all these tests. try: - str('foo', 'euc-jp') + str(b'foo', 'euc-jp') except LookupError: raise unittest.SkipTest @@ -22,11 +22,14 @@ except LookupError: class TestEmailAsianCodecs(TestEmailBase): def test_japanese_codecs(self): eq = self.ndiffAssertEqual - j = Charset("euc-jp") - g = Charset("iso-8859-1") + jcode = "euc-jp" + gcode = "iso-8859-1" + j = Charset(jcode) + g = Charset(gcode) h = Header("Hello World!") - jhello = '\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9\xa1\xaa' - ghello = 'Gr\xfc\xdf Gott!' + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode) + ghello = str(b'Gr\xfc\xdf Gott!', gcode) h.append(jhello, j) h.append(ghello, g) # BAW: This used to -- and maybe should -- fold the two iso-8859-1 @@ -36,13 +39,17 @@ class TestEmailAsianCodecs(TestEmailBase): # encoded word. eq(h.encode(), """\ Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?= - =?iso-8859-1?q?Gr=FC=DF?= =?iso-8859-1?q?_Gott!?=""") + =?iso-8859-1?q?Gr=FC=DF_Gott!?=""") eq(decode_header(h.encode()), - [('Hello World!', None), - ('\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'), - ('Gr\xfc\xdf Gott!', 'iso-8859-1')]) - int = 'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9' - h = Header(int, j, header_name="Subject") + [(b'Hello World!', None), + (b'\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'), + (b'Gr\xfc\xdf Gott!', gcode)]) + subject_bytes = (b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5' + b'\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2' + b'\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3' + b'\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9') + subject = str(subject_bytes, jcode) + h = Header(subject, j, header_name="Subject") # test a very long header enc = h.encode() # TK: splitting point may differ by codec design and/or Header encoding @@ -50,15 +57,24 @@ Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?= =?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKGyhC?= =?iso-2022-jp?b?GyRCMnE8VCROPjVHJyRyQlQkQyRGJCQkXiQ5GyhC?=""") # TK: full decode comparison - eq(h.__unicode__().encode('euc-jp'), int) + eq(str(h).encode(jcode), subject_bytes) + + def test_payload_encoding_utf8(self): + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') + msg = Message() + msg.set_payload(jhello, 'utf-8') + ustr = msg.get_payload(decode=True).decode(msg.get_content_charset()) + self.assertEqual(jhello, ustr) def test_payload_encoding(self): - jhello = '\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc\xa5\xeb\xa5\xc9\xa1\xaa' jcode = 'euc-jp' + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode) msg = Message() msg.set_payload(jhello, jcode) - ustr = str(msg.get_payload(), msg.get_content_charset()) - self.assertEqual(jhello, ustr.encode(jcode)) + ustr = msg.get_payload(decode=True).decode(msg.get_content_charset()) + self.assertEqual(jhello, ustr) diff --git a/Lib/test/test_email.py b/Lib/test/test_email.py index f0a18e71d02..5eebba5590c 100644 --- a/Lib/test/test_email.py +++ b/Lib/test/test_email.py @@ -3,10 +3,12 @@ # The specific tests now live in Lib/email/test from email.test.test_email import suite +from email.test.test_email_codecs import suite as codecs_suite from test import support def test_main(): support.run_unittest(suite()) + support.run_unittest(codecs_suite()) if __name__ == '__main__': test_main() diff --git a/Misc/ACKS b/Misc/ACKS index 453f86a0af8..f80a7f66112 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -375,6 +375,7 @@ Kevan Heydon Jason Hildebrand Richie Hindle Konrad Hinsen +Michael Henry David Hobley Tim Hochberg Joerg-Cyril Hoehle diff --git a/Misc/NEWS b/Misc/NEWS index 90e402b7577..018c62d8c8b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -71,6 +71,12 @@ Library - Issue #11289: `smtp.SMTP` class becomes a context manager so it can be used in a `with` statement. Contributed by Giampaolo Rodola. +- Issue #11554: Fixed support for Japanese codecs; previously the body output + encoding was not done if euc-jp or shift-jis was specified as the charset. + +- Issue #11509: Significantly increase test coverage of fileinput. + Patch by Denver Coneybeare at PyCon 2011 Sprints. + - Issue #11407: `TestCase.run` returns the result object used or created. Contributed by Janathan Hartley. @@ -206,6 +212,8 @@ Tools/Demos Tests ----- +- Issue #11554: Reactivated test_email_codecs. + - Issue #11505: improves test coverage of string.py - Issue #11490: test_subprocess:test_leaking_fds_on_error no longer gives a diff --git a/Modules/_ctypes/libffi/ChangeLog b/Modules/_ctypes/libffi/ChangeLog index 78853c42f56..00ba7efc1df 100644 --- a/Modules/_ctypes/libffi/ChangeLog +++ b/Modules/_ctypes/libffi/ChangeLog @@ -3084,7 +3084,7 @@ 2003-09-18 Kaz Kojima - * src/sh/ffi.c (ffi_prep_args): Take account into the alignment + * src/sh/ffi.c (ffi_prep_args): Take account into the alignement for the register size. (ffi_closure_helper_SYSV): Handle the structure return value address correctly. @@ -3344,7 +3344,7 @@ 2003-02-06 Andreas Tobler * libffi/src/powerpc/darwin_closure.S: - Fix alignment bug, allocate 8 bytes for the result. + Fix alignement bug, allocate 8 bytes for the result. * libffi/src/powerpc/aix_closure.S: Likewise. * libffi/src/powerpc/ffi_darwin.c: diff --git a/Modules/_ctypes/libffi/src/dlmalloc.c b/Modules/_ctypes/libffi/src/dlmalloc.c index 72673a1bc74..783c5c2a6f9 100644 --- a/Modules/_ctypes/libffi/src/dlmalloc.c +++ b/Modules/_ctypes/libffi/src/dlmalloc.c @@ -1326,7 +1326,7 @@ static void* win32direct_mmap(size_t size) { return (ptr != 0)? ptr: MFAIL; } -/* This function supports releasing coalesced segments */ +/* This function supports releasing coalesed segments */ static int win32munmap(void* ptr, size_t size) { MEMORY_BASIC_INFORMATION minfo; char* cptr = ptr; @@ -1362,7 +1362,7 @@ static int win32munmap(void* ptr, size_t size) { #define CALL_MORECORE(S) MFAIL #endif /* HAVE_MORECORE */ -/* mstate bit set if contiguous morecore disabled or failed */ +/* mstate bit set if continguous morecore disabled or failed */ #define USE_NONCONTIGUOUS_BIT (4U) /* segment bit set in create_mspace_with_base */ diff --git a/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c b/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c index 5e7772068ad..d84f1c393a8 100644 --- a/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c +++ b/Modules/_ctypes/libffi/src/powerpc/ffi_darwin.c @@ -592,7 +592,7 @@ typedef struct aix_fd_struct { +---------------------------------------+ 160 | result area 8 | +---------------------------------------+ 168 - | alignment to the next multiple of 16 | + | alignement to the next multiple of 16 | SP current --> +---------------------------------------+ 176 <- parent frame | back chain to caller 4 | +---------------------------------------+ 180 diff --git a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c index dba2184099a..8953d5fda35 100644 --- a/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c +++ b/Modules/_ctypes/libffi_osx/powerpc/ppc-ffi_darwin.c @@ -650,7 +650,7 @@ ffi_call( +---------------------------------------+ 160 | result area 8 | +---------------------------------------+ 168 - | alignment to the next multiple of 16 | + | alignement to the next multiple of 16 | SP current --> +---------------------------------------+ 176 <- parent frame | back chain to caller 4 | +---------------------------------------+ 180