diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 17695857e90..84a5f5138cc 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -196,13 +196,7 @@ Here are the methods of the :class:`Message` class: Set the entire message object's payload to *payload*. It is the client's responsibility to ensure the payload invariants. Optional *charset* sets - the message's character set; see :meth:`set_charset` for details. If - *payload* is a string containing non-ASCII characters, *charset* is - required. - - .. versionchanged:: 3.4 - Previous to 3.4 *charset* was not required when *payload* was a - non-ASCII string, but omitting it produced nonsense results. + the message's default character set; see :meth:`set_charset` for details. .. method:: set_charset(charset) diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 51f24b29710..892bab54a48 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -378,19 +378,18 @@ class Charset: return None def body_encode(self, string): - """Body-encode a string, converting it first to bytes if needed. + """Body-encode a string by converting it first to bytes. The type of encoding (base64 or quoted-printable) will be based on - self.body_encoding. If body_encoding is None, we perform no CTE - encoding (the CTE will be either 7bit or 8bit), we just encode the - binary representation to ascii using the surrogateescape error handler, - which will enable the Generators to produce the correct output. + self.body_encoding. If body_encoding is None, we assume the + output charset is a 7bit encoding, so re-encoding the decoded + string using the ascii codec produces the correct string version + of the content. """ - if not string: - return string - if isinstance(string, str): - string = string.encode(self.output_charset) + # 7bit/8bit encodings return the string unchanged (module conversions) if self.body_encoding is BASE64: + if isinstance(string, str): + string = string.encode(self.output_charset) return email.base64mime.body_encode(string) elif self.body_encoding is QP: # quopromime.body_encode takes a string, but operates on it as if @@ -399,7 +398,15 @@ class Charset: # character set, then, we must turn it into pseudo bytes via the # latin1 charset, which will encode any byte as a single code point # between 0 and 255, which is what body_encode is expecting. - string = string.decode('latin1') + # + # Note that this clause doesn't handle the case of a _payload that + # is already bytes. It never did, and the semantics of _payload + # being bytes has never been nailed down, so fixing that is a + # longer term TODO. + if isinstance(string, str): + string = string.encode(self.output_charset).decode('latin1') return email.quoprimime.body_encode(string) else: - return string.decode('ascii', 'surrogateescape') + if isinstance(string, str): + string = string.encode(self.output_charset).decode('ascii') + return string diff --git a/Lib/email/message.py b/Lib/email/message.py index 9d295fc3b41..ce673b0b92f 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -301,23 +301,9 @@ class Message: Optional charset sets the message's default character set. See set_charset() for details. """ - if hasattr(payload, 'encode'): - if charset is None: - try: - payload.encode('ascii', 'surrogateescape') - except UnicodeError: - raise TypeError("charset argument must be specified" - " when non-ASCII characters are used in the" - " payload") from None - self._payload = payload - return - if not isinstance(charset, Charset): - charset = Charset(charset) - payload = payload.encode(charset.output_charset) - if hasattr(payload, 'decode'): - self._payload = payload.decode('ascii', 'surrogateescape') - else: - self._payload = payload + if isinstance(payload, bytes): + payload = payload.decode('ascii', 'surrogateescape') + self._payload = payload if charset is not None: self.set_charset(charset) @@ -356,7 +342,7 @@ class Message: try: cte(self) except TypeError: - self._payload = charset.body_encode(self.get_payload(decode=True)) + self._payload = charset.body_encode(self._payload) self.add_header('Content-Transfer-Encoding', cte) def get_charset(self): diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index fa2d3d266a4..1629e2d8fd5 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -208,11 +208,12 @@ class TestRawDataManager(TestEmailBase): "Basìc tëxt.\n") def test_get_text_plain_utf8_base64_recoverable_bad_CTE_data(self): - m = self._bytes_msg(textwrap.dedent("""\ + m = self._str_msg(textwrap.dedent("""\ Content-Type: text/plain; charset="utf8" Content-Transfer-Encoding: base64 - QmFzw6xjIHTDq3h0Lgo""").encode('ascii') + b'\xFF=\n') + QmFzw6xjIHTDq3h0Lgo\xFF= + """)) self.assertEqual(raw_data_manager.get_content(m, errors='ignore'), "Basìc tëxt.\n") diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index acefb39504b..c66b5248cbc 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -92,44 +92,6 @@ class TestMessageAPI(TestEmailBase): msg.set_payload('This is a string payload', charset) self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') - def test_set_payload_with_8bit_data_and_charset(self): - data = b'\xd0\x90\xd0\x91\xd0\x92' - charset = Charset('utf-8') - msg = Message() - msg.set_payload(data, charset) - self.assertEqual(msg['content-transfer-encoding'], 'base64') - self.assertEqual(msg.get_payload(decode=True), data) - self.assertEqual(msg.get_payload(), '0JDQkdCS\n') - - def test_set_payload_with_non_ascii_and_charset_body_encoding_none(self): - data = b'\xd0\x90\xd0\x91\xd0\x92' - charset = Charset('utf-8') - charset.body_encoding = None # Disable base64 encoding - msg = Message() - msg.set_payload(data.decode('utf-8'), charset) - self.assertEqual(msg['content-transfer-encoding'], '8bit') - self.assertEqual(msg.get_payload(decode=True), data) - - def test_set_payload_with_8bit_data_and_charset_body_encoding_none(self): - data = b'\xd0\x90\xd0\x91\xd0\x92' - charset = Charset('utf-8') - charset.body_encoding = None # Disable base64 encoding - msg = Message() - msg.set_payload(data, charset) - self.assertEqual(msg['content-transfer-encoding'], '8bit') - self.assertEqual(msg.get_payload(decode=True), data) - - def test_set_payload_to_list(self): - msg = Message() - msg.set_payload([]) - self.assertEqual(msg.get_payload(), []) - - def test_set_payload_with_non_ascii_and_no_charset_raises(self): - data = b'\xd0\x90\xd0\x91\xd0\x92'.decode('utf-8') - msg = Message() - with self.assertRaises(TypeError): - msg.set_payload(data) - def test_get_charsets(self): eq = self.assertEqual @@ -596,10 +558,20 @@ class TestMessageAPI(TestEmailBase): self.assertIsInstance(msg.defects[0], errors.InvalidBase64CharactersDefect) + def test_broken_unicode_payload(self): + # This test improves coverage but is not a compliance test. + # The behavior in this situation is currently undefined by the API. + x = 'this is a br\xf6ken thing to do' + msg = Message() + msg['content-type'] = 'text/plain' + msg['content-transfer-encoding'] = '8bit' + msg.set_payload(x) + self.assertEqual(msg.get_payload(decode=True), + bytes(x, 'raw-unicode-escape')) + def test_questionable_bytes_payload(self): # This test improves coverage but is not a compliance test, - # since it involves poking inside the black box in a way - # that actually breaks the model invariants. + # since it involves poking inside the black box. x = 'this is a quéstionable thing to do'.encode('utf-8') msg = Message() msg['content-type'] = 'text/plain; charset="utf-8"'