traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #15379: Fix passing of non-BMP characters as integers for the charmap decoder (already working as unicode strings). 

Patch by Serhiy Storchaka.
This commit is contained in:
Antoine Pitrou 2012-11-17 21:14:58 +01:00
parent 1df43d33d2
commit e3ae321222
4 changed files with 135 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Lib/test/test_codeccallbacks.py
View File

@ -717,7 +717,7 @@ class CodecCallbackTest(unittest.TestCase):
raise ValueError
self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1})
self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: 0x110000})
def test_encodehelper(self):
# enhance coverage of:

105
Lib/test/test_codecs.py
View File

@ -1534,6 +1534,10 @@ class CharmapTest(unittest.TestCase):
(u"ab", 3)
)
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, b"\x00\x01\x02", "strict", u"ab"
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"),
(u"ab", 3)
@ -1545,6 +1549,107 @@ class CharmapTest(unittest.TestCase):
(u"", len(allbytes))
)
def test_decode_with_int2str_map(self):
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "strict",
{0: u'a', 1: u'b', 2: u'c'}),
(u"abc", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "strict",
{0: u'Aa', 1: u'Bb', 2: u'Cc'}),
(u"AaBbCc", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "strict",
{0: u'\U0010FFFF', 1: u'b', 2: u'c'}),
(u"\U0010FFFFbc", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "strict",
{0: u'a', 1: u'b', 2: u''}),
(u"ab", 3)
)
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, "\x00\x01\x02", "strict",
{0: u'a', 1: u'b'}
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace",
{0: u'a', 1: u'b'}),
(u"ab\ufffd", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace",
{0: u'a', 1: u'b', 2: None}),
(u"ab\ufffd", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore",
{0: u'a', 1: u'b'}),
(u"ab", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore",
{0: u'a', 1: u'b', 2: None}),
(u"ab", 3)
)
allbytes = bytes(range(256))
self.assertEqual(
codecs.charmap_decode(allbytes, "ignore", {}),
(u"", len(allbytes))
)
def test_decode_with_int2int_map(self):
a = ord(u'a')
b = ord(u'b')
c = ord(u'c')
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "strict",
{0: a, 1: b, 2: c}),
(u"abc", 3)
)
# Issue #15379
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "strict",
{0: 0x10FFFF, 1: b, 2: c}),
(u"\U0010FFFFbc", 3)
)
self.assertRaises(TypeError,
codecs.charmap_decode, "\x00\x01\x02", "strict",
{0: 0x110000, 1: b, 2: c}
)
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, "\x00\x01\x02", "strict",
{0: a, 1: b},
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace",
{0: a, 1: b}),
(u"ab\ufffd", 3)
)
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "ignore",
{0: a, 1: b}),
(u"ab", 3)
)
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
f = StringIO.StringIO("\xc3\xbc")

3
Misc/NEWS
View File

@ -9,6 +9,9 @@ What's New in Python 2.7.4
Core and Builtins
-----------------
- Issue #15379: Fix passing of non-BMP characters as integers for the charmap
decoder (already working as unicode strings). Patch by Serhiy Storchaka.
- Issue #16453: Fix equality testing of dead weakref objects.
- Issue #9535: Fix pending signals that have been received but not yet

28
Objects/unicodeobject.c
View File

@ -4127,12 +4127,36 @@ PyObject *PyUnicode_DecodeCharmap(const char *s,
/* Apply mapping */
if (PyInt_Check(x)) {
long value = PyInt_AS_LONG(x);
if (value < 0 || value > 65535) {
if (value < 0 || value > 0x10FFFF) {
PyErr_SetString(PyExc_TypeError,
"character mapping must be in range(65536)");
"character mapping must be in range(0x110000)");
Py_DECREF(x);
goto onError;
}
#ifndef Py_UNICODE_WIDE
if (value > 0xFFFF) {
/* see the code for 1-n mapping below */
if (extrachars < 2) {
/* resize first */
Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
Py_ssize_t needed = 10 - extrachars;
extrachars += needed;
/* XXX overflow detection missing */
if (_PyUnicode_Resize(&v,
PyUnicode_GET_SIZE(v) + needed) < 0) {
Py_DECREF(x);
goto onError;
}
p = PyUnicode_AS_UNICODE(v) + oldpos;
}
value -= 0x10000;
*p++ = 0xD800 | (value >> 10);
*p++ = 0xDC00 | (value & 0x3FF);
extrachars -= 2;
}
else
#endif
*p++ = (Py_UNICODE)value;
}
else if (x == Py_None) {