Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping" 

in any mapping, not only in an unicode string.
This commit is contained in:
Serhiy Storchaka 2013-01-15 15:30:04 +02:00
parent b946af5897 45d16d9924
commit 55e2cb497b
3 changed files with 82 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
Lib/test/test_codecs.py
View File

@ -1737,6 +1737,10 @@ class CharmapTest(unittest.TestCase):
codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
)
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe"
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
@ -1793,6 +1797,17 @@ class CharmapTest(unittest.TestCase):
{0: 'a', 1: 'b'}
)
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, b"\x00\x01\x02", "strict",
{0: 'a', 1: 'b', 2: None}
)
# Issue #14850
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, b"\x00\x01\x02", "strict",
{0: 'a', 1: 'b', 2: '\ufffe'}
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: 'a', 1: 'b'}),
@ -1805,6 +1820,13 @@ class CharmapTest(unittest.TestCase):
("ab\ufffd", 3)
)
# Issue #14850
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: 'a', 1: 'b', 2: '\ufffe'}),
("ab\ufffd", 3)
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: 'a', 1: 'b'}),
@ -1817,6 +1839,13 @@ class CharmapTest(unittest.TestCase):
("ab", 3)
)
# Issue #14850
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: 'a', 1: 'b', 2: '\ufffe'}),
("ab", 3)
)
allbytes = bytes(range(256))
self.assertEqual(
codecs.charmap_decode(allbytes, "ignore", {}),
@ -1857,18 +1886,35 @@ class CharmapTest(unittest.TestCase):
{0: a, 1: b},
)
self.assertRaises(UnicodeDecodeError,
codecs.charmap_decode, b"\x00\x01\x02", "strict",
{0: a, 1: b, 2: 0xFFFE},
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: a, 1: b}),
("ab\ufffd", 3)
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: a, 1: b, 2: 0xFFFE}),
("ab\ufffd", 3)
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: a, 1: b}),
("ab", 3)
)
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: a, 1: b, 2: 0xFFFE}),
("ab", 3)
)
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):

3
Misc/NEWS
View File

@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1?
Core and Builtins
-----------------
- Issue #14850: Now a chamap decoder treates U+FFFE as "undefined mapping"
in any mapping, not only in a string.
- Issue #16730: importlib.machinery.FileFinder now no longers raises an
exception when trying to populate its cache and it finds out the directory is
unreadable or has turned into a file. Reported and diagnosed by

52
Objects/unicodeobject.c
View File

@ -7393,15 +7393,18 @@ Error:
if (PyErr_ExceptionMatches(PyExc_LookupError)) {
/* No mapping found means: mapping is undefined. */
PyErr_Clear();
x = Py_None;
Py_INCREF(x);
goto Undefined;
} else
goto onError;
}
/* Apply mapping */
if (x == Py_None)
goto Undefined;
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
if (value == 0xFFFE)
goto Undefined;
if (value < 0 || value > MAX_UNICODE) {
PyErr_Format(PyExc_TypeError,
"character mapping must be in range(0x%lx)",
@ -7415,25 +7418,23 @@ Error:
PyUnicode_WRITE(writer.kind, writer.data, writer.pos, value);
writer.pos++;
}
else if (x == Py_None) {
/* undefined mapping */
startinpos = s-starts;
endinpos = startinpos+1;
if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"charmap", "character maps to <undefined>",
&starts, &e, &startinpos, &endinpos, &exc, &s,
&writer)) {
Py_DECREF(x);
goto onError;
}
Py_DECREF(x);
continue;
}
else if (PyUnicode_Check(x)) {
writer.overallocate = 1;
if (_PyUnicodeWriter_WriteStr(&writer, x) == -1)
if (PyUnicode_READY(x) == -1)
goto onError;
if (PyUnicode_GET_LENGTH(x) == 1) {
Py_UCS4 value = PyUnicode_READ_CHAR(x, 0);
if (value == 0xFFFE)
goto Undefined;
if (_PyUnicodeWriter_Prepare(&writer, 1, value) == -1)
goto onError;
PyUnicode_WRITE(writer.kind, writer.data, writer.pos, value);
writer.pos++;
}
else {
writer.overallocate = 1;
if (_PyUnicodeWriter_WriteStr(&writer, x) == -1)
goto onError;
}
}
else {
/* wrong return value */
@ -7444,6 +7445,19 @@ Error:
}
Py_DECREF(x);
++s;
continue;
Undefined:
/* undefined mapping */
Py_XDECREF(x);
startinpos = s-starts;
endinpos = startinpos+1;
if (unicode_decode_call_errorhandler_writer(
errors, &errorHandler,
"charmap", "character maps to <undefined>",
&starts, &e, &startinpos, &endinpos, &exc, &s,
&writer)) {
goto onError;
}
}
}
Py_XDECREF(errorHandler);