traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #4874: Most builtin decoders now reject unicode input.

This commit is contained in:
Antoine Pitrou 2009-01-22 10:11:36 +00:00
parent dd01f8f37b
commit 81fabdb437
3 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
Lib/test/test_codecs.py
View File

@ -1475,6 +1475,36 @@ class WithStmtTest(unittest.TestCase):
info.streamwriter, 'strict') as srw: info.streamwriter, 'strict') as srw:
self.assertEquals(srw.read(), "\xfc") self.assertEquals(srw.read(), "\xfc")
class TypesTest(unittest.TestCase):
def test_decode_unicode(self):
# Most decoders don't accept unicode input
decoders = [
codecs.utf_7_decode,
codecs.utf_8_decode,
codecs.utf_16_le_decode,
codecs.utf_16_be_decode,
codecs.utf_16_ex_decode,
codecs.utf_32_decode,
codecs.utf_32_le_decode,
codecs.utf_32_be_decode,
codecs.utf_32_ex_decode,
codecs.latin_1_decode,
codecs.ascii_decode,
codecs.charmap_decode,
]
if hasattr(codecs, "mbcs_decode"):
decoders.append(codecs.mbcs_decode)
for decoder in decoders:
self.assertRaises(TypeError, decoder, "xxx")
def test_unicode_escape(self):
# Escape-decoding an unicode string is supported ang gives the same
# result as decoding the equivalent ASCII bytes string.
self.assertEquals(codecs.unicode_escape_decode(r"\u1234"), ("\u1234", 6))
self.assertEquals(codecs.unicode_escape_decode(br"\u1234"), ("\u1234", 6))
self.assertEquals(codecs.raw_unicode_escape_decode(r"\u1234"), ("\u1234", 6))
self.assertEquals(codecs.raw_unicode_escape_decode(br"\u1234"), ("\u1234", 6))
def test_main(): def test_main():
support.run_unittest( support.run_unittest(
@ -1501,6 +1531,7 @@ def test_main():
BasicUnicodeTest, BasicUnicodeTest,
CharmapTest, CharmapTest,
WithStmtTest, WithStmtTest,
TypesTest,
) )

2
Misc/NEWS
View File

@ -12,6 +12,8 @@ What's New in Python 3.1 alpha 0
Core and Builtins Core and Builtins
----------------- -----------------
- Issue #4874: Most builtin decoders now reject unicode input.
- Issue #4842: Don't allow trailing 'L' when constructing an integer - Issue #4842: Don't allow trailing 'L' when constructing an integer
from a string. from a string.

28
Modules/_codecsmodule.c
View File

@ -258,7 +258,7 @@ utf_7_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded = NULL; PyObject *decoded = NULL;
if (!PyArg_ParseTuple(args, "s*|zi:utf_7_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_7_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; consumed = pbuf.len;
@ -281,7 +281,7 @@ utf_8_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded = NULL; PyObject *decoded = NULL;
if (!PyArg_ParseTuple(args, "s*|zi:utf_8_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_8_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; consumed = pbuf.len;
@ -305,7 +305,7 @@ utf_16_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded; PyObject *decoded;
if (!PyArg_ParseTuple(args, "s*|zi:utf_16_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_16_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; /* This is overwritten unless final is true. */ consumed = pbuf.len; /* This is overwritten unless final is true. */
@ -328,7 +328,7 @@ utf_16_le_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded = NULL; PyObject *decoded = NULL;
if (!PyArg_ParseTuple(args, "s*|zi:utf_16_le_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_16_le_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
@ -352,7 +352,7 @@ utf_16_be_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded = NULL; PyObject *decoded = NULL;
if (!PyArg_ParseTuple(args, "s*|zi:utf_16_be_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_16_be_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
@ -384,7 +384,7 @@ utf_16_ex_decode(PyObject *self,
int final = 0; int final = 0;
Py_ssize_t consumed; Py_ssize_t consumed;
if (!PyArg_ParseTuple(args, "s*|zii:utf_16_ex_decode", if (!PyArg_ParseTuple(args, "y*|zii:utf_16_ex_decode",
&pbuf, &errors, &byteorder, &final)) &pbuf, &errors, &byteorder, &final))
return NULL; return NULL;
consumed = pbuf.len; /* This is overwritten unless final is true. */ consumed = pbuf.len; /* This is overwritten unless final is true. */
@ -409,7 +409,7 @@ utf_32_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded; PyObject *decoded;
if (!PyArg_ParseTuple(args, "s*|zi:utf_32_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_32_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; /* This is overwritten unless final is true. */ consumed = pbuf.len; /* This is overwritten unless final is true. */
@ -432,7 +432,7 @@ utf_32_le_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded; PyObject *decoded;
if (!PyArg_ParseTuple(args, "s*|zi:utf_32_le_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_32_le_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; /* This is overwritten unless final is true. */ consumed = pbuf.len; /* This is overwritten unless final is true. */
@ -455,7 +455,7 @@ utf_32_be_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded; PyObject *decoded;
if (!PyArg_ParseTuple(args, "s*|zi:utf_32_be_decode", if (!PyArg_ParseTuple(args, "y*|zi:utf_32_be_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; /* This is overwritten unless final is true. */ consumed = pbuf.len; /* This is overwritten unless final is true. */
@ -486,7 +486,7 @@ utf_32_ex_decode(PyObject *self,
int final = 0; int final = 0;
Py_ssize_t consumed; Py_ssize_t consumed;
if (!PyArg_ParseTuple(args, "s*|zii:utf_32_ex_decode", if (!PyArg_ParseTuple(args, "y*|zii:utf_32_ex_decode",
&pbuf, &errors, &byteorder, &final)) &pbuf, &errors, &byteorder, &final))
return NULL; return NULL;
consumed = pbuf.len; /* This is overwritten unless final is true. */ consumed = pbuf.len; /* This is overwritten unless final is true. */
@ -542,7 +542,7 @@ latin_1_decode(PyObject *self,
PyObject *unicode; PyObject *unicode;
const char *errors = NULL; const char *errors = NULL;
if (!PyArg_ParseTuple(args, "s*|z:latin_1_decode", if (!PyArg_ParseTuple(args, "y*|z:latin_1_decode",
&pbuf, &errors)) &pbuf, &errors))
return NULL; return NULL;
@ -559,7 +559,7 @@ ascii_decode(PyObject *self,
PyObject *unicode; PyObject *unicode;
const char *errors = NULL; const char *errors = NULL;
if (!PyArg_ParseTuple(args, "s*|z:ascii_decode", if (!PyArg_ParseTuple(args, "y*|z:ascii_decode",
&pbuf, &errors)) &pbuf, &errors))
return NULL; return NULL;
@ -577,7 +577,7 @@ charmap_decode(PyObject *self,
const char *errors = NULL; const char *errors = NULL;
PyObject *mapping = NULL; PyObject *mapping = NULL;
if (!PyArg_ParseTuple(args, "s*|zO:charmap_decode", if (!PyArg_ParseTuple(args, "y*|zO:charmap_decode",
&pbuf, &errors, &mapping)) &pbuf, &errors, &mapping))
return NULL; return NULL;
if (mapping == Py_None) if (mapping == Py_None)
@ -600,7 +600,7 @@ mbcs_decode(PyObject *self,
Py_ssize_t consumed; Py_ssize_t consumed;
PyObject *decoded = NULL; PyObject *decoded = NULL;
if (!PyArg_ParseTuple(args, "s*|zi:mbcs_decode", if (!PyArg_ParseTuple(args, "y*|zi:mbcs_decode",
&pbuf, &errors, &final)) &pbuf, &errors, &final))
return NULL; return NULL;
consumed = pbuf.len; consumed = pbuf.len;