traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue 3739: The unicode-internal encoder now reports the number of *characters* 

consumed like any other encoder (instead of the number of bytes).
This commit is contained in:
Walter Dörwald 2009-05-06 14:28:24 +00:00
parent c924b3d115
commit a7fb408a02
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Lib/test/test_codecs.py
View File

@ -802,6 +802,12 @@ class UnicodeInternalTest(unittest.TestCase):
"UnicodeInternalTest") "UnicodeInternalTest")
self.assertEquals((u"ab", 12), ignored) self.assertEquals((u"ab", 12), ignored)
def test_encode_length(self):
# Issue 3739
encoder = codecs.getencoder("unicode_internal")
self.assertEquals(encoder(u"a")[1], 1)
self.assertEquals(encoder(u"\xe9\u0142")[1], 2)
# From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
nameprep_tests = [ nameprep_tests = [
# 3.1 Map to nothing. # 3.1 Map to nothing.
@ -1292,8 +1298,7 @@ class BasicUnicodeTest(unittest.TestCase):
name = "latin_1" name = "latin_1"
self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-")) self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
(bytes, size) = codecs.getencoder(encoding)(s) (bytes, size) = codecs.getencoder(encoding)(s)
if encoding != "unicode_internal": self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
(chars, size) = codecs.getdecoder(encoding)(bytes) (chars, size) = codecs.getdecoder(encoding)(bytes)
self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding)) self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding))

2
Modules/_codecsmodule.c
View File

@ -646,7 +646,7 @@ unicode_internal_encode(PyObject *self,
data = PyUnicode_AS_DATA(obj); data = PyUnicode_AS_DATA(obj);
size = PyUnicode_GET_DATA_SIZE(obj); size = PyUnicode_GET_DATA_SIZE(obj);
return codec_tuple(PyString_FromStringAndSize(data, size), return codec_tuple(PyString_FromStringAndSize(data, size),
size); PyUnicode_GET_SIZE(obj));
} }
else { else {
if (PyObject_AsReadBuffer(obj, (const void **)&data, &size)) if (PyObject_AsReadBuffer(obj, (const void **)&data, &size))