Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Merged revisions 81471-81472 via svnmerge from 

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r81471 | victor.stinner | 2010-05-22 15:37:56 +0200 (sam., 22 mai 2010) | 7 lines

  Issue #6268: More bugfixes about BOM, UTF-16 and UTF-32

   * Fix seek() method of codecs.open(), don't write the BOM twice after seek(0)
   * Fix reset() method of codecs, UTF-16, UTF-32 and StreamWriter classes
   * test_codecs: use "w+" mode instead of "wt+". "t" mode is not supported by
     Solaris or Windows, but does it really exist? I found it the in the issue.
........
  r81472 | victor.stinner | 2010-05-22 15:44:25 +0200 (sam., 22 mai 2010) | 4 lines

  Fix my last commit (r81471) about codecs

  Rememder: don't touch the code just before a commit
........
This commit is contained in:
Victor Stinner 2010-05-22 16:52:13 +00:00
parent e37049ccd7
commit 09c0f24f39
5 changed files with 78 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Lib/codecs.py
View File

@ -370,6 +370,11 @@ class StreamWriter(Codec):
"""
pass
def seek(self, offset, whence=0):
self.stream.seek(offset, whence)
if whence == 0 and offset == 0:
self.reset()
def __getattr__(self, name,
getattr=getattr):
@ -601,8 +606,8 @@ class StreamReader(Codec):
Resets the codec buffers used for keeping state.
"""
self.reset()
self.stream.seek(offset, whence)
self.reset()
def next(self):
@ -695,8 +700,10 @@ class StreamReaderWriter:
self.writer.reset()
def seek(self, offset, whence=0):
self.reader.seek(offset, whence)
self.writer.seek(offset, whence)
self.stream.seek(offset, whence)
self.reader.reset()
if whence == 0 and offset == 0:
self.writer.reset()
def __getattr__(self, name,
getattr=getattr):

20
Lib/encodings/utf_16.py
View File

@ -58,17 +58,23 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
class StreamWriter(codecs.StreamWriter):
def __init__(self, stream, errors='strict'):
self.bom_written = False
codecs.StreamWriter.__init__(self, stream, errors)
self.encoder = None
def reset(self):
codecs.StreamWriter.reset(self)
self.encoder = None
def encode(self, input, errors='strict'):
self.bom_written = True
result = codecs.utf_16_encode(input, errors)
if sys.byteorder == 'little':
self.encode = codecs.utf_16_le_encode
if self.encoder is None:
result = codecs.utf_16_encode(input, errors)
if sys.byteorder == 'little':
self.encoder = codecs.utf_16_le_encode
else:
self.encoder = codecs.utf_16_be_encode
return result
else:
self.encode = codecs.utf_16_be_encode
return result
return self.encoder(input, errors)
class StreamReader(codecs.StreamReader):

20
Lib/encodings/utf_32.py
View File

@ -98,17 +98,23 @@ class IncrementalDecoder(codecs.BufferedIncrementalDecoder):
class StreamWriter(codecs.StreamWriter):
def __init__(self, stream, errors='strict'):
self.bom_written = False
self.encoder = None
codecs.StreamWriter.__init__(self, stream, errors)
def reset(self):
codecs.StreamWriter.reset(self)
self.encoder = None
def encode(self, input, errors='strict'):
self.bom_written = True
result = codecs.utf_32_encode(input, errors)
if sys.byteorder == 'little':
self.encode = codecs.utf_32_le_encode
if self.encoder is None:
result = codecs.utf_32_encode(input, errors)
if sys.byteorder == 'little':
self.encoder = codecs.utf_32_le_encode
else:
self.encoder = codecs.utf_32_be_encode
return result
else:
self.encode = codecs.utf_32_be_encode
return result
return self.encoder(input, errors)
class StreamReader(codecs.StreamReader):

42
Lib/test/test_codecs.py
View File

@ -1490,7 +1490,7 @@ class WithStmtTest(unittest.TestCase):
class BomTest(unittest.TestCase):
def test_seek0(self):
data = "1234567890"
data = u"1234567890"
tests = ("utf-16",
"utf-16-le",
"utf-16-be",
@ -1498,8 +1498,8 @@ class BomTest(unittest.TestCase):
"utf-32-le",
"utf-32-be")
for encoding in tests:
with codecs.open('foo', 'wt+', encoding=encoding) as f:
# Check if the BOM is written only once
# Check if the BOM is written only once
with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
f.write(data)
f.write(data)
f.seek(0)
@ -1507,6 +1507,42 @@ class BomTest(unittest.TestCase):
f.seek(0)
self.assertEquals(f.read(), data * 2)
# Check that the BOM is written after a seek(0)
with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
f.write(data[0])
self.assertNotEquals(f.tell(), 0)
f.seek(0)
f.write(data)
f.seek(0)
self.assertEquals(f.read(), data)
# (StreamWriter) Check that the BOM is written after a seek(0)
with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
f.writer.write(data[0])
self.assertNotEquals(f.writer.tell(), 0)
f.writer.seek(0)
f.writer.write(data)
f.seek(0)
self.assertEquals(f.read(), data)
# Check that the BOM is not written after a seek() at a position
# different than the start
with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
f.write(data)
f.seek(f.tell())
f.write(data)
f.seek(0)
self.assertEquals(f.read(), data * 2)
# (StreamWriter) Check that the BOM is not written after a seek()
# at a position different than the start
with codecs.open(test_support.TESTFN, 'w+', encoding=encoding) as f:
f.writer.write(data)
f.writer.seek(f.writer.tell())
f.writer.write(data)
f.seek(0)
self.assertEquals(f.read(), data * 2)
def test_main():
test_support.run_unittest(

5
Misc/NEWS
View File

@ -57,8 +57,9 @@ Library
- Issue #3924: Ignore cookies with invalid "version" field in cookielib.
- Issue #6268: Fix seek() method of codecs.open(), don't read the BOM twice
after seek(0)
- Issue #6268: Fix seek() method of codecs.open(), don't read or write the BOM
twice after seek(0). Fix also reset() method of codecs, UTF-16, UTF-32 and
StreamWriter classes.
- Issue #5640: Fix Shift-JIS incremental encoder for error handlers different
than strict