Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bz2 uses bytes everywhere (even for the 'newlines' attribute). 

test_bz2 passes.
This commit is contained in:
Guido van Rossum 2007-05-22 23:13:45 +00:00
parent 776152b601
commit 522a6c66ac
2 changed files with 118 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Lib/test/test_bz2.py
View File

@ -15,9 +15,9 @@ has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx", "riscos")
class BaseTest(unittest.TestCase): class BaseTest(unittest.TestCase):
"Base for other testcases." "Base for other testcases."
TEXT = 'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' TEXT = b'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n'
DATA = 'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2<Q\xb5\x0fH\xd3\xd4\xdd\xd5\x87\xbb\xf8\x94\r\x8f\xafI\x12\xe1\xc9\xf8/E\x00pu\x89\x12]\xc9\xbbDL\nQ\x0e\t1\x12\xdf\xa0\xc0\x97\xac2O9\x89\x13\x94\x0e\x1c7\x0ed\x95I\x0c\xaaJ\xa4\x18L\x10\x05#\x9c\xaf\xba\xbc/\x97\x8a#C\xc8\xe1\x8cW\xf9\xe2\xd0\xd6M\xa7\x8bXa<e\x84t\xcbL\xb3\xa7\xd9\xcd\xd1\xcb\x84.\xaf\xb3\xab\xab\xad`n}\xa0lh\tE,\x8eZ\x15\x17VH>\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' DATA = b'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2<Q\xb5\x0fH\xd3\xd4\xdd\xd5\x87\xbb\xf8\x94\r\x8f\xafI\x12\xe1\xc9\xf8/E\x00pu\x89\x12]\xc9\xbbDL\nQ\x0e\t1\x12\xdf\xa0\xc0\x97\xac2O9\x89\x13\x94\x0e\x1c7\x0ed\x95I\x0c\xaaJ\xa4\x18L\x10\x05#\x9c\xaf\xba\xbc/\x97\x8a#C\xc8\xe1\x8cW\xf9\xe2\xd0\xd6M\xa7\x8bXa<e\x84t\xcbL\xb3\xa7\xd9\xcd\xd1\xcb\x84.\xaf\xb3\xab\xab\xad`n}\xa0lh\tE,\x8eZ\x15\x17VH>\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`'
DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1<l\xba\xcb_\xc00xY\x17r\x17\x88\x08\x08@\xa0\ry@\x10\x04$)`\xf2\xce\x89z\xb0s\xec\x9b.iW\x9d\x81\xb5-+t\x9f\x1a\'\x97dB\xf5x\xb5\xbe.[.\xd7\x0e\x81\xe7\x08\x1cN`\x88\x10\xca\x87\xc3!"\x80\x92R\xa1/\xd1\xc0\xe6mf\xac\xbd\x99\xcca\xb3\x8780>\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' DATA_CRLF = b'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1<l\xba\xcb_\xc00xY\x17r\x17\x88\x08\x08@\xa0\ry@\x10\x04$)`\xf2\xce\x89z\xb0s\xec\x9b.iW\x9d\x81\xb5-+t\x9f\x1a\'\x97dB\xf5x\xb5\xbe.[.\xd7\x0e\x81\xe7\x08\x1cN`\x88\x10\xca\x87\xc3!"\x80\x92R\xa1/\xd1\xc0\xe6mf\xac\xbd\x99\xcca\xb3\x8780>\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80'
if has_cmdline_bunzip2: if has_cmdline_bunzip2:
def decompress(self, data): def decompress(self, data):
@ -69,7 +69,7 @@ class BZ2FileTest(BaseTest):
# "Test BZ2File.read() in chunks of 10 bytes" # "Test BZ2File.read() in chunks of 10 bytes"
self.createTempFile() self.createTempFile()
bz2f = BZ2File(self.filename) bz2f = BZ2File(self.filename)
text = '' text = b''
while 1: while 1:
str = bz2f.read(10) str = bz2f.read(10)
if not str: if not str:
@ -117,7 +117,7 @@ class BZ2FileTest(BaseTest):
self.createTempFile() self.createTempFile()
bz2f = BZ2File(self.filename, "rU") bz2f = BZ2File(self.filename, "rU")
self.assertEqual(bz2f.read(), self.TEXT) self.assertEqual(bz2f.read(), self.TEXT)
self.assertEqual(bz2f.newlines, "\n") self.assertEqual(bz2f.newlines, b"\n")
bz2f.close() bz2f.close()
def testUniversalNewlinesCRLF(self): def testUniversalNewlinesCRLF(self):
@ -125,7 +125,7 @@ class BZ2FileTest(BaseTest):
self.createTempFile(crlf=1) self.createTempFile(crlf=1)
bz2f = BZ2File(self.filename, "rU") bz2f = BZ2File(self.filename, "rU")
self.assertEqual(bz2f.read(), self.TEXT) self.assertEqual(bz2f.read(), self.TEXT)
self.assertEqual(bz2f.newlines, "\r\n") self.assertEqual(bz2f.newlines, b"\r\n")
bz2f.close() bz2f.close()
def testWrite(self): def testWrite(self):
@ -207,7 +207,7 @@ class BZ2FileTest(BaseTest):
bz2f = BZ2File(self.filename) bz2f = BZ2File(self.filename)
bz2f.seek(150000) bz2f.seek(150000)
self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.tell(), len(self.TEXT))
self.assertEqual(bz2f.read(), "") self.assertEqual(bz2f.read(), b"")
bz2f.close() bz2f.close()
def testSeekPostEndTwice(self): def testSeekPostEndTwice(self):
@ -217,7 +217,7 @@ class BZ2FileTest(BaseTest):
bz2f.seek(150000) bz2f.seek(150000)
bz2f.seek(150000) bz2f.seek(150000)
self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.tell(), len(self.TEXT))
self.assertEqual(bz2f.read(), "") self.assertEqual(bz2f.read(), b"")
bz2f.close() bz2f.close()
def testSeekPreStart(self): def testSeekPreStart(self):
@ -252,18 +252,18 @@ class BZ2FileTest(BaseTest):
def testBug1191043(self): def testBug1191043(self):
# readlines() for files containing no newline # readlines() for files containing no newline
data = 'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' data = b'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t'
f = open(self.filename, "wb") f = open(self.filename, "wb")
f.write(data) f.write(data)
f.close() f.close()
bz2f = BZ2File(self.filename) bz2f = BZ2File(self.filename)
lines = bz2f.readlines() lines = bz2f.readlines()
bz2f.close() bz2f.close()
self.assertEqual(lines, ['Test']) self.assertEqual(lines, [b'Test'])
bz2f = BZ2File(self.filename) bz2f = BZ2File(self.filename)
xlines = list(bz2f.readlines()) xlines = list(bz2f.readlines())
bz2f.close() bz2f.close()
self.assertEqual(xlines, ['Test']) self.assertEqual(xlines, [b'Test'])
class BZ2CompressorTest(BaseTest): class BZ2CompressorTest(BaseTest):
@ -279,7 +279,7 @@ class BZ2CompressorTest(BaseTest):
# "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes"
bz2c = BZ2Compressor() bz2c = BZ2Compressor()
n = 0 n = 0
data = '' data = b''
while 1: while 1:
str = self.TEXT[n*10:(n+1)*10] str = self.TEXT[n*10:(n+1)*10]
if not str: if not str:
@ -303,7 +303,7 @@ class BZ2DecompressorTest(BaseTest):
def testDecompressChunks10(self): def testDecompressChunks10(self):
# "Test BZ2Decompressor.decompress() with chunks of 10 bytes" # "Test BZ2Decompressor.decompress() with chunks of 10 bytes"
bz2d = BZ2Decompressor() bz2d = BZ2Decompressor()
text = '' text = b''
n = 0 n = 0
while 1: while 1:
str = self.DATA[n*10:(n+1)*10] str = self.DATA[n*10:(n+1)*10]
@ -316,7 +316,7 @@ class BZ2DecompressorTest(BaseTest):
def testDecompressUnusedData(self): def testDecompressUnusedData(self):
# "Test BZ2Decompressor.decompress() with unused data" # "Test BZ2Decompressor.decompress() with unused data"
bz2d = BZ2Decompressor() bz2d = BZ2Decompressor()
unused_data = "this is unused data" unused_data = b"this is unused data"
text = bz2d.decompress(self.DATA+unused_data) text = bz2d.decompress(self.DATA+unused_data)
self.assertEqual(text, self.TEXT) self.assertEqual(text, self.TEXT)
self.assertEqual(bz2d.unused_data, unused_data) self.assertEqual(bz2d.unused_data, unused_data)
@ -343,8 +343,8 @@ class FuncTest(BaseTest):
def testDecompressEmpty(self): def testDecompressEmpty(self):
# "Test decompress() function with empty string" # "Test decompress() function with empty string"
text = bz2.decompress("") text = bz2.decompress(b"")
self.assertEqual(text, "") self.assertEqual(text, b"")
def testDecompressIncomplete(self): def testDecompressIncomplete(self):
# "Test decompress() function with incomplete data" # "Test decompress() function with incomplete data"

168
Modules/bz2module.c
View File

@ -34,7 +34,7 @@ typedef fpos_t Py_off_t;
#error "Large file support, but neither off_t nor fpos_t is large enough." #error "Large file support, but neither off_t nor fpos_t is large enough."
#endif #endif
#define BUF(v) PyString_AS_STRING((PyStringObject *)v) #define BUF(v) PyBytes_AS_STRING(v)
#define MODE_CLOSED 0 #define MODE_CLOSED 0
#define MODE_READ 1 #define MODE_READ 1
@ -238,7 +238,7 @@ Util_GetLine(BZ2FileObject *f, int n)
int univ_newline = f->f_univ_newline; int univ_newline = f->f_univ_newline;
total_v_size = n > 0 ? n : 100; total_v_size = n > 0 ? n : 100;
v = PyString_FromStringAndSize((char *)NULL, total_v_size); v = PyBytes_FromStringAndSize((char *)NULL, total_v_size);
if (v == NULL) if (v == NULL)
return NULL; return NULL;
@ -311,15 +311,21 @@ Util_GetLine(BZ2FileObject *f, int n)
Py_DECREF(v); Py_DECREF(v);
return NULL; return NULL;
} }
if (_PyString_Resize(&v, total_v_size) < 0) if (PyBytes_Resize(v, total_v_size) < 0) {
Py_DECREF(v);
return NULL; return NULL;
}
buf = BUF(v) + used_v_size; buf = BUF(v) + used_v_size;
end = BUF(v) + total_v_size; end = BUF(v) + total_v_size;
} }
used_v_size = buf - BUF(v); used_v_size = buf - BUF(v);
if (used_v_size != total_v_size) if (used_v_size != total_v_size) {
_PyString_Resize(&v, used_v_size); if (PyBytes_Resize(v, used_v_size) < 0) {
Py_DECREF(v);
v = NULL;
}
}
return v; return v;
} }
@ -442,10 +448,10 @@ Util_ReadAhead(BZ2FileObject *f, int bufsize)
/* This is a hacked version of Python's /* This is a hacked version of Python's
* fileobject.c:readahead_get_line_skip(). */ * fileobject.c:readahead_get_line_skip(). */
static PyStringObject * static PyBytesObject *
Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize) Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize)
{ {
PyStringObject* s; PyBytesObject* s;
char *bufptr; char *bufptr;
char *buf; char *buf;
int len; int len;
@ -456,17 +462,17 @@ Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize)
len = f->f_bufend - f->f_bufptr; len = f->f_bufend - f->f_bufptr;
if (len == 0) if (len == 0)
return (PyStringObject *) return (PyBytesObject *)
PyString_FromStringAndSize(NULL, skip); PyBytes_FromStringAndSize(NULL, skip);
bufptr = memchr(f->f_bufptr, '\n', len); bufptr = memchr(f->f_bufptr, '\n', len);
if (bufptr != NULL) { if (bufptr != NULL) {
bufptr++; /* Count the '\n' */ bufptr++; /* Count the '\n' */
len = bufptr - f->f_bufptr; len = bufptr - f->f_bufptr;
s = (PyStringObject *) s = (PyBytesObject *)
PyString_FromStringAndSize(NULL, skip+len); PyBytes_FromStringAndSize(NULL, skip+len);
if (s == NULL) if (s == NULL)
return NULL; return NULL;
memcpy(PyString_AS_STRING(s)+skip, f->f_bufptr, len); memcpy(PyBytes_AS_STRING(s)+skip, f->f_bufptr, len);
f->f_bufptr = bufptr; f->f_bufptr = bufptr;
if (bufptr == f->f_bufend) if (bufptr == f->f_bufend)
Util_DropReadAhead(f); Util_DropReadAhead(f);
@ -480,7 +486,7 @@ Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize)
PyMem_Free(buf); PyMem_Free(buf);
return NULL; return NULL;
} }
memcpy(PyString_AS_STRING(s)+skip, bufptr, len); memcpy(PyBytes_AS_STRING(s)+skip, bufptr, len);
PyMem_Free(buf); PyMem_Free(buf);
} }
return s; return s;
@ -513,7 +519,7 @@ BZ2File_read(BZ2FileObject *self, PyObject *args)
case MODE_READ: case MODE_READ:
break; break;
case MODE_READ_EOF: case MODE_READ_EOF:
ret = PyString_FromString(""); ret = PyBytes_FromStringAndSize("", 0);
goto cleanup; goto cleanup;
case MODE_CLOSED: case MODE_CLOSED:
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
@ -535,7 +541,7 @@ BZ2File_read(BZ2FileObject *self, PyObject *args)
"more than a Python string can hold"); "more than a Python string can hold");
goto cleanup; goto cleanup;
} }
ret = PyString_FromStringAndSize((char *)NULL, buffersize); ret = PyBytes_FromStringAndSize((char *)NULL, buffersize);
if (ret == NULL) if (ret == NULL)
goto cleanup; goto cleanup;
bytesread = 0; bytesread = 0;
@ -561,14 +567,21 @@ BZ2File_read(BZ2FileObject *self, PyObject *args)
} }
if (bytesrequested < 0) { if (bytesrequested < 0) {
buffersize = Util_NewBufferSize(buffersize); buffersize = Util_NewBufferSize(buffersize);
if (_PyString_Resize(&ret, buffersize) < 0) if (PyBytes_Resize(ret, buffersize) < 0) {
Py_DECREF(ret);
ret = NULL;
goto cleanup; goto cleanup;
}
} else { } else {
break; break;
} }
} }
if (bytesread != buffersize) if (bytesread != buffersize) {
_PyString_Resize(&ret, bytesread); if (PyBytes_Resize(ret, bytesread) < 0) {
Py_DECREF(ret);
ret = NULL;
}
}
cleanup: cleanup:
RELEASE_LOCK(self); RELEASE_LOCK(self);
@ -598,7 +611,7 @@ BZ2File_readline(BZ2FileObject *self, PyObject *args)
case MODE_READ: case MODE_READ:
break; break;
case MODE_READ_EOF: case MODE_READ_EOF:
ret = PyString_FromString(""); ret = PyBytes_FromStringAndSize("", 0);
goto cleanup; goto cleanup;
case MODE_CLOSED: case MODE_CLOSED:
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
@ -611,7 +624,7 @@ BZ2File_readline(BZ2FileObject *self, PyObject *args)
} }
if (sizehint == 0) if (sizehint == 0)
ret = PyString_FromString(""); ret = PyBytes_FromStringAndSize("", 0);
else else
ret = Util_GetLine(self, (sizehint < 0) ? 0 : sizehint); ret = Util_GetLine(self, (sizehint < 0) ? 0 : sizehint);
@ -705,26 +718,30 @@ BZ2File_readlines(BZ2FileObject *self, PyObject *args)
} }
if (big_buffer == NULL) { if (big_buffer == NULL) {
/* Create the big buffer */ /* Create the big buffer */
big_buffer = PyString_FromStringAndSize( big_buffer = PyBytes_FromStringAndSize(
NULL, buffersize); NULL, buffersize);
if (big_buffer == NULL) if (big_buffer == NULL)
goto error; goto error;
buffer = PyString_AS_STRING(big_buffer); buffer = PyBytes_AS_STRING(big_buffer);
memcpy(buffer, small_buffer, nfilled); memcpy(buffer, small_buffer, nfilled);
} }
else { else {
/* Grow the big buffer */ /* Grow the big buffer */
_PyString_Resize(&big_buffer, buffersize); if (PyBytes_Resize(big_buffer, buffersize) < 0){
buffer = PyString_AS_STRING(big_buffer); Py_DECREF(big_buffer);
big_buffer = NULL;
goto error;
}
buffer = PyBytes_AS_STRING(big_buffer);
} }
continue; continue;
} }
end = buffer+nfilled+nread; end = buffer+nfilled+nread;
q = buffer; q = buffer;
while (p != NULL) { while (p != NULL) {
/* Process complete lines */ /* Process complete lines */
p++; p++;
line = PyString_FromStringAndSize(q, p-q); line = PyBytes_FromStringAndSize(q, p-q);
if (line == NULL) if (line == NULL)
goto error; goto error;
err = PyList_Append(list, line); err = PyList_Append(list, line);
@ -747,18 +764,21 @@ BZ2File_readlines(BZ2FileObject *self, PyObject *args)
} }
if (nfilled != 0) { if (nfilled != 0) {
/* Partial last line */ /* Partial last line */
line = PyString_FromStringAndSize(buffer, nfilled); line = PyBytes_FromStringAndSize(buffer, nfilled);
if (line == NULL) if (line == NULL)
goto error; goto error;
if (sizehint > 0) { if (sizehint > 0) {
/* Need to complete the last line */ /* Need to complete the last line */
PyObject *rest = Util_GetLine(self, 0); PyObject *rest = Util_GetLine(self, 0);
PyObject *new;
if (rest == NULL) { if (rest == NULL) {
Py_DECREF(line); Py_DECREF(line);
goto error; goto error;
} }
PyString_Concat(&line, rest); new = PyBytes_Concat(line, rest);
Py_DECREF(line);
Py_DECREF(rest); Py_DECREF(rest);
line = new;
if (line == NULL) if (line == NULL)
goto error; goto error;
} }
@ -903,14 +923,14 @@ BZ2File_writelines(BZ2FileObject *self, PyObject *seq)
if (j == 0) if (j == 0)
break; break;
/* Check that all entries are indeed strings. If not, /* Check that all entries are indeed byte strings. If not,
apply the same rules as for file.write() and apply the same rules as for file.write() and
convert the rets to strings. This is slow, but convert the rets to strings. This is slow, but
seems to be the only way since all conversion APIs seems to be the only way since all conversion APIs
could potentially execute Python code. */ could potentially execute Python code. */
for (i = 0; i < j; i++) { for (i = 0; i < j; i++) {
PyObject *v = PyList_GET_ITEM(list, i); PyObject *v = PyList_GET_ITEM(list, i);
if (!PyString_Check(v)) { if (!PyBytes_Check(v)) {
const char *buffer; const char *buffer;
Py_ssize_t len; Py_ssize_t len;
if (PyObject_AsCharBuffer(v, &buffer, &len)) { if (PyObject_AsCharBuffer(v, &buffer, &len)) {
@ -918,10 +938,10 @@ BZ2File_writelines(BZ2FileObject *self, PyObject *seq)
"writelines() " "writelines() "
"argument must be " "argument must be "
"a sequence of " "a sequence of "
"strings"); "bytes objects");
goto error; goto error;
} }
line = PyString_FromStringAndSize(buffer, line = PyBytes_FromStringAndSize(buffer,
len); len);
if (line == NULL) if (line == NULL)
goto error; goto error;
@ -935,9 +955,9 @@ BZ2File_writelines(BZ2FileObject *self, PyObject *seq)
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
for (i = 0; i < j; i++) { for (i = 0; i < j; i++) {
line = PyList_GET_ITEM(list, i); line = PyList_GET_ITEM(list, i);
len = PyString_GET_SIZE(line); len = PyBytes_GET_SIZE(line);
BZ2_bzWrite (&bzerror, self->fp, BZ2_bzWrite (&bzerror, self->fp,
PyString_AS_STRING(line), len); PyBytes_AS_STRING(line), len);
if (bzerror != BZ_OK) { if (bzerror != BZ_OK) {
Py_BLOCK_THREADS Py_BLOCK_THREADS
Util_CatchBZ2Error(bzerror); Util_CatchBZ2Error(bzerror);
@ -1047,7 +1067,7 @@ BZ2File_seek(BZ2FileObject *self, PyObject *args)
offset = self->pos + offset; offset = self->pos + offset;
} }
/* Before getting here, offset must be the absolute position the file /* Before getting here, offset must be the absolute position the file
* pointer should be set to. */ * pointer should be set to. */
if (offset >= self->pos) { if (offset >= self->pos) {
@ -1206,13 +1226,13 @@ BZ2File_get_newlines(BZ2FileObject *self, void *closure)
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
case NEWLINE_CR: case NEWLINE_CR:
return PyString_FromString("\r"); return PyBytes_FromStringAndSize("\r", 1);
case NEWLINE_LF: case NEWLINE_LF:
return PyString_FromString("\n"); return PyBytes_FromStringAndSize("\n", 1);
case NEWLINE_CR|NEWLINE_LF: case NEWLINE_CR|NEWLINE_LF:
return Py_BuildValue("(ss)", "\r", "\n"); return Py_BuildValue("(ss)", "\r", "\n");
case NEWLINE_CRLF: case NEWLINE_CRLF:
return PyString_FromString("\r\n"); return PyBytes_FromStringAndSize("\r\n", 2);
case NEWLINE_CR|NEWLINE_CRLF: case NEWLINE_CR|NEWLINE_CRLF:
return Py_BuildValue("(ss)", "\r", "\r\n"); return Py_BuildValue("(ss)", "\r", "\r\n");
case NEWLINE_LF|NEWLINE_CRLF: case NEWLINE_LF|NEWLINE_CRLF:
@ -1220,8 +1240,8 @@ BZ2File_get_newlines(BZ2FileObject *self, void *closure)
case NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF: case NEWLINE_CR|NEWLINE_LF|NEWLINE_CRLF:
return Py_BuildValue("(sss)", "\r", "\n", "\r\n"); return Py_BuildValue("(sss)", "\r", "\n", "\r\n");
default: default:
PyErr_Format(PyExc_SystemError, PyErr_Format(PyExc_SystemError,
"Unknown newlines value 0x%x\n", "Unknown newlines value 0x%x\n",
self->f_newlinetypes); self->f_newlinetypes);
return NULL; return NULL;
} }
@ -1248,7 +1268,7 @@ BZ2File_get_name(BZ2FileObject *self, void *closure)
static PyGetSetDef BZ2File_getset[] = { static PyGetSetDef BZ2File_getset[] = {
{"closed", (getter)BZ2File_get_closed, NULL, {"closed", (getter)BZ2File_get_closed, NULL,
"True if the file is closed"}, "True if the file is closed"},
{"newlines", (getter)BZ2File_get_newlines, NULL, {"newlines", (getter)BZ2File_get_newlines, NULL,
"end-of-line convention used in this file"}, "end-of-line convention used in this file"},
{"mode", (getter)BZ2File_get_mode, NULL, {"mode", (getter)BZ2File_get_mode, NULL,
"file mode ('r', 'w', or 'U')"}, "file mode ('r', 'w', or 'U')"},
@ -1413,7 +1433,7 @@ BZ2File_getiter(BZ2FileObject *self)
static PyObject * static PyObject *
BZ2File_iternext(BZ2FileObject *self) BZ2File_iternext(BZ2FileObject *self)
{ {
PyStringObject* ret; PyBytesObject* ret;
ACQUIRE_LOCK(self); ACQUIRE_LOCK(self);
if (self->mode == MODE_CLOSED) { if (self->mode == MODE_CLOSED) {
PyErr_SetString(PyExc_ValueError, PyErr_SetString(PyExc_ValueError,
@ -1422,7 +1442,7 @@ BZ2File_iternext(BZ2FileObject *self)
} }
ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE); ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE);
RELEASE_LOCK(self); RELEASE_LOCK(self);
if (ret == NULL || PyString_GET_SIZE(ret) == 0) { if (ret == NULL || PyBytes_GET_SIZE(ret) == 0) {
Py_XDECREF(ret); Py_XDECREF(ret);
return NULL; return NULL;
} }
@ -1525,7 +1545,7 @@ BZ2Comp_compress(BZ2CompObject *self, PyObject *args)
return NULL; return NULL;
if (datasize == 0) if (datasize == 0)
return PyString_FromString(""); return PyBytes_FromStringAndSize("", 0);
ACQUIRE_LOCK(self); ACQUIRE_LOCK(self);
if (!self->running) { if (!self->running) {
@ -1534,7 +1554,7 @@ BZ2Comp_compress(BZ2CompObject *self, PyObject *args)
goto error; goto error;
} }
ret = PyString_FromStringAndSize(NULL, bufsize); ret = PyBytes_FromStringAndSize(NULL, bufsize);
if (!ret) if (!ret)
goto error; goto error;
@ -1557,7 +1577,7 @@ BZ2Comp_compress(BZ2CompObject *self, PyObject *args)
break; /* no more input data */ break; /* no more input data */
if (bzs->avail_out == 0) { if (bzs->avail_out == 0) {
bufsize = Util_NewBufferSize(bufsize); bufsize = Util_NewBufferSize(bufsize);
if (_PyString_Resize(&ret, bufsize) < 0) { if (PyBytes_Resize(ret, bufsize) < 0) {
BZ2_bzCompressEnd(bzs); BZ2_bzCompressEnd(bzs);
goto error; goto error;
} }
@ -1567,7 +1587,9 @@ BZ2Comp_compress(BZ2CompObject *self, PyObject *args)
} }
} }
_PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); if (PyBytes_Resize(ret,
(Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0)
goto error;
RELEASE_LOCK(self); RELEASE_LOCK(self);
return ret; return ret;
@ -1602,7 +1624,7 @@ BZ2Comp_flush(BZ2CompObject *self)
} }
self->running = 0; self->running = 0;
ret = PyString_FromStringAndSize(NULL, bufsize); ret = PyBytes_FromStringAndSize(NULL, bufsize);
if (!ret) if (!ret)
goto error; goto error;
@ -1623,7 +1645,7 @@ BZ2Comp_flush(BZ2CompObject *self)
} }
if (bzs->avail_out == 0) { if (bzs->avail_out == 0) {
bufsize = Util_NewBufferSize(bufsize); bufsize = Util_NewBufferSize(bufsize);
if (_PyString_Resize(&ret, bufsize) < 0) if (PyBytes_Resize(ret, bufsize) < 0)
goto error; goto error;
bzs->next_out = BUF(ret); bzs->next_out = BUF(ret);
bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs)
@ -1632,8 +1654,11 @@ BZ2Comp_flush(BZ2CompObject *self)
} }
} }
if (bzs->avail_out != 0) if (bzs->avail_out != 0) {
_PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); if (PyBytes_Resize(ret,
(Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0)
goto error;
}
RELEASE_LOCK(self); RELEASE_LOCK(self);
return ret; return ret;
@ -1816,7 +1841,7 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args)
goto error; goto error;
} }
ret = PyString_FromStringAndSize(NULL, bufsize); ret = PyBytes_FromStringAndSize(NULL, bufsize);
if (!ret) if (!ret)
goto error; goto error;
@ -1835,7 +1860,7 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args)
if (bzs->avail_in != 0) { if (bzs->avail_in != 0) {
Py_DECREF(self->unused_data); Py_DECREF(self->unused_data);
self->unused_data = self->unused_data =
PyString_FromStringAndSize(bzs->next_in, PyBytes_FromStringAndSize(bzs->next_in,
bzs->avail_in); bzs->avail_in);
} }
self->running = 0; self->running = 0;
@ -1849,7 +1874,7 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args)
break; /* no more input data */ break; /* no more input data */
if (bzs->avail_out == 0) { if (bzs->avail_out == 0) {
bufsize = Util_NewBufferSize(bufsize); bufsize = Util_NewBufferSize(bufsize);
if (_PyString_Resize(&ret, bufsize) < 0) { if (PyBytes_Resize(ret, bufsize) < 0) {
BZ2_bzDecompressEnd(bzs); BZ2_bzDecompressEnd(bzs);
goto error; goto error;
} }
@ -1860,8 +1885,11 @@ BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args)
} }
} }
if (bzs->avail_out != 0) if (bzs->avail_out != 0) {
_PyString_Resize(&ret, (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)); if (PyBytes_Resize(ret,
(Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0)
goto error;
}
RELEASE_LOCK(self); RELEASE_LOCK(self);
return ret; return ret;
@ -1897,7 +1925,7 @@ BZ2Decomp_init(BZ2DecompObject *self, PyObject *args, PyObject *kwargs)
} }
#endif #endif
self->unused_data = PyString_FromString(""); self->unused_data = PyBytes_FromStringAndSize("", 0);
if (!self->unused_data) if (!self->unused_data)
goto error; goto error;
@ -2031,7 +2059,7 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs)
* data in one shot. We will check it later anyway. */ * data in one shot. We will check it later anyway. */
bufsize = datasize + (datasize/100+1) + 600; bufsize = datasize + (datasize/100+1) + 600;
ret = PyString_FromStringAndSize(NULL, bufsize); ret = PyBytes_FromStringAndSize(NULL, bufsize);
if (!ret) if (!ret)
return NULL; return NULL;
@ -2063,7 +2091,7 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs)
} }
if (bzs->avail_out == 0) { if (bzs->avail_out == 0) {
bufsize = Util_NewBufferSize(bufsize); bufsize = Util_NewBufferSize(bufsize);
if (_PyString_Resize(&ret, bufsize) < 0) { if (PyBytes_Resize(ret, bufsize) < 0) {
BZ2_bzCompressEnd(bzs); BZ2_bzCompressEnd(bzs);
Py_DECREF(ret); Py_DECREF(ret);
return NULL; return NULL;
@ -2073,8 +2101,12 @@ bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs)
} }
} }
if (bzs->avail_out != 0) if (bzs->avail_out != 0) {
_PyString_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)); if (PyBytes_Resize(ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)) < 0) {
Py_DECREF(ret);
ret = NULL;
}
}
BZ2_bzCompressEnd(bzs); BZ2_bzCompressEnd(bzs);
return ret; return ret;
@ -2102,9 +2134,9 @@ bz2_decompress(PyObject *self, PyObject *args)
return NULL; return NULL;
if (datasize == 0) if (datasize == 0)
return PyString_FromString(""); return PyBytes_FromStringAndSize("", 0);
ret = PyString_FromStringAndSize(NULL, bufsize); ret = PyBytes_FromStringAndSize(NULL, bufsize);
if (!ret) if (!ret)
return NULL; return NULL;
@ -2143,7 +2175,7 @@ bz2_decompress(PyObject *self, PyObject *args)
} }
if (bzs->avail_out == 0) { if (bzs->avail_out == 0) {
bufsize = Util_NewBufferSize(bufsize); bufsize = Util_NewBufferSize(bufsize);
if (_PyString_Resize(&ret, bufsize) < 0) { if (PyBytes_Resize(ret, bufsize) < 0) {
BZ2_bzDecompressEnd(bzs); BZ2_bzDecompressEnd(bzs);
Py_DECREF(ret); Py_DECREF(ret);
return NULL; return NULL;
@ -2153,8 +2185,12 @@ bz2_decompress(PyObject *self, PyObject *args)
} }
} }
if (bzs->avail_out != 0) if (bzs->avail_out != 0) {
_PyString_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)); if (PyBytes_Resize(ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)) < 0) {
Py_DECREF(ret);
ret = NULL;
}
}
BZ2_bzDecompressEnd(bzs); BZ2_bzDecompressEnd(bzs);
return ret; return ret;