traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed issue #4533: File read operation was dreadfully slow

This commit is contained in:
Christian Heimes 2008-12-05 08:26:55 +00:00
parent 7a259ca8e3
commit a872de55dc
2 changed files with 31 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Misc/NEWS
View File

@ -12,6 +12,10 @@ What's New in Python 3.1 alpha 0
Core and Builtins
-----------------
- Issue #4533: File read operation was dreadfully slow due to a slowly
growing read buffer. Fixed by using the same growth rate algorithm as
Python 2.x.
Library
-------

31
Modules/_fileio.c
View File

@ -27,6 +27,20 @@
#include <windows.h>
#endif
#if BUFSIZ < (8*1024)
#define SMALLCHUNK (8*1024)
#elif (BUFSIZ >= (2 << 25))
#error "unreasonable BUFSIZ > 64MB defined"
#else
#define SMALLCHUNK BUFSIZ
#endif
#if SIZEOF_INT < 4
#define BIGCHUNK (512 * 32)
#else
#define BIGCHUNK (512 * 1024)
#endif
typedef struct {
PyObject_HEAD
int fd;
@ -387,8 +401,6 @@ fileio_readinto(PyFileIOObject *self, PyObject *args)
return PyLong_FromSsize_t(n);
}
#define DEFAULT_BUFFER_SIZE (8*1024)
static PyObject *
fileio_readall(PyFileIOObject *self)
{
@ -396,12 +408,23 @@ fileio_readall(PyFileIOObject *self)
Py_ssize_t total = 0;
int n;
result = PyBytes_FromStringAndSize(NULL, DEFAULT_BUFFER_SIZE);
result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK);
if (result == NULL)
return NULL;
while (1) {
Py_ssize_t newsize = total + DEFAULT_BUFFER_SIZE;
Py_ssize_t newsize = (total < SMALLCHUNK) ? SMALLCHUNK : total;
/* Keep doubling until we reach BIGCHUNK;
then keep adding BIGCHUNK. */
if (newsize <= BIGCHUNK) {
newsize += newsize;
}
else {
/* NOTE: overflow impossible due to limits on BUFSIZ */
newsize += BIGCHUNK;
}
if (PyBytes_GET_SIZE(result) < newsize) {
if (_PyBytes_Resize(&result, newsize) < 0) {
if (total == 0) {