traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Issue #17897: Optimized unpickle prefetching.

This commit is contained in:
Serhiy Storchaka 2013-11-30 23:15:38 +02:00
parent cc6e87b2cf
commit 6fe39b76a9
2 changed files with 27 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Misc/NEWS
View File

@ -18,6 +18,8 @@ Core and Builtins
Library Library
------- -------
- Issue #17897: Optimized unpickle prefetching.
- Issue #3693: Make the error message more helpful when the array.array() - Issue #3693: Make the error message more helpful when the array.array()
constructor is given a str. Move the array module typecode documentation to constructor is given a str. Move the array module typecode documentation to
the docstring of the constructor. the docstring of the constructor.

58
Modules/_pickle.c
View File

@ -1121,7 +1121,7 @@ static Py_ssize_t
_Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
{ {
PyObject *data; PyObject *data;
Py_ssize_t read_size, prefetched_size = 0; Py_ssize_t read_size;
assert(self->read != NULL); assert(self->read != NULL);
@ -1134,7 +1134,29 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
Py_DECREF(empty_tuple); Py_DECREF(empty_tuple);
} }
else { else {
PyObject *len = PyLong_FromSsize_t(n); PyObject *len;
/* Prefetch some data without advancing the file pointer, if possible */
if (self->peek && n < PREFETCH) {
len = PyLong_FromSsize_t(PREFETCH);
if (len == NULL)
return -1;
data = _Pickle_FastCall(self->peek, len);
if (data == NULL) {
if (!PyErr_ExceptionMatches(PyExc_NotImplementedError))
return -1;
/* peek() is probably not supported by the given file object */
PyErr_Clear();
Py_CLEAR(self->peek);
}
else {
read_size = _Unpickler_SetStringInput(self, data);
Py_DECREF(data);
self->prefetched_idx = 0;
if (n <= read_size)
return n;
}
}
len = PyLong_FromSsize_t(n);
if (len == NULL) if (len == NULL)
return -1; return -1;
data = _Pickle_FastCall(self->read, len); data = _Pickle_FastCall(self->read, len);
@ -1142,38 +1164,8 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
if (data == NULL) if (data == NULL)
return -1; return -1;
/* Prefetch some data without advancing the file pointer, if possible */ read_size = _Unpickler_SetStringInput(self, data);
if (self->peek) {
PyObject *len, *prefetched;
len = PyLong_FromSsize_t(PREFETCH);
if (len == NULL) {
Py_DECREF(data);
return -1;
}
prefetched = _Pickle_FastCall(self->peek, len);
if (prefetched == NULL) {
if (PyErr_ExceptionMatches(PyExc_NotImplementedError)) {
/* peek() is probably not supported by the given file object */
PyErr_Clear();
Py_CLEAR(self->peek);
}
else {
Py_DECREF(data);
return -1;
}
}
else {
assert(PyBytes_Check(prefetched));
prefetched_size = PyBytes_GET_SIZE(prefetched);
PyBytes_ConcatAndDel(&data, prefetched);
if (data == NULL)
return -1;
}
}
read_size = _Unpickler_SetStringInput(self, data) - prefetched_size;
Py_DECREF(data); Py_DECREF(data);
self->prefetched_idx = read_size;
return read_size; return read_size;
} }