mirror of https://github.com/python/cpython
Issue #11564: Avoid crashes when trying to pickle huge objects or containers
(more than 2**31 items). Instead, in most cases, an OverflowError is raised.
This commit is contained in:
commit
ee763e2acc
|
@ -2,11 +2,15 @@ import io
|
||||||
import unittest
|
import unittest
|
||||||
import pickle
|
import pickle
|
||||||
import pickletools
|
import pickletools
|
||||||
|
import sys
|
||||||
import copyreg
|
import copyreg
|
||||||
import weakref
|
import weakref
|
||||||
from http.cookies import SimpleCookie
|
from http.cookies import SimpleCookie
|
||||||
|
|
||||||
from test.support import TestFailed, TESTFN, run_with_locale, no_tracing
|
from test.support import (
|
||||||
|
TestFailed, TESTFN, run_with_locale, no_tracing,
|
||||||
|
_2G, _4G, precisionbigmemtest,
|
||||||
|
)
|
||||||
|
|
||||||
from pickle import bytes_types
|
from pickle import bytes_types
|
||||||
|
|
||||||
|
@ -15,6 +19,8 @@ from pickle import bytes_types
|
||||||
# kind of outer loop.
|
# kind of outer loop.
|
||||||
protocols = range(pickle.HIGHEST_PROTOCOL + 1)
|
protocols = range(pickle.HIGHEST_PROTOCOL + 1)
|
||||||
|
|
||||||
|
character_size = 4 if sys.maxunicode > 0xFFFF else 2
|
||||||
|
|
||||||
|
|
||||||
# Return True if opcode code appears in the pickle, else False.
|
# Return True if opcode code appears in the pickle, else False.
|
||||||
def opcode_in_pickle(code, pickle):
|
def opcode_in_pickle(code, pickle):
|
||||||
|
@ -1127,6 +1133,99 @@ class AbstractPickleTests(unittest.TestCase):
|
||||||
if proto >= 2:
|
if proto >= 2:
|
||||||
self.assertLessEqual(sizes[-1], 14)
|
self.assertLessEqual(sizes[-1], 14)
|
||||||
|
|
||||||
|
def check_negative_32b_binXXX(self, dumped):
|
||||||
|
if sys.maxsize > 2**32:
|
||||||
|
self.skipTest("test is only meaningful on 32-bit builds")
|
||||||
|
# XXX Pure Python pickle reads lengths as signed and passes
|
||||||
|
# them directly to read() (hence the EOFError)
|
||||||
|
with self.assertRaises((pickle.UnpicklingError, EOFError,
|
||||||
|
ValueError, OverflowError)):
|
||||||
|
self.loads(dumped)
|
||||||
|
|
||||||
|
def test_negative_32b_binbytes(self):
|
||||||
|
# On 32-bit builds, a BINBYTES of 2**31 or more is refused
|
||||||
|
self.check_negative_32b_binXXX(b'\x80\x03B\xff\xff\xff\xffxyzq\x00.')
|
||||||
|
|
||||||
|
def test_negative_32b_binunicode(self):
|
||||||
|
# On 32-bit builds, a BINUNICODE of 2**31 or more is refused
|
||||||
|
self.check_negative_32b_binXXX(b'\x80\x03X\xff\xff\xff\xffxyzq\x00.')
|
||||||
|
|
||||||
|
|
||||||
|
class BigmemPickleTests(unittest.TestCase):
|
||||||
|
|
||||||
|
# Binary protocols can serialize longs of up to 2GB-1
|
||||||
|
|
||||||
|
@precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
|
||||||
|
def test_huge_long_32b(self, size):
|
||||||
|
data = 1 << (8 * size)
|
||||||
|
try:
|
||||||
|
for proto in protocols:
|
||||||
|
if proto < 2:
|
||||||
|
continue
|
||||||
|
with self.assertRaises((ValueError, OverflowError)):
|
||||||
|
self.dumps(data, protocol=proto)
|
||||||
|
finally:
|
||||||
|
data = None
|
||||||
|
|
||||||
|
# Protocol 3 can serialize up to 4GB-1 as a bytes object
|
||||||
|
# (older protocols don't have a dedicated opcode for bytes and are
|
||||||
|
# too inefficient)
|
||||||
|
|
||||||
|
@precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
|
||||||
|
def test_huge_bytes_32b(self, size):
|
||||||
|
data = b"abcd" * (size // 4)
|
||||||
|
try:
|
||||||
|
for proto in protocols:
|
||||||
|
if proto < 3:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pickled = self.dumps(data, protocol=proto)
|
||||||
|
self.assertTrue(b"abcd" in pickled[:15])
|
||||||
|
self.assertTrue(b"abcd" in pickled[-15:])
|
||||||
|
finally:
|
||||||
|
pickled = None
|
||||||
|
finally:
|
||||||
|
data = None
|
||||||
|
|
||||||
|
@precisionbigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
|
||||||
|
def test_huge_bytes_64b(self, size):
|
||||||
|
data = b"a" * size
|
||||||
|
try:
|
||||||
|
for proto in protocols:
|
||||||
|
if proto < 3:
|
||||||
|
continue
|
||||||
|
with self.assertRaises((ValueError, OverflowError)):
|
||||||
|
self.dumps(data, protocol=proto)
|
||||||
|
finally:
|
||||||
|
data = None
|
||||||
|
|
||||||
|
# All protocols use 1-byte per printable ASCII character; we add another
|
||||||
|
# byte because the encoded form has to be copied into the internal buffer.
|
||||||
|
|
||||||
|
@precisionbigmemtest(size=_2G, memuse=2 + character_size, dry_run=False)
|
||||||
|
def test_huge_str_32b(self, size):
|
||||||
|
data = "abcd" * (size // 4)
|
||||||
|
try:
|
||||||
|
for proto in protocols:
|
||||||
|
try:
|
||||||
|
pickled = self.dumps(data, protocol=proto)
|
||||||
|
self.assertTrue(b"abcd" in pickled[:15])
|
||||||
|
self.assertTrue(b"abcd" in pickled[-15:])
|
||||||
|
finally:
|
||||||
|
pickled = None
|
||||||
|
finally:
|
||||||
|
data = None
|
||||||
|
|
||||||
|
@precisionbigmemtest(size=_4G, memuse=1 + character_size, dry_run=False)
|
||||||
|
def test_huge_str_64b(self, size):
|
||||||
|
data = "a" * size
|
||||||
|
try:
|
||||||
|
for proto in protocols:
|
||||||
|
with self.assertRaises((ValueError, OverflowError)):
|
||||||
|
self.dumps(data, protocol=proto)
|
||||||
|
finally:
|
||||||
|
data = None
|
||||||
|
|
||||||
|
|
||||||
# Test classes for reduce_ex
|
# Test classes for reduce_ex
|
||||||
|
|
||||||
|
|
|
@ -1142,7 +1142,7 @@ def bigmemtest(minsize, memuse):
|
||||||
return wrapper
|
return wrapper
|
||||||
return decorator
|
return decorator
|
||||||
|
|
||||||
def precisionbigmemtest(size, memuse):
|
def precisionbigmemtest(size, memuse, dry_run=True):
|
||||||
"""Decorator for bigmem tests that need exact sizes.
|
"""Decorator for bigmem tests that need exact sizes.
|
||||||
|
|
||||||
Like bigmemtest, but without the size scaling upward to fill available
|
Like bigmemtest, but without the size scaling upward to fill available
|
||||||
|
@ -1157,10 +1157,11 @@ def precisionbigmemtest(size, memuse):
|
||||||
else:
|
else:
|
||||||
maxsize = size
|
maxsize = size
|
||||||
|
|
||||||
if real_max_memuse and real_max_memuse < maxsize * memuse:
|
if ((real_max_memuse or not dry_run)
|
||||||
raise unittest.SkipTest(
|
and real_max_memuse < maxsize * memuse):
|
||||||
"not enough memory: %.1fG minimum needed"
|
raise unittest.SkipTest(
|
||||||
% (size * memuse / (1024 ** 3)))
|
"not enough memory: %.1fG minimum needed"
|
||||||
|
% (size * memuse / (1024 ** 3)))
|
||||||
|
|
||||||
return f(self, maxsize)
|
return f(self, maxsize)
|
||||||
wrapper.size = size
|
wrapper.size = size
|
||||||
|
|
|
@ -7,6 +7,7 @@ from test.pickletester import AbstractPickleTests
|
||||||
from test.pickletester import AbstractPickleModuleTests
|
from test.pickletester import AbstractPickleModuleTests
|
||||||
from test.pickletester import AbstractPersistentPicklerTests
|
from test.pickletester import AbstractPersistentPicklerTests
|
||||||
from test.pickletester import AbstractPicklerUnpicklerObjectTests
|
from test.pickletester import AbstractPicklerUnpicklerObjectTests
|
||||||
|
from test.pickletester import BigmemPickleTests
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import _pickle
|
import _pickle
|
||||||
|
@ -37,13 +38,13 @@ class PyPicklerTests(AbstractPickleTests):
|
||||||
return u.load()
|
return u.load()
|
||||||
|
|
||||||
|
|
||||||
class InMemoryPickleTests(AbstractPickleTests):
|
class InMemoryPickleTests(AbstractPickleTests, BigmemPickleTests):
|
||||||
|
|
||||||
pickler = pickle._Pickler
|
pickler = pickle._Pickler
|
||||||
unpickler = pickle._Unpickler
|
unpickler = pickle._Unpickler
|
||||||
|
|
||||||
def dumps(self, arg, proto=None):
|
def dumps(self, arg, protocol=None):
|
||||||
return pickle.dumps(arg, proto)
|
return pickle.dumps(arg, protocol)
|
||||||
|
|
||||||
def loads(self, buf, **kwds):
|
def loads(self, buf, **kwds):
|
||||||
return pickle.loads(buf, **kwds)
|
return pickle.loads(buf, **kwds)
|
||||||
|
|
|
@ -268,6 +268,9 @@ Core and Builtins
|
||||||
Library
|
Library
|
||||||
-------
|
-------
|
||||||
|
|
||||||
|
- Issue #11564: Avoid crashes when trying to pickle huge objects or containers
|
||||||
|
(more than 2**31 items). Instead, in most cases, an OverflowError is raised.
|
||||||
|
|
||||||
- Issue #12287: Fix a stack corruption in ossaudiodev module when the FD is
|
- Issue #12287: Fix a stack corruption in ossaudiodev module when the FD is
|
||||||
greater than FD_SETSIZE.
|
greater than FD_SETSIZE.
|
||||||
|
|
||||||
|
|
|
@ -153,7 +153,7 @@ typedef struct {
|
||||||
static void
|
static void
|
||||||
Pdata_dealloc(Pdata *self)
|
Pdata_dealloc(Pdata *self)
|
||||||
{
|
{
|
||||||
int i = Py_SIZE(self);
|
Py_ssize_t i = Py_SIZE(self);
|
||||||
while (--i >= 0) {
|
while (--i >= 0) {
|
||||||
Py_DECREF(self->data[i]);
|
Py_DECREF(self->data[i]);
|
||||||
}
|
}
|
||||||
|
@ -190,9 +190,9 @@ Pdata_New(void)
|
||||||
* number of items, this is a (non-erroneous) NOP.
|
* number of items, this is a (non-erroneous) NOP.
|
||||||
*/
|
*/
|
||||||
static int
|
static int
|
||||||
Pdata_clear(Pdata *self, int clearto)
|
Pdata_clear(Pdata *self, Py_ssize_t clearto)
|
||||||
{
|
{
|
||||||
int i = Py_SIZE(self);
|
Py_ssize_t i = Py_SIZE(self);
|
||||||
|
|
||||||
if (clearto < 0)
|
if (clearto < 0)
|
||||||
return stack_underflow();
|
return stack_underflow();
|
||||||
|
@ -303,7 +303,7 @@ Pdata_poplist(Pdata *self, Py_ssize_t start)
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
PyObject *me_key;
|
PyObject *me_key;
|
||||||
long me_value;
|
Py_ssize_t me_value;
|
||||||
} PyMemoEntry;
|
} PyMemoEntry;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
|
@ -328,7 +328,7 @@ typedef struct PicklerObject {
|
||||||
Py_ssize_t max_output_len; /* Allocation size of output_buffer. */
|
Py_ssize_t max_output_len; /* Allocation size of output_buffer. */
|
||||||
int proto; /* Pickle protocol number, >= 0 */
|
int proto; /* Pickle protocol number, >= 0 */
|
||||||
int bin; /* Boolean, true if proto > 0 */
|
int bin; /* Boolean, true if proto > 0 */
|
||||||
int buf_size; /* Size of the current buffered pickle data */
|
Py_ssize_t buf_size; /* Size of the current buffered pickle data */
|
||||||
int fast; /* Enable fast mode if set to a true value.
|
int fast; /* Enable fast mode if set to a true value.
|
||||||
The fast mode disable the usage of memo,
|
The fast mode disable the usage of memo,
|
||||||
therefore speeding the pickling process by
|
therefore speeding the pickling process by
|
||||||
|
@ -369,7 +369,7 @@ typedef struct UnpicklerObject {
|
||||||
char *errors; /* Name of errors handling scheme to used when
|
char *errors; /* Name of errors handling scheme to used when
|
||||||
decoding strings. The default value is
|
decoding strings. The default value is
|
||||||
"strict". */
|
"strict". */
|
||||||
int *marks; /* Mark stack, used for unpickling container
|
Py_ssize_t *marks; /* Mark stack, used for unpickling container
|
||||||
objects. */
|
objects. */
|
||||||
Py_ssize_t num_marks; /* Number of marks in the mark stack. */
|
Py_ssize_t num_marks; /* Number of marks in the mark stack. */
|
||||||
Py_ssize_t marks_size; /* Current allocated size of the mark stack. */
|
Py_ssize_t marks_size; /* Current allocated size of the mark stack. */
|
||||||
|
@ -556,7 +556,7 @@ _PyMemoTable_ResizeTable(PyMemoTable *self, Py_ssize_t min_size)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Returns NULL on failure, a pointer to the value otherwise. */
|
/* Returns NULL on failure, a pointer to the value otherwise. */
|
||||||
static long *
|
static Py_ssize_t *
|
||||||
PyMemoTable_Get(PyMemoTable *self, PyObject *key)
|
PyMemoTable_Get(PyMemoTable *self, PyObject *key)
|
||||||
{
|
{
|
||||||
PyMemoEntry *entry = _PyMemoTable_Lookup(self, key);
|
PyMemoEntry *entry = _PyMemoTable_Lookup(self, key);
|
||||||
|
@ -567,7 +567,7 @@ PyMemoTable_Get(PyMemoTable *self, PyObject *key)
|
||||||
|
|
||||||
/* Returns -1 on failure, 0 on success. */
|
/* Returns -1 on failure, 0 on success. */
|
||||||
static int
|
static int
|
||||||
PyMemoTable_Set(PyMemoTable *self, PyObject *key, long value)
|
PyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value)
|
||||||
{
|
{
|
||||||
PyMemoEntry *entry;
|
PyMemoEntry *entry;
|
||||||
|
|
||||||
|
@ -700,7 +700,7 @@ _Pickler_FlushToFile(PicklerObject *self)
|
||||||
return (result == NULL) ? -1 : 0;
|
return (result == NULL) ? -1 : 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static Py_ssize_t
|
||||||
_Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t n)
|
_Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t n)
|
||||||
{
|
{
|
||||||
Py_ssize_t i, required;
|
Py_ssize_t i, required;
|
||||||
|
@ -735,7 +735,7 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t n)
|
||||||
PyErr_NoMemory();
|
PyErr_NoMemory();
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
self->max_output_len = (self->output_len + n) * 2;
|
self->max_output_len = (self->output_len + n) / 2 * 3;
|
||||||
if (_PyBytes_Resize(&self->output_buffer, self->max_output_len) < 0)
|
if (_PyBytes_Resize(&self->output_buffer, self->max_output_len) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
@ -1219,9 +1219,9 @@ _Unpickler_SetInputEncoding(UnpicklerObject *self,
|
||||||
static int
|
static int
|
||||||
memo_get(PicklerObject *self, PyObject *key)
|
memo_get(PicklerObject *self, PyObject *key)
|
||||||
{
|
{
|
||||||
long *value;
|
Py_ssize_t *value;
|
||||||
char pdata[30];
|
char pdata[30];
|
||||||
int len;
|
Py_ssize_t len;
|
||||||
|
|
||||||
value = PyMemoTable_Get(self->memo, key);
|
value = PyMemoTable_Get(self->memo, key);
|
||||||
if (value == NULL) {
|
if (value == NULL) {
|
||||||
|
@ -1231,8 +1231,9 @@ memo_get(PicklerObject *self, PyObject *key)
|
||||||
|
|
||||||
if (!self->bin) {
|
if (!self->bin) {
|
||||||
pdata[0] = GET;
|
pdata[0] = GET;
|
||||||
PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", *value);
|
PyOS_snprintf(pdata + 1, sizeof(pdata) - 1,
|
||||||
len = (int)strlen(pdata);
|
"%" PY_FORMAT_SIZE_T "d\n", *value);
|
||||||
|
len = strlen(pdata);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
if (*value < 256) {
|
if (*value < 256) {
|
||||||
|
@ -1266,9 +1267,9 @@ memo_get(PicklerObject *self, PyObject *key)
|
||||||
static int
|
static int
|
||||||
memo_put(PicklerObject *self, PyObject *obj)
|
memo_put(PicklerObject *self, PyObject *obj)
|
||||||
{
|
{
|
||||||
long x;
|
Py_ssize_t x;
|
||||||
char pdata[30];
|
char pdata[30];
|
||||||
int len;
|
Py_ssize_t len;
|
||||||
int status = 0;
|
int status = 0;
|
||||||
|
|
||||||
if (self->fast)
|
if (self->fast)
|
||||||
|
@ -1280,7 +1281,8 @@ memo_put(PicklerObject *self, PyObject *obj)
|
||||||
|
|
||||||
if (!self->bin) {
|
if (!self->bin) {
|
||||||
pdata[0] = PUT;
|
pdata[0] = PUT;
|
||||||
PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);
|
PyOS_snprintf(pdata + 1, sizeof(pdata) - 1,
|
||||||
|
"%" PY_FORMAT_SIZE_T "d\n", x);
|
||||||
len = strlen(pdata);
|
len = strlen(pdata);
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
|
@ -1482,7 +1484,7 @@ static int
|
||||||
save_int(PicklerObject *self, long x)
|
save_int(PicklerObject *self, long x)
|
||||||
{
|
{
|
||||||
char pdata[32];
|
char pdata[32];
|
||||||
int len = 0;
|
Py_ssize_t len = 0;
|
||||||
|
|
||||||
if (!self->bin
|
if (!self->bin
|
||||||
#if SIZEOF_LONG > 4
|
#if SIZEOF_LONG > 4
|
||||||
|
@ -1612,7 +1614,7 @@ save_long(PicklerObject *self, PyObject *obj)
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
header[0] = LONG4;
|
header[0] = LONG4;
|
||||||
size = (int)nbytes;
|
size = (Py_ssize_t) nbytes;
|
||||||
for (i = 1; i < 5; i++) {
|
for (i = 1; i < 5; i++) {
|
||||||
header[i] = (unsigned char)(size & 0xff);
|
header[i] = (unsigned char)(size & 0xff);
|
||||||
size >>= 8;
|
size >>= 8;
|
||||||
|
@ -1726,7 +1728,7 @@ save_bytes(PicklerObject *self, PyObject *obj)
|
||||||
else {
|
else {
|
||||||
Py_ssize_t size;
|
Py_ssize_t size;
|
||||||
char header[5];
|
char header[5];
|
||||||
int len;
|
Py_ssize_t len;
|
||||||
|
|
||||||
size = PyBytes_Size(obj);
|
size = PyBytes_Size(obj);
|
||||||
if (size < 0)
|
if (size < 0)
|
||||||
|
@ -1746,6 +1748,8 @@ save_bytes(PicklerObject *self, PyObject *obj)
|
||||||
len = 5;
|
len = 5;
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
|
PyErr_SetString(PyExc_OverflowError,
|
||||||
|
"cannot serialize a bytes object larger than 4GB");
|
||||||
return -1; /* string too large */
|
return -1; /* string too large */
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1870,8 +1874,11 @@ save_unicode(PicklerObject *self, PyObject *obj)
|
||||||
goto error;
|
goto error;
|
||||||
|
|
||||||
size = PyBytes_GET_SIZE(encoded);
|
size = PyBytes_GET_SIZE(encoded);
|
||||||
if (size < 0 || size > 0xffffffffL)
|
if (size > 0xffffffffL) {
|
||||||
|
PyErr_SetString(PyExc_OverflowError,
|
||||||
|
"cannot serialize a string larger than 4GB");
|
||||||
goto error; /* string too large */
|
goto error; /* string too large */
|
||||||
|
}
|
||||||
|
|
||||||
pdata[0] = BINUNICODE;
|
pdata[0] = BINUNICODE;
|
||||||
pdata[1] = (unsigned char)(size & 0xff);
|
pdata[1] = (unsigned char)(size & 0xff);
|
||||||
|
@ -1916,9 +1923,9 @@ save_unicode(PicklerObject *self, PyObject *obj)
|
||||||
|
|
||||||
/* A helper for save_tuple. Push the len elements in tuple t on the stack. */
|
/* A helper for save_tuple. Push the len elements in tuple t on the stack. */
|
||||||
static int
|
static int
|
||||||
store_tuple_elements(PicklerObject *self, PyObject *t, int len)
|
store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len)
|
||||||
{
|
{
|
||||||
int i;
|
Py_ssize_t i;
|
||||||
|
|
||||||
assert(PyTuple_Size(t) == len);
|
assert(PyTuple_Size(t) == len);
|
||||||
|
|
||||||
|
@ -1943,7 +1950,7 @@ store_tuple_elements(PicklerObject *self, PyObject *t, int len)
|
||||||
static int
|
static int
|
||||||
save_tuple(PicklerObject *self, PyObject *obj)
|
save_tuple(PicklerObject *self, PyObject *obj)
|
||||||
{
|
{
|
||||||
int len, i;
|
Py_ssize_t len, i;
|
||||||
|
|
||||||
const char mark_op = MARK;
|
const char mark_op = MARK;
|
||||||
const char tuple_op = TUPLE;
|
const char tuple_op = TUPLE;
|
||||||
|
@ -2166,7 +2173,7 @@ static int
|
||||||
batch_list_exact(PicklerObject *self, PyObject *obj)
|
batch_list_exact(PicklerObject *self, PyObject *obj)
|
||||||
{
|
{
|
||||||
PyObject *item = NULL;
|
PyObject *item = NULL;
|
||||||
int this_batch, total;
|
Py_ssize_t this_batch, total;
|
||||||
|
|
||||||
const char append_op = APPEND;
|
const char append_op = APPEND;
|
||||||
const char appends_op = APPENDS;
|
const char appends_op = APPENDS;
|
||||||
|
@ -2211,7 +2218,7 @@ static int
|
||||||
save_list(PicklerObject *self, PyObject *obj)
|
save_list(PicklerObject *self, PyObject *obj)
|
||||||
{
|
{
|
||||||
char header[3];
|
char header[3];
|
||||||
int len;
|
Py_ssize_t len;
|
||||||
int status = 0;
|
int status = 0;
|
||||||
|
|
||||||
if (self->fast && !fast_save_enter(self, obj))
|
if (self->fast && !fast_save_enter(self, obj))
|
||||||
|
@ -2471,7 +2478,7 @@ save_dict(PicklerObject *self, PyObject *obj)
|
||||||
{
|
{
|
||||||
PyObject *items, *iter;
|
PyObject *items, *iter;
|
||||||
char header[3];
|
char header[3];
|
||||||
int len;
|
Py_ssize_t len;
|
||||||
int status = 0;
|
int status = 0;
|
||||||
|
|
||||||
if (self->fast && !fast_save_enter(self, obj))
|
if (self->fast && !fast_save_enter(self, obj))
|
||||||
|
@ -2606,7 +2613,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name)
|
||||||
PyObject *code_obj; /* extension code as Python object */
|
PyObject *code_obj; /* extension code as Python object */
|
||||||
long code; /* extension code as C value */
|
long code; /* extension code as C value */
|
||||||
char pdata[5];
|
char pdata[5];
|
||||||
int n;
|
Py_ssize_t n;
|
||||||
|
|
||||||
PyTuple_SET_ITEM(two_tuple, 0, module_name);
|
PyTuple_SET_ITEM(two_tuple, 0, module_name);
|
||||||
PyTuple_SET_ITEM(two_tuple, 1, global_name);
|
PyTuple_SET_ITEM(two_tuple, 1, global_name);
|
||||||
|
@ -2629,9 +2636,10 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name)
|
||||||
}
|
}
|
||||||
code = PyLong_AS_LONG(code_obj);
|
code = PyLong_AS_LONG(code_obj);
|
||||||
if (code <= 0 || code > 0x7fffffffL) {
|
if (code <= 0 || code > 0x7fffffffL) {
|
||||||
PyErr_Format(PicklingError,
|
if (!PyErr_Occurred())
|
||||||
"Can't pickle %R: extension code %ld is out of range",
|
PyErr_Format(PicklingError,
|
||||||
obj, code);
|
"Can't pickle %R: extension code %ld is out of range",
|
||||||
|
obj, code);
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3497,7 +3505,7 @@ pmp_copy(PicklerMemoProxyObject *self)
|
||||||
PyObject *key, *value;
|
PyObject *key, *value;
|
||||||
|
|
||||||
key = PyLong_FromVoidPtr(entry.me_key);
|
key = PyLong_FromVoidPtr(entry.me_key);
|
||||||
value = Py_BuildValue("lO", entry.me_value, entry.me_key);
|
value = Py_BuildValue("nO", entry.me_value, entry.me_key);
|
||||||
|
|
||||||
if (key == NULL || value == NULL) {
|
if (key == NULL || value == NULL) {
|
||||||
Py_XDECREF(key);
|
Py_XDECREF(key);
|
||||||
|
@ -3658,7 +3666,7 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
while (PyDict_Next(obj, &i, &key, &value)) {
|
while (PyDict_Next(obj, &i, &key, &value)) {
|
||||||
long memo_id;
|
Py_ssize_t memo_id;
|
||||||
PyObject *memo_obj;
|
PyObject *memo_obj;
|
||||||
|
|
||||||
if (!PyTuple_Check(value) || Py_SIZE(value) != 2) {
|
if (!PyTuple_Check(value) || Py_SIZE(value) != 2) {
|
||||||
|
@ -3666,7 +3674,7 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj)
|
||||||
"'memo' values must be 2-item tuples");
|
"'memo' values must be 2-item tuples");
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
memo_id = PyLong_AsLong(PyTuple_GET_ITEM(value, 0));
|
memo_id = PyLong_AsSsize_t(PyTuple_GET_ITEM(value, 0));
|
||||||
if (memo_id == -1 && PyErr_Occurred())
|
if (memo_id == -1 && PyErr_Occurred())
|
||||||
goto error;
|
goto error;
|
||||||
memo_obj = PyTuple_GET_ITEM(value, 1);
|
memo_obj = PyTuple_GET_ITEM(value, 1);
|
||||||
|
@ -3797,7 +3805,7 @@ find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name)
|
||||||
module_name, global_name);
|
module_name, global_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static Py_ssize_t
|
||||||
marker(UnpicklerObject *self)
|
marker(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
if (self->num_marks < 1) {
|
if (self->num_marks < 1) {
|
||||||
|
@ -3875,6 +3883,28 @@ load_bool(UnpicklerObject *self, PyObject *boolean)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* s contains x bytes of an unsigned little-endian integer. Return its value
|
||||||
|
* as a C Py_ssize_t, or -1 if it's higher than PY_SSIZE_T_MAX.
|
||||||
|
*/
|
||||||
|
static Py_ssize_t
|
||||||
|
calc_binsize(char *bytes, int size)
|
||||||
|
{
|
||||||
|
unsigned char *s = (unsigned char *)bytes;
|
||||||
|
size_t x = 0;
|
||||||
|
|
||||||
|
assert(size == 4);
|
||||||
|
|
||||||
|
x = (size_t) s[0];
|
||||||
|
x |= (size_t) s[1] << 8;
|
||||||
|
x |= (size_t) s[2] << 16;
|
||||||
|
x |= (size_t) s[3] << 24;
|
||||||
|
|
||||||
|
if (x > PY_SSIZE_T_MAX)
|
||||||
|
return -1;
|
||||||
|
else
|
||||||
|
return (Py_ssize_t) x;
|
||||||
|
}
|
||||||
|
|
||||||
/* s contains x bytes of a little-endian integer. Return its value as a
|
/* s contains x bytes of a little-endian integer. Return its value as a
|
||||||
* C int. Obscure: when x is 1 or 2, this is an unsigned little-endian
|
* C int. Obscure: when x is 1 or 2, this is an unsigned little-endian
|
||||||
* int, but when x is 4 it's a signed one. This is an historical source
|
* int, but when x is 4 it's a signed one. This is an historical source
|
||||||
|
@ -4119,16 +4149,18 @@ static int
|
||||||
load_binbytes(UnpicklerObject *self)
|
load_binbytes(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *bytes;
|
PyObject *bytes;
|
||||||
long x;
|
Py_ssize_t x;
|
||||||
char *s;
|
char *s;
|
||||||
|
|
||||||
if (_Unpickler_Read(self, &s, 4) < 0)
|
if (_Unpickler_Read(self, &s, 4) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
x = calc_binint(s, 4);
|
x = calc_binsize(s, 4);
|
||||||
if (x < 0) {
|
if (x < 0) {
|
||||||
PyErr_SetString(UnpicklingError,
|
PyErr_Format(PyExc_OverflowError,
|
||||||
"BINBYTES pickle has negative byte count");
|
"BINBYTES exceeds system's maximum size of %zd bytes",
|
||||||
|
PY_SSIZE_T_MAX
|
||||||
|
);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -4146,7 +4178,7 @@ static int
|
||||||
load_short_binbytes(UnpicklerObject *self)
|
load_short_binbytes(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *bytes;
|
PyObject *bytes;
|
||||||
unsigned char x;
|
Py_ssize_t x;
|
||||||
char *s;
|
char *s;
|
||||||
|
|
||||||
if (_Unpickler_Read(self, &s, 1) < 0)
|
if (_Unpickler_Read(self, &s, 1) < 0)
|
||||||
|
@ -4169,7 +4201,7 @@ static int
|
||||||
load_binstring(UnpicklerObject *self)
|
load_binstring(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *str;
|
PyObject *str;
|
||||||
long x;
|
Py_ssize_t x;
|
||||||
char *s;
|
char *s;
|
||||||
|
|
||||||
if (_Unpickler_Read(self, &s, 4) < 0)
|
if (_Unpickler_Read(self, &s, 4) < 0)
|
||||||
|
@ -4198,7 +4230,7 @@ static int
|
||||||
load_short_binstring(UnpicklerObject *self)
|
load_short_binstring(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *str;
|
PyObject *str;
|
||||||
unsigned char x;
|
Py_ssize_t x;
|
||||||
char *s;
|
char *s;
|
||||||
|
|
||||||
if (_Unpickler_Read(self, &s, 1) < 0)
|
if (_Unpickler_Read(self, &s, 1) < 0)
|
||||||
|
@ -4242,19 +4274,22 @@ static int
|
||||||
load_binunicode(UnpicklerObject *self)
|
load_binunicode(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *str;
|
PyObject *str;
|
||||||
long size;
|
Py_ssize_t size;
|
||||||
char *s;
|
char *s;
|
||||||
|
|
||||||
if (_Unpickler_Read(self, &s, 4) < 0)
|
if (_Unpickler_Read(self, &s, 4) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
size = calc_binint(s, 4);
|
size = calc_binsize(s, 4);
|
||||||
if (size < 0) {
|
if (size < 0) {
|
||||||
PyErr_SetString(UnpicklingError,
|
PyErr_Format(PyExc_OverflowError,
|
||||||
"BINUNICODE pickle has negative byte count");
|
"BINUNICODE exceeds system's maximum size of %zd bytes",
|
||||||
|
PY_SSIZE_T_MAX
|
||||||
|
);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (_Unpickler_Read(self, &s, size) < 0)
|
if (_Unpickler_Read(self, &s, size) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
|
@ -4270,7 +4305,7 @@ static int
|
||||||
load_tuple(UnpicklerObject *self)
|
load_tuple(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *tuple;
|
PyObject *tuple;
|
||||||
int i;
|
Py_ssize_t i;
|
||||||
|
|
||||||
if ((i = marker(self)) < 0)
|
if ((i = marker(self)) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
@ -4329,7 +4364,7 @@ static int
|
||||||
load_list(UnpicklerObject *self)
|
load_list(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *list;
|
PyObject *list;
|
||||||
int i;
|
Py_ssize_t i;
|
||||||
|
|
||||||
if ((i = marker(self)) < 0)
|
if ((i = marker(self)) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
@ -4345,7 +4380,7 @@ static int
|
||||||
load_dict(UnpicklerObject *self)
|
load_dict(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *dict, *key, *value;
|
PyObject *dict, *key, *value;
|
||||||
int i, j, k;
|
Py_ssize_t i, j, k;
|
||||||
|
|
||||||
if ((i = marker(self)) < 0)
|
if ((i = marker(self)) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
@ -4389,7 +4424,7 @@ static int
|
||||||
load_obj(UnpicklerObject *self)
|
load_obj(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *cls, *args, *obj = NULL;
|
PyObject *cls, *args, *obj = NULL;
|
||||||
int i;
|
Py_ssize_t i;
|
||||||
|
|
||||||
if ((i = marker(self)) < 0)
|
if ((i = marker(self)) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
@ -4420,7 +4455,7 @@ load_inst(UnpicklerObject *self)
|
||||||
PyObject *module_name;
|
PyObject *module_name;
|
||||||
PyObject *class_name;
|
PyObject *class_name;
|
||||||
Py_ssize_t len;
|
Py_ssize_t len;
|
||||||
int i;
|
Py_ssize_t i;
|
||||||
char *s;
|
char *s;
|
||||||
|
|
||||||
if ((i = marker(self)) < 0)
|
if ((i = marker(self)) < 0)
|
||||||
|
@ -4614,7 +4649,7 @@ load_binpersid(UnpicklerObject *self)
|
||||||
static int
|
static int
|
||||||
load_pop(UnpicklerObject *self)
|
load_pop(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
int len = Py_SIZE(self->stack);
|
Py_ssize_t len = Py_SIZE(self->stack);
|
||||||
|
|
||||||
/* Note that we split the (pickle.py) stack into two stacks,
|
/* Note that we split the (pickle.py) stack into two stacks,
|
||||||
* an object stack and a mark stack. We have to be clever and
|
* an object stack and a mark stack. We have to be clever and
|
||||||
|
@ -4638,7 +4673,7 @@ load_pop(UnpicklerObject *self)
|
||||||
static int
|
static int
|
||||||
load_pop_mark(UnpicklerObject *self)
|
load_pop_mark(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
int i;
|
Py_ssize_t i;
|
||||||
|
|
||||||
if ((i = marker(self)) < 0)
|
if ((i = marker(self)) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
@ -4652,7 +4687,7 @@ static int
|
||||||
load_dup(UnpicklerObject *self)
|
load_dup(UnpicklerObject *self)
|
||||||
{
|
{
|
||||||
PyObject *last;
|
PyObject *last;
|
||||||
int len;
|
Py_ssize_t len;
|
||||||
|
|
||||||
if ((len = Py_SIZE(self->stack)) <= 0)
|
if ((len = Py_SIZE(self->stack)) <= 0)
|
||||||
return stack_underflow();
|
return stack_underflow();
|
||||||
|
@ -4731,10 +4766,7 @@ load_long_binget(UnpicklerObject *self)
|
||||||
if (_Unpickler_Read(self, &s, 4) < 0)
|
if (_Unpickler_Read(self, &s, 4) < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
idx = (long)Py_CHARMASK(s[0]);
|
idx = calc_binsize(s, 4);
|
||||||
idx |= (long)Py_CHARMASK(s[1]) << 8;
|
|
||||||
idx |= (long)Py_CHARMASK(s[2]) << 16;
|
|
||||||
idx |= (long)Py_CHARMASK(s[3]) << 24;
|
|
||||||
|
|
||||||
value = _Unpickler_MemoGet(self, idx);
|
value = _Unpickler_MemoGet(self, idx);
|
||||||
if (value == NULL) {
|
if (value == NULL) {
|
||||||
|
@ -4880,20 +4912,17 @@ load_long_binput(UnpicklerObject *self)
|
||||||
return stack_underflow();
|
return stack_underflow();
|
||||||
value = self->stack->data[Py_SIZE(self->stack) - 1];
|
value = self->stack->data[Py_SIZE(self->stack) - 1];
|
||||||
|
|
||||||
idx = (long)Py_CHARMASK(s[0]);
|
idx = calc_binsize(s, 4);
|
||||||
idx |= (long)Py_CHARMASK(s[1]) << 8;
|
|
||||||
idx |= (long)Py_CHARMASK(s[2]) << 16;
|
|
||||||
idx |= (long)Py_CHARMASK(s[3]) << 24;
|
|
||||||
|
|
||||||
return _Unpickler_MemoPut(self, idx, value);
|
return _Unpickler_MemoPut(self, idx, value);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
do_append(UnpicklerObject *self, int x)
|
do_append(UnpicklerObject *self, Py_ssize_t x)
|
||||||
{
|
{
|
||||||
PyObject *value;
|
PyObject *value;
|
||||||
PyObject *list;
|
PyObject *list;
|
||||||
int len, i;
|
Py_ssize_t len, i;
|
||||||
|
|
||||||
len = Py_SIZE(self->stack);
|
len = Py_SIZE(self->stack);
|
||||||
if (x > len || x <= 0)
|
if (x > len || x <= 0)
|
||||||
|
@ -4906,14 +4935,15 @@ do_append(UnpicklerObject *self, int x)
|
||||||
if (PyList_Check(list)) {
|
if (PyList_Check(list)) {
|
||||||
PyObject *slice;
|
PyObject *slice;
|
||||||
Py_ssize_t list_len;
|
Py_ssize_t list_len;
|
||||||
|
int ret;
|
||||||
|
|
||||||
slice = Pdata_poplist(self->stack, x);
|
slice = Pdata_poplist(self->stack, x);
|
||||||
if (!slice)
|
if (!slice)
|
||||||
return -1;
|
return -1;
|
||||||
list_len = PyList_GET_SIZE(list);
|
list_len = PyList_GET_SIZE(list);
|
||||||
i = PyList_SetSlice(list, list_len, list_len, slice);
|
ret = PyList_SetSlice(list, list_len, list_len, slice);
|
||||||
Py_DECREF(slice);
|
Py_DECREF(slice);
|
||||||
return i;
|
return ret;
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
PyObject *append_func;
|
PyObject *append_func;
|
||||||
|
@ -4952,11 +4982,11 @@ load_appends(UnpicklerObject *self)
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
do_setitems(UnpicklerObject *self, int x)
|
do_setitems(UnpicklerObject *self, Py_ssize_t x)
|
||||||
{
|
{
|
||||||
PyObject *value, *key;
|
PyObject *value, *key;
|
||||||
PyObject *dict;
|
PyObject *dict;
|
||||||
int len, i;
|
Py_ssize_t len, i;
|
||||||
int status = 0;
|
int status = 0;
|
||||||
|
|
||||||
len = Py_SIZE(self->stack);
|
len = Py_SIZE(self->stack);
|
||||||
|
@ -5124,20 +5154,21 @@ load_mark(UnpicklerObject *self)
|
||||||
|
|
||||||
if ((self->num_marks + 1) >= self->marks_size) {
|
if ((self->num_marks + 1) >= self->marks_size) {
|
||||||
size_t alloc;
|
size_t alloc;
|
||||||
int *marks;
|
Py_ssize_t *marks;
|
||||||
|
|
||||||
/* Use the size_t type to check for overflow. */
|
/* Use the size_t type to check for overflow. */
|
||||||
alloc = ((size_t)self->num_marks << 1) + 20;
|
alloc = ((size_t)self->num_marks << 1) + 20;
|
||||||
if (alloc > PY_SSIZE_T_MAX ||
|
if (alloc > (PY_SSIZE_T_MAX / sizeof(Py_ssize_t)) ||
|
||||||
alloc <= ((size_t)self->num_marks + 1)) {
|
alloc <= ((size_t)self->num_marks + 1)) {
|
||||||
PyErr_NoMemory();
|
PyErr_NoMemory();
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (self->marks == NULL)
|
if (self->marks == NULL)
|
||||||
marks = (int *)PyMem_Malloc(alloc * sizeof(int));
|
marks = (Py_ssize_t *) PyMem_Malloc(alloc * sizeof(Py_ssize_t));
|
||||||
else
|
else
|
||||||
marks = (int *)PyMem_Realloc(self->marks, alloc * sizeof(int));
|
marks = (Py_ssize_t *) PyMem_Realloc(self->marks,
|
||||||
|
alloc * sizeof(Py_ssize_t));
|
||||||
if (marks == NULL) {
|
if (marks == NULL) {
|
||||||
PyErr_NoMemory();
|
PyErr_NoMemory();
|
||||||
return -1;
|
return -1;
|
||||||
|
|
Loading…
Reference in New Issue