From 4e1f3d669b83c21da9314b70a58a18e92807bd54 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 5 Oct 2013 03:07:03 +0300 Subject: [PATCH 01/12] #19166: use an unused var in a test. Patch by Vajrasky Kok. --- Lib/test/test_dict.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index d1f31923708..a38895945ed 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -45,6 +45,9 @@ class DictTest(unittest.TestCase): self.assertEqual(set(d.keys()), set()) d = {'a': 1, 'b': 2} k = d.keys() + self.assertEqual(set(k), {'a', 'b'}) + self.assertIn('a', k) + self.assertIn('b', k) self.assertIn('a', d) self.assertIn('b', d) self.assertRaises(TypeError, d.keys, None) From 93d7dda914c95c6d011b09ab86e0364e8eac55fb Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 5 Oct 2013 04:13:18 +0300 Subject: [PATCH 02/12] #19148: fix markup errors and wording in enum docs. Patch by Esa Peuha. --- Doc/library/enum.rst | 14 +++++++------- Misc/ACKS | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 0c46e1c6ec5..9d9e46e564f 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -250,7 +250,7 @@ Equality comparisons are defined though:: True Comparisons against non-enumeration values will always compare not equal -(again, class:`IntEnum` was explicitly designed to behave differently, see +(again, :class:`IntEnum` was explicitly designed to behave differently, see below):: >>> Color.blue == 2 @@ -594,8 +594,8 @@ alias:: .. note:: This is a useful example for subclassing Enum to add or change other - behaviors as well as disallowing aliases. If the only change desired is - no aliases allowed the :func:`unique` decorator can be used instead. + behaviors as well as disallowing aliases. If the only desired change is + disallowing aliases the :func:`unique` decorator can be used instead. Planet @@ -671,11 +671,11 @@ the member:: ... AttributeError: 'Color' object has no attribute 'blue' - Likewise, the :attr:`__members__` is only available on the class. +Likewise, the :attr:`__members__` is only available on the class. - If you give your :class:`Enum` subclass extra methods, like the `Planet`_ - class above, those methods will show up in a :func:`dir` of the member, - but not of the class:: +If you give your :class:`Enum` subclass extra methods, like the `Planet`_ +class above, those methods will show up in a :func:`dir` of the member, +but not of the class:: >>> dir(Planet) ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] diff --git a/Misc/ACKS b/Misc/ACKS index bad2c517807..1e06e49a3ec 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -988,6 +988,7 @@ Chris Petrilli Roumen Petrov Bjorn Pettersen Justin D. Pettit +Esa Peuha Ronny Pfannschmidt Geoff Philbrick Gavrie Philipson From 17f1eddcfe886fbb11a71550978e37533e19d79e Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 5 Oct 2013 04:26:06 +0300 Subject: [PATCH 03/12] Add missing comma. --- Doc/library/enum.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 9d9e46e564f..f09c60c00dc 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -595,7 +595,7 @@ alias:: This is a useful example for subclassing Enum to add or change other behaviors as well as disallowing aliases. If the only desired change is - disallowing aliases the :func:`unique` decorator can be used instead. + disallowing aliases, the :func:`unique` decorator can be used instead. Planet From 7e70fa53142564bed7415b3798c3e77f0574f816 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 4 Oct 2013 20:28:52 -0600 Subject: [PATCH 04/12] [issue 19152] Ensure we have actually registered ExtensionFileLoader as an ExecutionLoader. --- Lib/importlib/abc.py | 4 ++-- Lib/test/test_importlib/test_abc.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index dcbe01097af..01de67d4574 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -188,7 +188,7 @@ class InspectLoader(Loader): load_module = _bootstrap._LoaderBasics.load_module _register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter, - machinery.ExtensionFileLoader, _bootstrap.NamespaceLoader) + _bootstrap.NamespaceLoader) class ExecutionLoader(InspectLoader): @@ -237,7 +237,7 @@ class ExecutionLoader(InspectLoader): super().init_module_attrs(module) _bootstrap._init_file_attrs(self, module) -_register(machinery.ExtensionFileLoader) +_register(ExecutionLoader, machinery.ExtensionFileLoader) class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader): diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index bfdd6f16a91..979a481b6b7 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -100,6 +100,7 @@ Frozen_InspectLoaderInheritanceTests, Source_InspectLoaderInheritanceTests = tes class ExecutionLoader(InheritanceTests): superclass_names = ['InspectLoader'] + subclass_names = ['ExtensionFileLoader'] tests = create_inheritance_tests(ExecutionLoader) Frozen_ExecutionLoaderInheritanceTests, Source_ExecutionLoaderInheritanceTests = tests From dcd01b49329d5596c3ad3254ae9eeabd15f100d2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 4 Oct 2013 20:35:34 -0600 Subject: [PATCH 05/12] [issue 19152] Add versionadded for ExtensionFileLoader.get_filename(). --- Doc/library/importlib.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index e867a9094e6..443756f36e8 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -874,6 +874,8 @@ find and load modules. Returns :attr:`path`. + .. versionadded:: 3.4 + :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- From 5df8a8a1fd6cc6f4469dc7d3994d06e2aea24c52 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 5 Oct 2013 21:12:18 +0200 Subject: [PATCH 06/12] Issue #19087: Improve bytearray allocation in order to allow cheap popping of data at the front (slice deletion). --- Include/bytearrayobject.h | 11 +- Lib/test/test_bytes.py | 9 ++ Lib/test/test_sys.py | 2 +- Misc/NEWS | 3 + Objects/bytearrayobject.c | 308 ++++++++++++++++++++------------------ 5 files changed, 182 insertions(+), 151 deletions(-) diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index eccd44c7517..19e32a21ea4 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -22,10 +22,11 @@ extern "C" { #ifndef Py_LIMITED_API typedef struct { PyObject_VAR_HEAD + Py_ssize_t ob_alloc; /* How many bytes allocated in ob_buffer */ + char *ob_bytes; /* Physical backing buffer */ + char *ob_start; /* Logical start inside ob_bytes */ /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ - int ob_exports; /* how many buffer exports */ - Py_ssize_t ob_alloc; /* How many bytes allocated */ - char *ob_bytes; + int ob_exports; /* How many buffer exports */ } PyByteArrayObject; #endif @@ -49,8 +50,8 @@ PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); #ifndef Py_LIMITED_API #define PyByteArray_AS_STRING(self) \ (assert(PyByteArray_Check(self)), \ - Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_bytes : _PyByteArray_empty_string) -#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)),Py_SIZE(self)) + Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string) +#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self)) PyAPI_DATA(char) _PyByteArray_empty_string[]; #endif diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f12f911428d..847c7a613fe 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -909,6 +909,15 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): with self.assertRaises(ValueError): b[3:4] = elem + def test_setslice_extend(self): + # Exercise the resizing logic (see issue #19087) + b = bytearray(range(100)) + self.assertEqual(list(b), list(range(100))) + del b[:10] + self.assertEqual(list(b), list(range(10, 100))) + b.extend(range(100, 110)) + self.assertEqual(list(b), list(range(10, 110))) + def test_extended_set_del_slice(self): indices = (0, None, 1, 3, 19, 300, 1<<333, -1, -2, -31, -300) for start in indices: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 30eca4a3ec7..70b55b23169 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -721,7 +721,7 @@ class SizeofTest(unittest.TestCase): samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, vsize('inP') + x.__alloc__()) + check(x, vsize('n2Pi') + x.__alloc__()) # bytearray_iterator check(iter(bytearray()), size('nP')) # cell diff --git a/Misc/NEWS b/Misc/NEWS index 07000dd19e7..60f416ae93e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Projected release date: 2013-10-20 Core and Builtins ----------------- +- Issue #19087: Improve bytearray allocation in order to allow cheap popping + of data at the front (slice deletion). + - Issue #19014: memoryview.cast() is now allowed on zero-length views. - Issue #18690: memoryview is now automatically registered with diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 6ea654e4b6e..c0f0819c122 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -150,6 +150,7 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) } Py_SIZE(new) = size; new->ob_alloc = alloc; + new->ob_start = new->ob_bytes; new->ob_exports = 0; return (PyObject *)new; @@ -177,48 +178,70 @@ int PyByteArray_Resize(PyObject *self, Py_ssize_t size) { void *sval; - Py_ssize_t alloc = ((PyByteArrayObject *)self)->ob_alloc; + PyByteArrayObject *obj = ((PyByteArrayObject *)self); + Py_ssize_t alloc = obj->ob_alloc; + Py_ssize_t logical_offset = obj->ob_start - obj->ob_bytes; assert(self != NULL); assert(PyByteArray_Check(self)); assert(size >= 0); + assert(logical_offset >= 0); + assert(logical_offset <= alloc); if (size == Py_SIZE(self)) { return 0; } - if (!_canresize((PyByteArrayObject *)self)) { + if (!_canresize(obj)) { return -1; } - if (size < alloc / 2) { - /* Major downsize; resize down to exact size */ - alloc = size + 1; - } - else if (size < alloc) { - /* Within allocated size; quick exit */ - Py_SIZE(self) = size; - ((PyByteArrayObject *)self)->ob_bytes[size] = '\0'; /* Trailing null */ - return 0; - } - else if (size <= alloc * 1.125) { - /* Moderate upsize; overallocate similar to list_resize() */ - alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + if (size + logical_offset + 1 < alloc) { + /* Current buffer is large enough to host the requested size, + decide on a strategy. */ + if (size < alloc / 2) { + /* Major downsize; resize down to exact size */ + alloc = size + 1; + } + else { + /* Minor downsize; quick exit */ + Py_SIZE(self) = size; + PyByteArray_AS_STRING(self)[size] = '\0'; /* Trailing null */ + return 0; + } } else { - /* Major upsize; resize up to exact size */ - alloc = size + 1; + /* Need growing, decide on a strategy */ + if (size <= alloc * 1.125) { + /* Moderate upsize; overallocate similar to list_resize() */ + alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + } + else { + /* Major upsize; resize up to exact size */ + alloc = size + 1; + } } - sval = PyObject_Realloc(((PyByteArrayObject *)self)->ob_bytes, alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; + if (logical_offset > 0) { + sval = PyObject_Malloc(alloc); + if (sval == NULL) { + PyErr_NoMemory(); + return -1; + } + memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN(size, Py_SIZE(self))); + PyObject_Free(obj->ob_bytes); + } + else { + sval = PyObject_Realloc(obj->ob_bytes, alloc); + if (sval == NULL) { + PyErr_NoMemory(); + return -1; + } } - ((PyByteArrayObject *)self)->ob_bytes = sval; + obj->ob_bytes = obj->ob_start = sval; Py_SIZE(self) = size; - ((PyByteArrayObject *)self)->ob_alloc = alloc; - ((PyByteArrayObject *)self)->ob_bytes[size] = '\0'; /* Trailing null byte */ + obj->ob_alloc = alloc; + obj->ob_bytes[size] = '\0'; /* Trailing null byte */ return 0; } @@ -288,13 +311,13 @@ bytearray_iconcat(PyByteArrayObject *self, PyObject *other) } if (size < self->ob_alloc) { Py_SIZE(self) = size; - self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ + PyByteArray_AS_STRING(self)[Py_SIZE(self)] = '\0'; /* Trailing null byte */ } else if (PyByteArray_Resize((PyObject *)self, size) < 0) { PyBuffer_Release(&vo); return NULL; } - memcpy(self->ob_bytes + mysize, vo.buf, vo.len); + memcpy(PyByteArray_AS_STRING(self) + mysize, vo.buf, vo.len); PyBuffer_Release(&vo); Py_INCREF(self); return (PyObject *)self; @@ -331,6 +354,7 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) { Py_ssize_t mysize; Py_ssize_t size; + char *buf; if (count < 0) count = 0; @@ -338,19 +362,16 @@ bytearray_irepeat(PyByteArrayObject *self, Py_ssize_t count) if (count > 0 && mysize > PY_SSIZE_T_MAX / count) return PyErr_NoMemory(); size = mysize * count; - if (size < self->ob_alloc) { - Py_SIZE(self) = size; - self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ - } - else if (PyByteArray_Resize((PyObject *)self, size) < 0) + if (PyByteArray_Resize((PyObject *)self, size) < 0) return NULL; + buf = PyByteArray_AS_STRING(self); if (mysize == 1) - memset(self->ob_bytes, self->ob_bytes[0], size); + memset(buf, buf[0], size); else { Py_ssize_t i; for (i = 1; i < count; i++) - memcpy(self->ob_bytes + i*mysize, self->ob_bytes, mysize); + memcpy(buf + i*mysize, buf, mysize); } Py_INCREF(self); @@ -366,7 +387,7 @@ bytearray_getitem(PyByteArrayObject *self, Py_ssize_t i) PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); return NULL; } - return PyLong_FromLong((unsigned char)(self->ob_bytes[i])); + return PyLong_FromLong((unsigned char)(PyByteArray_AS_STRING(self)[i])); } static PyObject * @@ -385,7 +406,7 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); return NULL; } - return PyLong_FromLong((unsigned char)(self->ob_bytes[i])); + return PyLong_FromLong((unsigned char)(PyByteArray_AS_STRING(self)[i])); } else if (PySlice_Check(index)) { Py_ssize_t start, stop, step, slicelength, cur, i; @@ -398,8 +419,8 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) if (slicelength <= 0) return PyByteArray_FromStringAndSize("", 0); else if (step == 1) { - return PyByteArray_FromStringAndSize(self->ob_bytes + start, - slicelength); + return PyByteArray_FromStringAndSize( + PyByteArray_AS_STRING(self) + start, slicelength); } else { char *source_buf = PyByteArray_AS_STRING(self); @@ -424,11 +445,69 @@ bytearray_subscript(PyByteArrayObject *self, PyObject *index) } } +static int +bytearray_setslice_linear(PyByteArrayObject *self, + Py_ssize_t lo, Py_ssize_t hi, + char *bytes, Py_ssize_t bytes_len) +{ + Py_ssize_t avail = hi - lo; + char *buf = PyByteArray_AS_STRING(self); + Py_ssize_t growth = bytes_len - avail; + assert(avail >= 0); + + if (growth != 0) { + if (growth < 0) { + if (!_canresize(self)) + return -1; + if (lo == 0) { + /* Shrink the buffer by advancing its logical start */ + self->ob_start -= growth; + /* + 0 lo hi old_size + | |<----avail----->|<-----tail------>| + | |<-bytes_len->|<-----tail------>| + 0 new_lo new_hi new_size + */ + } + else { + /* + 0 lo hi old_size + | |<----avail----->|<-----tomove------>| + | |<-bytes_len->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(buf + lo + bytes_len, buf + hi, + Py_SIZE(self) - hi); + } + } + /* XXX(nnorwitz): need to verify this can't overflow! */ + if (PyByteArray_Resize( + (PyObject *)self, Py_SIZE(self) + growth) < 0) + return -1; + buf = PyByteArray_AS_STRING(self); + if (growth > 0) { + /* Make the place for the additional bytes */ + /* + 0 lo hi old_size + | |<-avail->|<-----tomove------>| + | |<---bytes_len-->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(buf + lo + bytes_len, buf + hi, + Py_SIZE(self) - lo - bytes_len); + } + } + + if (bytes_len > 0) + memcpy(buf + lo, bytes, bytes_len); + return 0; +} + static int bytearray_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, PyObject *values) { - Py_ssize_t avail, needed; + Py_ssize_t needed; void *bytes; Py_buffer vbytes; int res = 0; @@ -467,50 +546,9 @@ bytearray_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, if (hi > Py_SIZE(self)) hi = Py_SIZE(self); - avail = hi - lo; - if (avail < 0) - lo = hi = avail = 0; - - if (avail != needed) { - if (avail > needed) { - if (!_canresize(self)) { - res = -1; - goto finish; - } - /* - 0 lo hi old_size - | |<----avail----->|<-----tomove------>| - | |<-needed->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, - Py_SIZE(self) - hi); - } - /* XXX(nnorwitz): need to verify this can't overflow! */ - if (PyByteArray_Resize((PyObject *)self, - Py_SIZE(self) + needed - avail) < 0) { - res = -1; - goto finish; - } - if (avail < needed) { - /* - 0 lo hi old_size - | |<-avail->|<-----tomove------>| - | |<----needed---->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, - Py_SIZE(self) - lo - needed); - } - } - - if (needed > 0) - memcpy(self->ob_bytes + lo, bytes, needed); - - - finish: + res = bytearray_setslice_linear(self, lo, hi, bytes, needed); if (vbytes.len != -1) - PyBuffer_Release(&vbytes); + PyBuffer_Release(&vbytes); return res; } @@ -533,7 +571,7 @@ bytearray_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) if (!_getbytevalue(value, &ival)) return -1; - self->ob_bytes[i] = ival; + PyByteArray_AS_STRING(self)[i] = ival; return 0; } @@ -541,7 +579,8 @@ static int bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *values) { Py_ssize_t start, stop, step, slicelen, needed; - char *bytes; + char *buf, *bytes; + buf = PyByteArray_AS_STRING(self); if (PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -568,7 +607,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu int ival; if (!_getbytevalue(values, &ival)) return -1; - self->ob_bytes[i] = (char)ival; + buf[i] = (char)ival; return 0; } } @@ -606,7 +645,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu } else { assert(PyByteArray_Check(values)); - bytes = ((PyByteArrayObject *)values)->ob_bytes; + bytes = PyByteArray_AS_STRING(values); needed = Py_SIZE(values); } /* Make sure b[5:2] = ... inserts before 5, not before 2. */ @@ -614,38 +653,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu (step > 0 && start > stop)) stop = start; if (step == 1) { - if (slicelen != needed) { - if (!_canresize(self)) - return -1; - if (slicelen > needed) { - /* - 0 start stop old_size - | |<---slicelen--->|<-----tomove------>| - | |<-needed->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, - Py_SIZE(self) - stop); - } - if (PyByteArray_Resize((PyObject *)self, - Py_SIZE(self) + needed - slicelen) < 0) - return -1; - if (slicelen < needed) { - /* - 0 lo hi old_size - | |<-avail->|<-----tomove------>| - | |<----needed---->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, - Py_SIZE(self) - start - needed); - } - } - - if (needed > 0) - memcpy(self->ob_bytes + start, bytes, needed); - - return 0; + return bytearray_setslice_linear(self, start, stop, bytes, needed); } else { if (needed == 0) { @@ -672,14 +680,14 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu if (cur + step >= (size_t)PyByteArray_GET_SIZE(self)) lim = PyByteArray_GET_SIZE(self) - cur - 1; - memmove(self->ob_bytes + cur - i, - self->ob_bytes + cur + 1, lim); + memmove(buf + cur - i, + buf + cur + 1, lim); } /* Move the tail of the bytes, in one chunk */ cur = start + (size_t)slicelen*step; if (cur < (size_t)PyByteArray_GET_SIZE(self)) { - memmove(self->ob_bytes + cur - slicelen, - self->ob_bytes + cur, + memmove(buf + cur - slicelen, + buf + cur, PyByteArray_GET_SIZE(self) - cur); } if (PyByteArray_Resize((PyObject *)self, @@ -701,7 +709,7 @@ bytearray_ass_subscript(PyByteArrayObject *self, PyObject *index, PyObject *valu return -1; } for (cur = start, i = 0; i < slicelen; cur += step, i++) - self->ob_bytes[cur] = bytes[i]; + buf[cur] = bytes[i]; return 0; } } @@ -781,7 +789,7 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) if (count > 0) { if (PyByteArray_Resize((PyObject *)self, count)) return -1; - memset(self->ob_bytes, 0, count); + memset(PyByteArray_AS_STRING(self), 0, count); } return 0; } @@ -794,7 +802,8 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) return -1; size = view.len; if (PyByteArray_Resize((PyObject *)self, size) < 0) goto fail; - if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) + if (PyBuffer_ToContiguous(PyByteArray_AS_STRING(self), + &view, size, 'C') < 0) goto fail; PyBuffer_Release(&view); return 0; @@ -838,7 +847,7 @@ bytearray_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) Py_SIZE(self)++; else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) goto error; - self->ob_bytes[Py_SIZE(self)-1] = value; + PyByteArray_AS_STRING(self)[Py_SIZE(self)-1] = value; } /* Clean up and return success */ @@ -863,6 +872,7 @@ bytearray_repr(PyByteArrayObject *self) size_t newsize; PyObject *v; Py_ssize_t i; + char *bytes; char c; char *p; int quote; @@ -899,11 +909,12 @@ bytearray_repr(PyByteArrayObject *self) *p++ = *quote_prefix++; *p++ = quote; + bytes = PyByteArray_AS_STRING(self); for (i = 0; i < length; i++) { /* There's at least enough room for a hex escape and a closing quote. */ assert(newsize - (p - buffer) >= 5); - c = self->ob_bytes[i]; + c = bytes[i]; if (c == '\'' || c == '\\') *p++ = '\\', *p++ = c; else if (c == '\t') @@ -2194,7 +2205,7 @@ bytearray_reverse(PyByteArrayObject *self, PyObject *unused) Py_ssize_t i, j, n = Py_SIZE(self); j = n / 2; - head = self->ob_bytes; + head = PyByteArray_AS_STRING(self); tail = head + n - 1; for (i = 0; i < j; i++) { swap = *head; @@ -2215,6 +2226,7 @@ bytearray_insert(PyByteArrayObject *self, PyObject *args) PyObject *value; int ival; Py_ssize_t where, n = Py_SIZE(self); + char *buf; if (!PyArg_ParseTuple(args, "nO:insert", &where, &value)) return NULL; @@ -2228,6 +2240,7 @@ bytearray_insert(PyByteArrayObject *self, PyObject *args) return NULL; if (PyByteArray_Resize((PyObject *)self, n + 1) < 0) return NULL; + buf = PyByteArray_AS_STRING(self); if (where < 0) { where += n; @@ -2236,8 +2249,8 @@ bytearray_insert(PyByteArrayObject *self, PyObject *args) } if (where > n) where = n; - memmove(self->ob_bytes + where + 1, self->ob_bytes + where, n - where); - self->ob_bytes[where] = ival; + memmove(buf + where + 1, buf + where, n - where); + buf[where] = ival; Py_RETURN_NONE; } @@ -2262,7 +2275,7 @@ bytearray_append(PyByteArrayObject *self, PyObject *arg) if (PyByteArray_Resize((PyObject *)self, n + 1) < 0) return NULL; - self->ob_bytes[n] = value; + PyByteArray_AS_STRING(self)[n] = value; Py_RETURN_NONE; } @@ -2355,6 +2368,7 @@ bytearray_pop(PyByteArrayObject *self, PyObject *args) { int value; Py_ssize_t where = -1, n = Py_SIZE(self); + char *buf; if (!PyArg_ParseTuple(args, "|n:pop", &where)) return NULL; @@ -2373,8 +2387,9 @@ bytearray_pop(PyByteArrayObject *self, PyObject *args) if (!_canresize(self)) return NULL; - value = self->ob_bytes[where]; - memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); + buf = PyByteArray_AS_STRING(self); + value = buf[where]; + memmove(buf + where, buf + where + 1, n - where); if (PyByteArray_Resize((PyObject *)self, n - 1) < 0) return NULL; @@ -2390,12 +2405,13 @@ bytearray_remove(PyByteArrayObject *self, PyObject *arg) { int value; Py_ssize_t where, n = Py_SIZE(self); + char *buf = PyByteArray_AS_STRING(self); if (! _getbytevalue(arg, &value)) return NULL; for (where = 0; where < n; where++) { - if (self->ob_bytes[where] == value) + if (buf[where] == value) break; } if (where == n) { @@ -2405,7 +2421,7 @@ bytearray_remove(PyByteArrayObject *self, PyObject *arg) if (!_canresize(self)) return NULL; - memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); + memmove(buf + where, buf + where + 1, n - where); if (PyByteArray_Resize((PyObject *)self, n - 1) < 0) return NULL; @@ -2459,7 +2475,7 @@ bytearray_strip(PyByteArrayObject *self, PyObject *args) argptr = varg.buf; argsize = varg.len; } - myptr = self->ob_bytes; + myptr = PyByteArray_AS_STRING(self); mysize = Py_SIZE(self); left = lstrip_helper(myptr, mysize, argptr, argsize); if (left == mysize) @@ -2468,7 +2484,7 @@ bytearray_strip(PyByteArrayObject *self, PyObject *args) right = rstrip_helper(myptr, mysize, argptr, argsize); if (arg != Py_None) PyBuffer_Release(&varg); - return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); + return PyByteArray_FromStringAndSize(myptr + left, right - left); } PyDoc_STRVAR(lstrip__doc__, @@ -2496,13 +2512,13 @@ bytearray_lstrip(PyByteArrayObject *self, PyObject *args) argptr = varg.buf; argsize = varg.len; } - myptr = self->ob_bytes; + myptr = PyByteArray_AS_STRING(self); mysize = Py_SIZE(self); left = lstrip_helper(myptr, mysize, argptr, argsize); right = mysize; if (arg != Py_None) PyBuffer_Release(&varg); - return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); + return PyByteArray_FromStringAndSize(myptr + left, right - left); } PyDoc_STRVAR(rstrip__doc__, @@ -2530,12 +2546,12 @@ bytearray_rstrip(PyByteArrayObject *self, PyObject *args) argptr = varg.buf; argsize = varg.len; } - myptr = self->ob_bytes; + myptr = PyByteArray_AS_STRING(self); mysize = Py_SIZE(self); right = rstrip_helper(myptr, mysize, argptr, argsize); if (arg != Py_None) PyBuffer_Release(&varg); - return PyByteArray_FromStringAndSize(self->ob_bytes, right); + return PyByteArray_FromStringAndSize(myptr, right); } PyDoc_STRVAR(decode_doc, @@ -2686,6 +2702,7 @@ _common_reduce(PyByteArrayObject *self, int proto) { PyObject *dict; _Py_IDENTIFIER(__dict__); + char *buf; dict = _PyObject_GetAttrId((PyObject *)self, &PyId___dict__); if (dict == NULL) { @@ -2694,19 +2711,20 @@ _common_reduce(PyByteArrayObject *self, int proto) Py_INCREF(dict); } + buf = PyByteArray_AS_STRING(self); if (proto < 3) { /* use str based reduction for backwards compatibility with Python 2.x */ PyObject *latin1; - if (self->ob_bytes) - latin1 = PyUnicode_DecodeLatin1(self->ob_bytes, Py_SIZE(self), NULL); + if (Py_SIZE(self)) + latin1 = PyUnicode_DecodeLatin1(buf, Py_SIZE(self), NULL); else latin1 = PyUnicode_FromString(""); return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); } else { /* use more efficient byte based reduction */ - if (self->ob_bytes) { - return Py_BuildValue("(O(y#)N)", Py_TYPE(self), self->ob_bytes, Py_SIZE(self), dict); + if (Py_SIZE(self)) { + return Py_BuildValue("(O(y#)N)", Py_TYPE(self), buf, Py_SIZE(self), dict); } else { return Py_BuildValue("(O()N)", Py_TYPE(self), dict); @@ -2938,7 +2956,7 @@ bytearrayiter_next(bytesiterobject *it) if (it->it_index < PyByteArray_GET_SIZE(seq)) { item = PyLong_FromLong( - (unsigned char)seq->ob_bytes[it->it_index]); + (unsigned char)PyByteArray_AS_STRING(seq)[it->it_index]); if (item != NULL) ++it->it_index; return item; From 5b720753872ff522a19fcc9ba6181caf785df2f1 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 5 Oct 2013 21:24:10 +0200 Subject: [PATCH 07/12] Hopefully fix Windows compilation error following 499a96611baa --- Objects/bytearrayobject.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index c0f0819c122..2358e05d251 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2431,21 +2431,21 @@ bytearray_remove(PyByteArrayObject *self, PyObject *arg) /* XXX These two helpers could be optimized if argsize == 1 */ static Py_ssize_t -lstrip_helper(unsigned char *myptr, Py_ssize_t mysize, +lstrip_helper(char *myptr, Py_ssize_t mysize, void *argptr, Py_ssize_t argsize) { Py_ssize_t i = 0; - while (i < mysize && memchr(argptr, myptr[i], argsize)) + while (i < mysize && memchr(argptr, (unsigned char) myptr[i], argsize)) i++; return i; } static Py_ssize_t -rstrip_helper(unsigned char *myptr, Py_ssize_t mysize, +rstrip_helper(char *myptr, Py_ssize_t mysize, void *argptr, Py_ssize_t argsize) { Py_ssize_t i = mysize - 1; - while (i >= 0 && memchr(argptr, myptr[i], argsize)) + while (i >= 0 && memchr(argptr, (unsigned char) myptr[i], argsize)) i--; return i + 1; } @@ -2460,7 +2460,7 @@ static PyObject * bytearray_strip(PyByteArrayObject *self, PyObject *args) { Py_ssize_t left, right, mysize, argsize; - void *myptr, *argptr; + char *myptr, *argptr; PyObject *arg = Py_None; Py_buffer varg; if (!PyArg_ParseTuple(args, "|O:strip", &arg)) @@ -2472,7 +2472,7 @@ bytearray_strip(PyByteArrayObject *self, PyObject *args) else { if (_getbuffer(arg, &varg) < 0) return NULL; - argptr = varg.buf; + argptr = (char *) varg.buf; argsize = varg.len; } myptr = PyByteArray_AS_STRING(self); @@ -2497,7 +2497,7 @@ static PyObject * bytearray_lstrip(PyByteArrayObject *self, PyObject *args) { Py_ssize_t left, right, mysize, argsize; - void *myptr, *argptr; + char *myptr, *argptr; PyObject *arg = Py_None; Py_buffer varg; if (!PyArg_ParseTuple(args, "|O:lstrip", &arg)) @@ -2509,7 +2509,7 @@ bytearray_lstrip(PyByteArrayObject *self, PyObject *args) else { if (_getbuffer(arg, &varg) < 0) return NULL; - argptr = varg.buf; + argptr = (char *) varg.buf; argsize = varg.len; } myptr = PyByteArray_AS_STRING(self); @@ -2531,7 +2531,7 @@ static PyObject * bytearray_rstrip(PyByteArrayObject *self, PyObject *args) { Py_ssize_t right, mysize, argsize; - void *myptr, *argptr; + char *myptr, *argptr; PyObject *arg = Py_None; Py_buffer varg; if (!PyArg_ParseTuple(args, "|O:rstrip", &arg)) @@ -2543,7 +2543,7 @@ bytearray_rstrip(PyByteArrayObject *self, PyObject *args) else { if (_getbuffer(arg, &varg) < 0) return NULL; - argptr = varg.buf; + argptr = (char *) varg.buf; argsize = varg.len; } myptr = PyByteArray_AS_STRING(self); From 9185a5ae7ec6fa6e27ca1e1b0562c86f4c928726 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 5 Oct 2013 16:28:04 -0400 Subject: [PATCH 08/12] fix comment --- Include/bytearrayobject.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 19e32a21ea4..a757b880592 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -22,7 +22,7 @@ extern "C" { #ifndef Py_LIMITED_API typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_buffer */ + Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ From 5792ce151c0b8b1f9f241d7e99d2c2e7ad9782c3 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 6 Oct 2013 00:36:45 +0300 Subject: [PATCH 09/12] #19067: use imperative mood in range object docstrings. Patch by Marco Buttu. --- Objects/rangeobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index 343881ae317..d790ca8f476 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -139,7 +139,7 @@ PyDoc_STRVAR(range_doc, "range(stop) -> range object\n\ range(start, stop[, step]) -> range object\n\ \n\ -Returns a virtual sequence of numbers from start to stop by step."); +Return a virtual sequence of numbers from start to stop by step."); static void range_dealloc(rangeobject *r) @@ -865,14 +865,14 @@ static PyObject * range_iter(PyObject *seq); static PyObject * range_reverse(PyObject *seq); PyDoc_STRVAR(reverse_doc, -"Returns a reverse iterator."); +"Return a reverse iterator."); PyDoc_STRVAR(count_doc, "rangeobject.count(value) -> integer -- return number of occurrences of value"); PyDoc_STRVAR(index_doc, "rangeobject.index(value, [start, [stop]]) -> integer -- return index of value.\n" -"Raises ValueError if the value is not present."); +"Raise ValueError if the value is not present."); static PyMethodDef range_methods[] = { {"__reversed__", (PyCFunction)range_reverse, METH_NOARGS, reverse_doc}, From 488d244e1ab9e50c5b7e83904897d8e068546086 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 6 Oct 2013 00:39:18 +0300 Subject: [PATCH 10/12] #19068: use imperative mood in complex object docstrings. Patch by Marco Buttu. --- Objects/complexobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 403c60c917d..b08aa6f001f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -681,7 +681,7 @@ complex_conjugate(PyObject *self) PyDoc_STRVAR(complex_conjugate_doc, "complex.conjugate() -> complex\n" "\n" -"Returns the complex conjugate of its argument. (3-4j).conjugate() == 3+4j."); +"Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j."); static PyObject * complex_getnewargs(PyComplexObject *v) @@ -693,7 +693,7 @@ complex_getnewargs(PyComplexObject *v) PyDoc_STRVAR(complex__format__doc, "complex.__format__() -> str\n" "\n" -"Converts to a string according to format_spec."); +"Convert to a string according to format_spec."); static PyObject * complex__format__(PyObject* self, PyObject* args) From 7760b4eb4bedba81eb69d14f98ac978d37bc691a Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sun, 6 Oct 2013 00:45:11 +0300 Subject: [PATCH 11/12] #19069: use imperative mood in float object docstrings. Patch by Marco Buttu. --- Objects/floatobject.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 5fcdf3225c6..a08a85272c1 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1503,9 +1503,9 @@ error: PyDoc_STRVAR(float_as_integer_ratio_doc, "float.as_integer_ratio() -> (int, int)\n" "\n" -"Returns a pair of integers, whose ratio is exactly equal to the original\n" +"Return a pair of integers, whose ratio is exactly equal to the original\n" "float and with a positive denominator.\n" -"Raises OverflowError on infinities and a ValueError on NaNs.\n" +"Raise OverflowError on infinities and a ValueError on NaNs.\n" "\n" ">>> (10.0).as_integer_ratio()\n" "(10, 1)\n" @@ -1692,7 +1692,7 @@ PyDoc_STRVAR(float_setformat_doc, "'IEEE, big-endian' or 'IEEE, little-endian', and in addition can only be\n" "one of the latter two if it appears to match the underlying C reality.\n" "\n" -"Overrides the automatic determination of C-level floating point type.\n" +"Override the automatic determination of C-level floating point type.\n" "This affects how floats are converted to and from binary strings."); static PyObject * @@ -1731,12 +1731,12 @@ PyDoc_STRVAR(float__format__doc, static PyMethodDef float_methods[] = { {"conjugate", (PyCFunction)float_float, METH_NOARGS, - "Returns self, the complex conjugate of any float."}, + "Return self, the complex conjugate of any float."}, {"__trunc__", (PyCFunction)float_trunc, METH_NOARGS, - "Returns the Integral closest to x between 0 and x."}, + "Return the Integral closest to x between 0 and x."}, {"__round__", (PyCFunction)float_round, METH_VARARGS, - "Returns the Integral closest to x, rounding half toward even.\n" - "When an argument is passed, works like built-in round(x, ndigits)."}, + "Return the Integral closest to x, rounding half toward even.\n" + "When an argument is passed, work like built-in round(x, ndigits)."}, {"as_integer_ratio", (PyCFunction)float_as_integer_ratio, METH_NOARGS, float_as_integer_ratio_doc}, {"fromhex", (PyCFunction)float_fromhex, @@ -1744,14 +1744,14 @@ static PyMethodDef float_methods[] = { {"hex", (PyCFunction)float_hex, METH_NOARGS, float_hex_doc}, {"is_integer", (PyCFunction)float_is_integer, METH_NOARGS, - "Returns True if the float is an integer."}, + "Return True if the float is an integer."}, #if 0 {"is_inf", (PyCFunction)float_is_inf, METH_NOARGS, - "Returns True if the float is positive or negative infinite."}, + "Return True if the float is positive or negative infinite."}, {"is_finite", (PyCFunction)float_is_finite, METH_NOARGS, - "Returns True if the float is finite, neither infinite nor NaN."}, + "Return True if the float is finite, neither infinite nor NaN."}, {"is_nan", (PyCFunction)float_is_nan, METH_NOARGS, - "Returns True if the float is not a number (NaN)."}, + "Return True if the float is not a number (NaN)."}, #endif {"__getnewargs__", (PyCFunction)float_getnewargs, METH_NOARGS}, {"__getformat__", (PyCFunction)float_getformat, From 81a93159d7a4bfc6a6d06f44528d9d17a8c634c2 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 5 Oct 2013 16:53:52 -0500 Subject: [PATCH 12/12] Issue #19171: speed some cases of 3-argument long pow(). Reduce the base by the modulus when the base is larger than the modulus. This can unboundedly speed the "startup costs" of doing modular exponentiation, particularly in cases where the base is much larger than the modulus. Original patch by Armin Rigo, inspired by https://github.com/pyca/ed25519. --- Objects/longobject.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 985b1ec83c6..6383507a5c6 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3868,10 +3868,16 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) goto Done; } - /* if base < 0: - base = base % modulus - Having the base positive just makes things easier. */ - if (Py_SIZE(a) < 0) { + /* Reduce base by modulus in some cases: + 1. If base < 0. Forcing the base non-negative makes things easier. + 2. If base is obviously larger than the modulus. The "small + exponent" case later can multiply directly by base repeatedly, + while the "large exponent" case multiplies directly by base 31 + times. It can be unboundedly faster to multiply by + base % modulus instead. + We could _always_ do this reduction, but l_divmod() isn't cheap, + so we only do it when it buys something. */ + if (Py_SIZE(a) < 0 || Py_SIZE(a) > Py_SIZE(c)) { if (l_divmod(a, c, NULL, &temp) < 0) goto Error; Py_DECREF(a);