traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update the array overallocation scheme to match the approach used for 

lists.  Speeds append() operations and reduces memory requirements
(because of more conservative overallocation).

Paves the way for the feature request for array.extend() to support
arbitrary iterable arguments.
This commit is contained in:
Raymond Hettinger 2004-03-14 04:37:50 +00:00
parent 118e1277a6
commit 6e2ee866fa
2 changed files with 79 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Misc/NEWS
View File

@ -180,7 +180,9 @@ Core and builtins
Extension modules
-----------------
- array objects now support the copy module
- array objects now support the copy module. Also, their resizing
scheme has been updated the same as for list objects. The improves
performance for append() operations.
- cStringIO.writelines() now accepts any iterable argument and writes
the lines one at a time rather than joining them and writing once.

131
Modules/arraymodule.c
View File

@ -13,52 +13,6 @@
#endif /* DONT_HAVE_SYS_TYPES_H */
#endif /* !STDC_HEADERS */
/* Shamelessy stolen from listobject.c */
static int
roundupsize(int n)
{
unsigned int nbits = 0;
unsigned int n2 = (unsigned int)n >> 5;
/* Round up:
* If n < 256, to a multiple of 8.
* If n < 2048, to a multiple of 64.
* If n < 16384, to a multiple of 512.
* If n < 131072, to a multiple of 4096.
* If n < 1048576, to a multiple of 32768.
* If n < 8388608, to a multiple of 262144.
* If n < 67108864, to a multiple of 2097152.
* If n < 536870912, to a multiple of 16777216.
* ...
* If n < 2**(5+3*i), to a multiple of 2**(3*i).
*
* This over-allocates proportional to the list size, making room
* for additional growth. The over-allocation is mild, but is
* enough to give linear-time amortized behavior over a long
* sequence of appends() in the presence of a poorly-performing
* system realloc() (which is a reality, e.g., across all flavors
* of Windows, with Win9x behavior being particularly bad -- and
* we've still got address space fragmentation problems on Win9x
* even with this scheme, although it requires much longer lists to
* provoke them than it used to).
*/
do {
n2 >>= 3;
nbits += 3;
} while (n2);
return ((n >> nbits) + 1) << nbits;
}
#define NRESIZE(var, type, nitems) \
do { \
size_t _new_size = roundupsize(nitems); \
if (_new_size <= ((~(size_t)0) / sizeof(type))) \
PyMem_RESIZE(var, type, _new_size); \
else \
var = NULL; \
} while (0)
/* END SHAMELESSLY STOLEN CODE */
struct arrayobject; /* Forward */
/* All possible arraydescr values are defined in the vector "descriptors"
@ -76,6 +30,7 @@ typedef struct arrayobject {
PyObject_HEAD
int ob_size;
char *ob_item;
int allocated;
struct arraydescr *ob_descr;
} arrayobject;
@ -84,6 +39,54 @@ static PyTypeObject Arraytype;
#define array_Check(op) PyObject_TypeCheck(op, &Arraytype)
#define array_CheckExact(op) ((op)->ob_type == &Arraytype)
static int
array_resize(arrayobject *self, int newsize)
{
char *items;
size_t _new_size;
/* Bypass realloc() when a previous overallocation is large enough
to accommodate the newsize. If the newsize is 16 smaller than the
current size, then proceed with the realloc() to shrink the list.
*/
if (self->allocated >= newsize &&
self->ob_size < newsize + 16 &&
self->ob_item != NULL) {
self->ob_size = newsize;
return 0;
}
/* This over-allocates proportional to the array size, making room
* for additional growth. The over-allocation is mild, but is
* enough to give linear-time amortized behavior over a long
* sequence of appends() in the presence of a poorly-performing
* system realloc().
* The growth pattern is: 0, 4, 8, 16, 25, 34, 46, 56, 67, 79, ...
* Note, the pattern starts out the same as for lists but then
* grows at a smaller rate so that larger arrays only overallocate
* by about 1/16th -- this is done because arrays are presumed to be more
* memory critical.
*/
_new_size = (newsize >> 4) + (self->ob_size < 8 ? 3 : 7) + newsize;
items = self->ob_item;
/* XXX The following multiplication and division does not optimize away
like it does for lists since the size is not known at compile time */
if (_new_size <= ((~(size_t)0) / self->ob_descr->itemsize))
PyMem_RESIZE(items, char, (_new_size * self->ob_descr->itemsize));
else
items = NULL;
if (items == NULL) {
PyErr_NoMemory();
return -1;
}
self->ob_item = items;
self->ob_size = newsize;
self->allocated = _new_size;
return 0;
}
/****************************************************************************
Get and Set functions for each type.
A Get function takes an arrayobject* and an integer index, returning the
@ -438,6 +441,7 @@ newarrayobject(PyTypeObject *type, int size, struct arraydescr *descr)
}
}
op->ob_descr = descr;
op->allocated = size;
return (PyObject *) op;
}
@ -455,30 +459,29 @@ static int
ins1(arrayobject *self, int where, PyObject *v)
{
char *items;
int n = self->ob_size;
if (v == NULL) {
PyErr_BadInternalCall();
return -1;
}
if ((*self->ob_descr->setitem)(self, -1, v) < 0)
return -1;
items = self->ob_item;
NRESIZE(items, char, (self->ob_size+1) * self->ob_descr->itemsize);
if (items == NULL) {
PyErr_NoMemory();
if (array_resize(self, n+1) == -1)
return -1;
}
items = self->ob_item;
if (where < 0) {
where += self->ob_size;
where += n;
if (where < 0)
where = 0;
}
if (where > self->ob_size)
where = self->ob_size;
if (where > n)
where = n;
/* appends don't need to call memmove() */
if (where != n)
memmove(items + (where+1)*self->ob_descr->itemsize,
items + where*self->ob_descr->itemsize,
(self->ob_size-where)*self->ob_descr->itemsize);
self->ob_item = items;
self->ob_size++;
(n-where)*self->ob_descr->itemsize);
return (*self->ob_descr->setitem)(self, where, v);
}
@ -728,6 +731,7 @@ array_ass_slice(arrayobject *a, int ilow, int ihigh, PyObject *v)
PyMem_RESIZE(item, char, a->ob_size*a->ob_descr->itemsize);
/* Can't fail */
a->ob_item = item;
a->allocated = a->ob_size;
}
else if (d > 0) { /* Insert d items */
PyMem_RESIZE(item, char,
@ -741,6 +745,7 @@ array_ass_slice(arrayobject *a, int ilow, int ihigh, PyObject *v)
(a->ob_size-ihigh)*a->ob_descr->itemsize);
a->ob_item = item;
a->ob_size += d;
a->allocated = a->ob_size;
}
if (n > 0)
memcpy(item + ilow*a->ob_descr->itemsize, b->ob_item,
@ -796,6 +801,7 @@ array_do_extend(arrayobject *self, PyObject *bb)
memcpy(self->ob_item + self->ob_size*self->ob_descr->itemsize,
b->ob_item, b->ob_size*b->ob_descr->itemsize);
self->ob_size = size;
self->allocated = size;
return 0;
#undef b
@ -825,6 +831,7 @@ array_inplace_repeat(arrayobject *self, int n)
PyMem_FREE(items);
self->ob_item = NULL;
self->ob_size = 0;
self->allocated = 0;
}
else {
PyMem_Resize(items, char, n * size);
@ -837,6 +844,7 @@ array_inplace_repeat(arrayobject *self, int n)
}
self->ob_item = items;
self->ob_size *= n;
self->allocated = self->ob_size;
}
}
Py_INCREF(self);
@ -1158,12 +1166,14 @@ array_fromfile(arrayobject *self, PyObject *args)
}
self->ob_item = item;
self->ob_size += n;
self->allocated = self->ob_size;
nread = fread(item + (self->ob_size - n) * itemsize,
itemsize, n, fp);
if (nread < (size_t)n) {
self->ob_size -= (n - nread);
PyMem_RESIZE(item, char, self->ob_size*itemsize);
self->ob_item = item;
self->allocated = self->ob_size;
PyErr_SetString(PyExc_EOFError,
"not enough items in file");
return NULL;
@ -1230,6 +1240,7 @@ array_fromlist(arrayobject *self, PyObject *list)
}
self->ob_item = item;
self->ob_size += n;
self->allocated = self->ob_size;
for (i = 0; i < n; i++) {
PyObject *v = PyList_GetItem(list, i);
if ((*self->ob_descr->setitem)(self,
@ -1238,6 +1249,7 @@ array_fromlist(arrayobject *self, PyObject *list)
PyMem_RESIZE(item, char,
self->ob_size * itemsize);
self->ob_item = item;
self->allocated = self->ob_size;
return NULL;
}
}
@ -1300,6 +1312,7 @@ array_fromstring(arrayobject *self, PyObject *args)
}
self->ob_item = item;
self->ob_size += n;
self->allocated = self->ob_size;
memcpy(item + (self->ob_size - n) * itemsize,
str, itemsize*n);
}
@ -1353,6 +1366,7 @@ array_fromunicode(arrayobject *self, PyObject *args)
}
self->ob_item = (char *) item;
self->ob_size += n;
self->allocated = self->ob_size;
memcpy(item + self->ob_size - n,
ustr, n * sizeof(Py_UNICODE));
}
@ -1611,7 +1625,7 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value)
self->ob_size -= slicelength;
self->ob_item = PyMem_REALLOC(self->ob_item, itemsize*self->ob_size);
self->allocated = self->ob_size;
return 0;
}
@ -1811,6 +1825,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self->ob_item = item;
self->ob_size = n / sizeof(Py_UNICODE);
memcpy(item, PyUnicode_AS_DATA(initial), n);
self->allocated = self->ob_size;
}
#endif
}