Patch #1953
I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers. The patch also renames sys._cleartypecache to sys._clear_type_cache
This commit is contained in:
parent
a26cf9b760
commit
422051a367
|
@ -84,3 +84,12 @@ Floating Point Objects
|
|||
Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
|
||||
|
||||
.. versionadded:: 2.6
|
||||
|
||||
|
||||
.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
|
||||
|
||||
Compact the float free list. *bc* is the number of allocated blocks before
|
||||
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
|
||||
of remaining objects in the blocks.
|
||||
|
||||
.. versionadded:: 2.6
|
||||
|
|
|
@ -120,3 +120,12 @@ Plain Integer Objects
|
|||
|
||||
Return the system's idea of the largest integer it can handle
|
||||
(:const:`LONG_MAX`, as defined in the system header files).
|
||||
|
||||
|
||||
.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
|
||||
|
||||
Compact the integer free list. *bc* is the number of allocated blocks before
|
||||
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
|
||||
of remaining objects in the blocks.
|
||||
|
||||
.. versionadded:: 2.6
|
||||
|
|
|
@ -58,9 +58,29 @@ always available.
|
|||
A string containing the copyright pertaining to the Python interpreter.
|
||||
|
||||
|
||||
.. function:: _cleartypecache()
|
||||
.. function:: _compact_freelists()
|
||||
|
||||
Clear the internal type lookup cache.
|
||||
Compact the free lists of integers and floats by deallocating unused blocks.
|
||||
It can reduce the memory usage of the Python process several tenth of
|
||||
thousands of integers or floats have been allocated at once.
|
||||
|
||||
The return value is a tuple of tuples each containing three elements,
|
||||
amount of used objects, total block count before the blocks are deallocated
|
||||
and amount of freed blocks. The first tuple refers to ints, the second to
|
||||
floats.
|
||||
|
||||
This function should be used for specialized purposes only.
|
||||
|
||||
.. versionadded:: 2.6
|
||||
|
||||
|
||||
.. function:: _clear_type_cache()
|
||||
|
||||
Clear the internal type cache. The type cache is used to speed up attribute
|
||||
and method lookups. Use the function *only* to drop unnecessary references
|
||||
during reference leak debugging.
|
||||
|
||||
This function should be used for internal and specialized purposes only.
|
||||
|
||||
.. versionadded:: 2.6
|
||||
|
||||
|
|
|
@ -101,6 +101,8 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void);
|
|||
PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
|
||||
PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
|
||||
|
||||
/* free list api */
|
||||
PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -59,6 +59,9 @@ PyAPI_FUNC(long) PyInt_GetMax(void);
|
|||
PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int);
|
||||
PyAPI_FUNC(long) PyOS_strtol(char *, char **, int);
|
||||
|
||||
/* free list api */
|
||||
PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -710,7 +710,7 @@ def dash_R_cleanup(fs, ps, pic, abcs):
|
|||
sys.path_importer_cache.update(pic)
|
||||
|
||||
# clear type cache
|
||||
sys._cleartypecache()
|
||||
sys._clear_type_cache()
|
||||
|
||||
# Clear ABC registries, restoring previously saved ABC registries.
|
||||
for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:
|
||||
|
|
|
@ -363,6 +363,24 @@ class SysModuleTest(unittest.TestCase):
|
|||
self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
|
||||
self.assert_(repr(sys.flags))
|
||||
|
||||
def test_clear_type_cache(self):
|
||||
sys._clear_type_cache()
|
||||
|
||||
def test_compact_freelists(self):
|
||||
sys._compact_freelists()
|
||||
r = sys._compact_freelists()
|
||||
# freed blocks shouldn't change
|
||||
self.assertEqual(r[0][2], 0)
|
||||
self.assertEqual(r[1][2], 0)
|
||||
# fill freelists
|
||||
ints = list(range(12000))
|
||||
floats = [float(i) for i in ints]
|
||||
del ints
|
||||
del floats
|
||||
# should free more than 200 blocks each
|
||||
r = sys._compact_freelists()
|
||||
self.assert_(r[0][2] > 200, r[0][2])
|
||||
self.assert_(r[1][2] > 200, r[1][2])
|
||||
|
||||
def test_main():
|
||||
test.test_support.run_unittest(SysModuleTest)
|
||||
|
|
|
@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
|
|||
Core and builtins
|
||||
-----------------
|
||||
|
||||
- Patch #1953: Added ´´sys._compact_freelists()´´ and the C API functions
|
||||
´´PyInt_CompactFreeList´´ and ´´PyFloat_CompactFreeList´´
|
||||
to compact the internal free lists of pre-allocted ints and floats.
|
||||
|
||||
- Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls.
|
||||
Python expected the return type int but the fork familie returns pi_t.
|
||||
|
||||
|
@ -21,7 +25,7 @@ Core and builtins
|
|||
- Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and linebreak
|
||||
detection
|
||||
|
||||
- Added ``PyType_ClearCache()`` and ``sys._cleartypecache`` to clear the
|
||||
- Added ``PyType_ClearCache()`` and ``sys._clear_type_cache`` to clear the
|
||||
internal lookup cache for ref leak tests.
|
||||
|
||||
- Patch #1473257: generator objects gain a gi_code attribute. This is the
|
||||
|
|
|
@ -1609,17 +1609,15 @@ _PyFloat_Init(void)
|
|||
}
|
||||
|
||||
void
|
||||
PyFloat_Fini(void)
|
||||
PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
|
||||
{
|
||||
PyFloatObject *p;
|
||||
PyFloatBlock *list, *next;
|
||||
unsigned i;
|
||||
int bc, bf; /* block count, number of freed blocks */
|
||||
int frem, fsum; /* remaining unfreed floats per block, total */
|
||||
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
|
||||
size_t fsum = 0; /* total unfreed ints */
|
||||
int frem; /* remaining unfreed ints per block */
|
||||
|
||||
bc = 0;
|
||||
bf = 0;
|
||||
fsum = 0;
|
||||
list = block_list;
|
||||
block_list = NULL;
|
||||
free_list = NULL;
|
||||
|
@ -1654,6 +1652,22 @@ PyFloat_Fini(void)
|
|||
fsum += frem;
|
||||
list = next;
|
||||
}
|
||||
*pbc = bc;
|
||||
*pbf = bf;
|
||||
*bsum = fsum;
|
||||
}
|
||||
|
||||
void
|
||||
PyFloat_Fini(void)
|
||||
{
|
||||
PyFloatObject *p;
|
||||
PyFloatBlock *list;
|
||||
unsigned i;
|
||||
size_t bc, bf; /* block count, number of freed blocks */
|
||||
size_t fsum; /* total unfreed floats per block */
|
||||
|
||||
PyFloat_CompactFreeList(&bc, &bf, &fsum);
|
||||
|
||||
if (!Py_VerboseFlag)
|
||||
return;
|
||||
fprintf(stderr, "# cleanup floats");
|
||||
|
@ -1662,7 +1676,9 @@ PyFloat_Fini(void)
|
|||
}
|
||||
else {
|
||||
fprintf(stderr,
|
||||
": %d unfreed float%s in %d out of %d block%s\n",
|
||||
": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
|
||||
PY_FORMAT_SIZE_T "d out of %"
|
||||
PY_FORMAT_SIZE_T "d block%s\n",
|
||||
fsum, fsum == 1 ? "" : "s",
|
||||
bc - bf, bc, bc == 1 ? "" : "s");
|
||||
}
|
||||
|
|
|
@ -1202,28 +1202,15 @@ _PyInt_Init(void)
|
|||
}
|
||||
|
||||
void
|
||||
PyInt_Fini(void)
|
||||
PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
|
||||
{
|
||||
PyIntObject *p;
|
||||
PyIntBlock *list, *next;
|
||||
int i;
|
||||
unsigned int ctr;
|
||||
int bc, bf; /* block count, number of freed blocks */
|
||||
int irem, isum; /* remaining unfreed ints per block, total */
|
||||
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
|
||||
size_t isum = 0; /* total unfreed ints */
|
||||
int irem; /* remaining unfreed ints per block */
|
||||
|
||||
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
|
||||
PyIntObject **q;
|
||||
|
||||
i = NSMALLNEGINTS + NSMALLPOSINTS;
|
||||
q = small_ints;
|
||||
while (--i >= 0) {
|
||||
Py_XDECREF(*q);
|
||||
*q++ = NULL;
|
||||
}
|
||||
#endif
|
||||
bc = 0;
|
||||
bf = 0;
|
||||
isum = 0;
|
||||
list = block_list;
|
||||
block_list = NULL;
|
||||
free_list = NULL;
|
||||
|
@ -1268,6 +1255,33 @@ PyInt_Fini(void)
|
|||
isum += irem;
|
||||
list = next;
|
||||
}
|
||||
|
||||
*pbc = bc;
|
||||
*pbf = bf;
|
||||
*bsum = isum;
|
||||
}
|
||||
|
||||
void
|
||||
PyInt_Fini(void)
|
||||
{
|
||||
PyIntObject *p;
|
||||
PyIntBlock *list;
|
||||
unsigned int ctr;
|
||||
size_t bc, bf; /* block count, number of freed blocks */
|
||||
size_t isum; /* total unfreed ints per block */
|
||||
|
||||
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
|
||||
int i;
|
||||
PyIntObject **q;
|
||||
|
||||
i = NSMALLNEGINTS + NSMALLPOSINTS;
|
||||
q = small_ints;
|
||||
while (--i >= 0) {
|
||||
Py_XDECREF(*q);
|
||||
*q++ = NULL;
|
||||
}
|
||||
#endif
|
||||
PyInt_CompactFreeList(&bc, &bf, &isum);
|
||||
if (!Py_VerboseFlag)
|
||||
return;
|
||||
fprintf(stderr, "# cleanup ints");
|
||||
|
@ -1276,7 +1290,9 @@ PyInt_Fini(void)
|
|||
}
|
||||
else {
|
||||
fprintf(stderr,
|
||||
": %d unfreed int%s in %d out of %d block%s\n",
|
||||
": %" PY_FORMAT_SIZE_T "d unfreed ints%s in %"
|
||||
PY_FORMAT_SIZE_T "d out of %"
|
||||
PY_FORMAT_SIZE_T "d block%s\n",
|
||||
isum, isum == 1 ? "" : "s",
|
||||
bc - bf, bc, bc == 1 ? "" : "s");
|
||||
}
|
||||
|
|
|
@ -754,17 +754,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\
|
|||
10. Number of stack pops performed by call_function()"
|
||||
);
|
||||
|
||||
static PyObject *
|
||||
sys_cleartypecache(PyObject* self, PyObject* args)
|
||||
{
|
||||
PyType_ClearCache();
|
||||
Py_RETURN_NONE;
|
||||
}
|
||||
|
||||
PyDoc_STRVAR(cleartypecache_doc,
|
||||
"_cleartypecache() -> None\n\
|
||||
Clear the internal type lookup cache.");
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
@ -783,12 +772,44 @@ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *);
|
|||
}
|
||||
#endif
|
||||
|
||||
static PyObject *
|
||||
sys_clear_type_cache(PyObject* self, PyObject* args)
|
||||
{
|
||||
PyType_ClearCache();
|
||||
Py_RETURN_NONE;
|
||||
}
|
||||
|
||||
PyDoc_STRVAR(sys_clear_type_cache__doc__,
|
||||
"_clear_type_cache() -> None\n\
|
||||
Clear the internal type lookup cache.");
|
||||
|
||||
|
||||
static PyObject *
|
||||
sys_compact_freelists(PyObject* self, PyObject* args)
|
||||
{
|
||||
size_t isum, ibc, ibf;
|
||||
size_t fsum, fbc, fbf;
|
||||
|
||||
PyInt_CompactFreeList(&ibc, &ibf, &isum);
|
||||
PyFloat_CompactFreeList(&fbc, &fbf, &fsum);
|
||||
|
||||
return Py_BuildValue("(kkk)(kkk)", isum, ibc, ibf,
|
||||
fsum, fbc, fbf);
|
||||
|
||||
}
|
||||
|
||||
PyDoc_STRVAR(sys_compact_freelists__doc__,
|
||||
"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks), ...)\n\
|
||||
Compact the free lists of ints and floats.");
|
||||
|
||||
static PyMethodDef sys_methods[] = {
|
||||
/* Might as well keep this in alphabetic order */
|
||||
{"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS,
|
||||
callstats_doc},
|
||||
{"_cleartypecache", sys_cleartypecache, METH_NOARGS,
|
||||
cleartypecache_doc},
|
||||
{"_clear_type_cache", sys_clear_type_cache, METH_NOARGS,
|
||||
sys_clear_type_cache__doc__},
|
||||
{"_compact_freelists", sys_compact_freelists, METH_NOARGS,
|
||||
sys_compact_freelists__doc__},
|
||||
{"_current_frames", sys_current_frames, METH_NOARGS,
|
||||
current_frames_doc},
|
||||
{"displayhook", sys_displayhook, METH_O, displayhook_doc},
|
||||
|
|
Loading…
Reference in New Issue