I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers.
The patch also renames sys._cleartypecache to sys._clear_type_cache
This commit is contained in:
Christian Heimes 2008-02-04 18:00:12 +00:00
parent a26cf9b760
commit 422051a367
11 changed files with 160 additions and 42 deletions

View File

@ -84,3 +84,12 @@ Floating Point Objects
Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
.. versionadded:: 2.6
.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
Compact the float free list. *bc* is the number of allocated blocks before
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
of remaining objects in the blocks.
.. versionadded:: 2.6

View File

@ -120,3 +120,12 @@ Plain Integer Objects
Return the system's idea of the largest integer it can handle
(:const:`LONG_MAX`, as defined in the system header files).
.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
Compact the integer free list. *bc* is the number of allocated blocks before
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
of remaining objects in the blocks.
.. versionadded:: 2.6

View File

@ -58,9 +58,29 @@ always available.
A string containing the copyright pertaining to the Python interpreter.
.. function:: _cleartypecache()
.. function:: _compact_freelists()
Clear the internal type lookup cache.
Compact the free lists of integers and floats by deallocating unused blocks.
It can reduce the memory usage of the Python process several tenth of
thousands of integers or floats have been allocated at once.
The return value is a tuple of tuples each containing three elements,
amount of used objects, total block count before the blocks are deallocated
and amount of freed blocks. The first tuple refers to ints, the second to
floats.
This function should be used for specialized purposes only.
.. versionadded:: 2.6
.. function:: _clear_type_cache()
Clear the internal type cache. The type cache is used to speed up attribute
and method lookups. Use the function *only* to drop unnecessary references
during reference leak debugging.
This function should be used for internal and specialized purposes only.
.. versionadded:: 2.6

View File

@ -101,6 +101,8 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void);
PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
/* free list api */
PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
#ifdef __cplusplus
}

View File

@ -59,6 +59,9 @@ PyAPI_FUNC(long) PyInt_GetMax(void);
PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int);
PyAPI_FUNC(long) PyOS_strtol(char *, char **, int);
/* free list api */
PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *);
#ifdef __cplusplus
}
#endif

View File

@ -710,7 +710,7 @@ def dash_R_cleanup(fs, ps, pic, abcs):
sys.path_importer_cache.update(pic)
# clear type cache
sys._cleartypecache()
sys._clear_type_cache()
# Clear ABC registries, restoring previously saved ABC registries.
for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:

View File

@ -363,6 +363,24 @@ class SysModuleTest(unittest.TestCase):
self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
self.assert_(repr(sys.flags))
def test_clear_type_cache(self):
sys._clear_type_cache()
def test_compact_freelists(self):
sys._compact_freelists()
r = sys._compact_freelists()
# freed blocks shouldn't change
self.assertEqual(r[0][2], 0)
self.assertEqual(r[1][2], 0)
# fill freelists
ints = list(range(12000))
floats = [float(i) for i in ints]
del ints
del floats
# should free more than 200 blocks each
r = sys._compact_freelists()
self.assert_(r[0][2] > 200, r[0][2])
self.assert_(r[1][2] > 200, r[1][2])
def test_main():
test.test_support.run_unittest(SysModuleTest)

View File

@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
Core and builtins
-----------------
- Patch #1953: Added ´´sys._compact_freelists()´´ and the C API functions
´´PyInt_CompactFreeList´´ and ´´PyFloat_CompactFreeList´´
to compact the internal free lists of pre-allocted ints and floats.
- Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls.
Python expected the return type int but the fork familie returns pi_t.
@ -21,7 +25,7 @@ Core and builtins
- Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and linebreak
detection
- Added ``PyType_ClearCache()`` and ``sys._cleartypecache`` to clear the
- Added ``PyType_ClearCache()`` and ``sys._clear_type_cache`` to clear the
internal lookup cache for ref leak tests.
- Patch #1473257: generator objects gain a gi_code attribute. This is the

View File

@ -1609,17 +1609,15 @@ _PyFloat_Init(void)
}
void
PyFloat_Fini(void)
PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
{
PyFloatObject *p;
PyFloatBlock *list, *next;
unsigned i;
int bc, bf; /* block count, number of freed blocks */
int frem, fsum; /* remaining unfreed floats per block, total */
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
size_t fsum = 0; /* total unfreed ints */
int frem; /* remaining unfreed ints per block */
bc = 0;
bf = 0;
fsum = 0;
list = block_list;
block_list = NULL;
free_list = NULL;
@ -1654,6 +1652,22 @@ PyFloat_Fini(void)
fsum += frem;
list = next;
}
*pbc = bc;
*pbf = bf;
*bsum = fsum;
}
void
PyFloat_Fini(void)
{
PyFloatObject *p;
PyFloatBlock *list;
unsigned i;
size_t bc, bf; /* block count, number of freed blocks */
size_t fsum; /* total unfreed floats per block */
PyFloat_CompactFreeList(&bc, &bf, &fsum);
if (!Py_VerboseFlag)
return;
fprintf(stderr, "# cleanup floats");
@ -1662,7 +1676,9 @@ PyFloat_Fini(void)
}
else {
fprintf(stderr,
": %d unfreed float%s in %d out of %d block%s\n",
": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
PY_FORMAT_SIZE_T "d out of %"
PY_FORMAT_SIZE_T "d block%s\n",
fsum, fsum == 1 ? "" : "s",
bc - bf, bc, bc == 1 ? "" : "s");
}

View File

@ -1202,28 +1202,15 @@ _PyInt_Init(void)
}
void
PyInt_Fini(void)
PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
{
PyIntObject *p;
PyIntBlock *list, *next;
int i;
unsigned int ctr;
int bc, bf; /* block count, number of freed blocks */
int irem, isum; /* remaining unfreed ints per block, total */
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
size_t isum = 0; /* total unfreed ints */
int irem; /* remaining unfreed ints per block */
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
PyIntObject **q;
i = NSMALLNEGINTS + NSMALLPOSINTS;
q = small_ints;
while (--i >= 0) {
Py_XDECREF(*q);
*q++ = NULL;
}
#endif
bc = 0;
bf = 0;
isum = 0;
list = block_list;
block_list = NULL;
free_list = NULL;
@ -1268,6 +1255,33 @@ PyInt_Fini(void)
isum += irem;
list = next;
}
*pbc = bc;
*pbf = bf;
*bsum = isum;
}
void
PyInt_Fini(void)
{
PyIntObject *p;
PyIntBlock *list;
unsigned int ctr;
size_t bc, bf; /* block count, number of freed blocks */
size_t isum; /* total unfreed ints per block */
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
int i;
PyIntObject **q;
i = NSMALLNEGINTS + NSMALLPOSINTS;
q = small_ints;
while (--i >= 0) {
Py_XDECREF(*q);
*q++ = NULL;
}
#endif
PyInt_CompactFreeList(&bc, &bf, &isum);
if (!Py_VerboseFlag)
return;
fprintf(stderr, "# cleanup ints");
@ -1276,7 +1290,9 @@ PyInt_Fini(void)
}
else {
fprintf(stderr,
": %d unfreed int%s in %d out of %d block%s\n",
": %" PY_FORMAT_SIZE_T "d unfreed ints%s in %"
PY_FORMAT_SIZE_T "d out of %"
PY_FORMAT_SIZE_T "d block%s\n",
isum, isum == 1 ? "" : "s",
bc - bf, bc, bc == 1 ? "" : "s");
}

View File

@ -754,17 +754,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\
10. Number of stack pops performed by call_function()"
);
static PyObject *
sys_cleartypecache(PyObject* self, PyObject* args)
{
PyType_ClearCache();
Py_RETURN_NONE;
}
PyDoc_STRVAR(cleartypecache_doc,
"_cleartypecache() -> None\n\
Clear the internal type lookup cache.");
#ifdef __cplusplus
extern "C" {
#endif
@ -783,12 +772,44 @@ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *);
}
#endif
static PyObject *
sys_clear_type_cache(PyObject* self, PyObject* args)
{
PyType_ClearCache();
Py_RETURN_NONE;
}
PyDoc_STRVAR(sys_clear_type_cache__doc__,
"_clear_type_cache() -> None\n\
Clear the internal type lookup cache.");
static PyObject *
sys_compact_freelists(PyObject* self, PyObject* args)
{
size_t isum, ibc, ibf;
size_t fsum, fbc, fbf;
PyInt_CompactFreeList(&ibc, &ibf, &isum);
PyFloat_CompactFreeList(&fbc, &fbf, &fsum);
return Py_BuildValue("(kkk)(kkk)", isum, ibc, ibf,
fsum, fbc, fbf);
}
PyDoc_STRVAR(sys_compact_freelists__doc__,
"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks), ...)\n\
Compact the free lists of ints and floats.");
static PyMethodDef sys_methods[] = {
/* Might as well keep this in alphabetic order */
{"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS,
callstats_doc},
{"_cleartypecache", sys_cleartypecache, METH_NOARGS,
cleartypecache_doc},
{"_clear_type_cache", sys_clear_type_cache, METH_NOARGS,
sys_clear_type_cache__doc__},
{"_compact_freelists", sys_compact_freelists, METH_NOARGS,
sys_compact_freelists__doc__},
{"_current_frames", sys_current_frames, METH_NOARGS,
current_frames_doc},
{"displayhook", sys_displayhook, METH_O, displayhook_doc},