Merged revisions 60481,60485,60489-60492,60494-60496,60498-60499,60501-60503,60505-60506,60508-60509,60523-60524,60532,60543,60545,60547-60548,60552-60567 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r60553 | neal.norwitz | 2008-02-03 17:53:09 +0100 (Sun, 03 Feb 2008) | 1 line

  Ignore leaky warnings from test_asynchat
........
  r60555 | christian.heimes | 2008-02-03 20:51:13 +0100 (Sun, 03 Feb 2008) | 1 line

  Another int -> pid_t case
........
  r60560 | amaury.forgeotdarc | 2008-02-03 23:51:43 +0100 (Sun, 03 Feb 2008) | 6 lines

  Ensure that PySet_Add() operates on a newly created frozenset, like PyTuple_SetItem does.

  Add PyFrozenSet_Check(), which was not needed before; The list of Py*Set_Check* macros seems to be complete now.

  Add missing NEWS entries about all this.
........
  r60563 | amaury.forgeotdarc | 2008-02-04 00:14:32 +0100 (Mon, 04 Feb 2008) | 2 lines

  Nasty typo in setobject.h
........
  r60564 | amaury.forgeotdarc | 2008-02-04 00:15:32 +0100 (Mon, 04 Feb 2008) | 3 lines

  Correct test_mailbox on win32: since the test sets a custom 'colon' attribute
  to the main mailbox, copy it to secondary mailbox instances.
........
  r60565 | amaury.forgeotdarc | 2008-02-04 00:57:24 +0100 (Mon, 04 Feb 2008) | 2 lines

  Let test_socketserver pass on win32, which does not have AF_UNIX sockets.
........
  r60566 | jeffrey.yasskin | 2008-02-04 02:04:35 +0100 (Mon, 04 Feb 2008) | 2 lines

  Make int() and long() fall back to __trunc__(). See issue 2002.
........
  r60567 | christian.heimes | 2008-02-04 19:00:12 +0100 (Mon, 04 Feb 2008) | 3 lines

  Patch #1953
  I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers.
  The patch also renames sys._cleartypecache to sys._clear_type_cache
........
This commit is contained in:
Christian Heimes 2008-02-04 18:48:49 +00:00
parent fdb6bb56c1
commit 15ebc88d87
18 changed files with 256 additions and 36 deletions

View File

@ -72,3 +72,9 @@ Floating Point Objects
.. cfunction:: double PyFloat_GetMin(void)
Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
Compact the float free list. *bc* is the number of allocated blocks before
blocks are freed, *bf* is the number of freed blocks and *sum* is the number
of remaining objects in the blocks.

View File

@ -56,6 +56,13 @@ the constructor functions work with any iterable Python object.
.. versionadded:: 2.6
.. cfunction:: int PyFrozenSet_Check(PyObject *p)
Return true if *p* is a :class:`frozenset` object or an instance of a
subtype.
.. versionadded:: 2.6
.. cfunction:: int PyAnySet_Check(PyObject *p)
Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an

View File

@ -54,9 +54,28 @@ always available.
A string containing the copyright pertaining to the Python interpreter.
.. function:: _cleartypecache()
.. function:: _compact_freelists()
Clear the internal type lookup cache.
Compact the free list of floats by deallocating unused blocks.
It can reduce the memory usage of the Python process several tenth of
thousands of integers or floats have been allocated at once.
The return value is a tuple of tuples each containing three elements,
amount of used objects, total block count before the blocks are deallocated
and amount of freed blocks.
This function should be used for specialized purposes only.
.. versionadded:: 2.6
.. function:: _clear_type_cache()
Clear the internal type cache. The type cache is used to speed up attribute
and method lookups. Use the function *only* to drop unnecessary references
during reference leak debugging.
This function should be used for internal and specialized purposes only.
.. versionadded:: 2.6

View File

@ -792,6 +792,19 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
PyAPI_FUNC(Py_ssize_t) PyNumber_AsSsize_t(PyObject *o, PyObject *exc);
/*
Returns the Integral instance converted to an int. The
instance is expected to be int or long or have an __int__
method. Steals integral's reference. error_format will be
used to create the TypeError if integral isn't actually an
Integral instance. error_format should be a format string
that can accept a char* naming integral's type.
*/
PyAPI_FUNC(PyObject *) _PyNumber_ConvertIntegralToInt(
PyObject *integral,
const char* error_format);
/*
Returns the object converted to Py_ssize_t by going through
PyNumber_Index first. If an overflow error occurs while

View File

@ -91,6 +91,9 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void);
PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
/* free list api */
PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
#ifdef __cplusplus
}
#endif

View File

@ -75,7 +75,11 @@ PyAPI_DATA(PyTypeObject) PySetIter_Type;
PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \
PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
#define PySet_Check(ob) \
(Py_TYPE(ob) == &PySet_Type || PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))
(Py_TYPE(ob) == &PySet_Type || \
PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))
#define PyFrozenSet_Check(ob) \
(Py_TYPE(ob) == &PyFrozenSet_Type || \
PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
PyAPI_FUNC(PyObject *) PySet_New(PyObject *);
PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *);

View File

@ -406,8 +406,6 @@ class Rational(RationalAbc):
else:
return a.numerator // a.denominator
__int__ = __trunc__
def __floor__(a):
"""Will be math.floor(a) in 3.0."""
return a.numerator // a.denominator

View File

@ -753,7 +753,7 @@ def dash_R_cleanup(fs, ps, pic, abcs):
sys.path_importer_cache.update(pic)
# clear type cache
sys._cleartypecache()
sys._clear_type_cache()
# Clear ABC registries, restoring previously saved ABC registries.
for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:

View File

@ -861,6 +861,14 @@ class BuiltinTest(unittest.TestCase):
def test_intconversion(self):
# Test __int__()
class ClassicMissingMethods:
pass
self.assertRaises(TypeError, int, ClassicMissingMethods())
class MissingMethods(object):
pass
self.assertRaises(TypeError, int, MissingMethods())
class Foo0:
def __int__(self):
return 42
@ -892,6 +900,49 @@ class BuiltinTest(unittest.TestCase):
self.assertEqual(int(Foo4()), 42)
self.assertRaises(TypeError, int, Foo5())
class Classic:
pass
for base in (object, Classic):
class IntOverridesTrunc(base):
def __int__(self):
return 42
def __trunc__(self):
return -12
self.assertEqual(int(IntOverridesTrunc()), 42)
class JustTrunc(base):
def __trunc__(self):
return 42
self.assertEqual(int(JustTrunc()), 42)
for trunc_result_base in (object, Classic):
class Integral(trunc_result_base):
def __int__(self):
return 42
class TruncReturnsNonInt(base):
def __trunc__(self):
return Integral()
self.assertEqual(int(TruncReturnsNonInt()), 42)
class NonIntegral(trunc_result_base):
def __trunc__(self):
# Check that we avoid infinite recursion.
return NonIntegral()
class TruncReturnsNonIntegral(base):
def __trunc__(self):
return NonIntegral()
try:
int(TruncReturnsNonIntegral())
except TypeError as e:
self.assertEquals(str(e),
"__trunc__ returned non-Integral"
" (type NonIntegral)")
else:
self.fail("Failed to raise TypeError with %s" %
((base, trunc_result_base),))
def test_iter(self):
self.assertRaises(TypeError, iter)
self.assertRaises(TypeError, iter, 42, 42)
@ -1095,7 +1146,6 @@ class BuiltinTest(unittest.TestCase):
self.assertEqual(int('2br45qc', 35), 4294967297)
self.assertEqual(int('1z141z5', 36), 4294967297)
def test_longconversion(self):
# Test __long__()
class Foo0:

View File

@ -517,6 +517,7 @@ class TestMaildir(TestMailbox):
class FakeMessage(mailbox.MaildirMessage):
pass
box = mailbox.Maildir(self._path, factory=FakeMessage)
box.colon = self._box.colon
msg2 = box.get_message(key)
self.assert_(isinstance(msg2, FakeMessage))

View File

@ -51,13 +51,14 @@ class MyDatagramHandler(MyMixinHandler,
SocketServer.DatagramRequestHandler):
pass
class ForkingUnixStreamServer(SocketServer.ForkingMixIn,
SocketServer.UnixStreamServer):
pass
if HAVE_UNIX_SOCKETS:
class ForkingUnixStreamServer(SocketServer.ForkingMixIn,
SocketServer.UnixStreamServer):
pass
class ForkingUnixDatagramServer(SocketServer.ForkingMixIn,
SocketServer.UnixDatagramServer):
pass
class ForkingUnixDatagramServer(SocketServer.ForkingMixIn,
SocketServer.UnixDatagramServer):
pass
class MyMixinServer:

View File

@ -330,6 +330,20 @@ class SysModuleTest(unittest.TestCase):
self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
self.assert_(repr(sys.flags))
def test_clear_type_cache(self):
sys._clear_type_cache()
def test_compact_freelists(self):
sys._compact_freelists()
r = sys._compact_freelists()
# freed blocks shouldn't change
self.assertEqual(r[0][2], 0)
# fill freelists
floats = [float(i) for i in range(12000)]
del floats
# should free more than 200 blocks
r = sys._compact_freelists()
self.assert_(r[0][2] > 200, r[0][2])
def test_main():
test.test_support.run_unittest(SysModuleTest)

View File

@ -67,7 +67,7 @@ REFLOG="build/reflog.txt.out"
# Note: test_XXX (none currently) really leak, but are disabled
# so we don't send spam. Any test which really leaks should only
# be listed here if there are also test cases under Lib/test/leakers.
LEAKY_TESTS="test_(cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)"
LEAKY_TESTS="test_(asynchat|cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)"
# These tests always fail, so skip them so we don't get false positives.
_ALWAYS_SKIP=""

View File

@ -4626,7 +4626,8 @@ Return the process group associated with the terminal given by a fd.");
static PyObject *
posix_tcgetpgrp(PyObject *self, PyObject *args)
{
int fd, pgid;
int fd;
pid_t pgid;
if (!PyArg_ParseTuple(args, "i:tcgetpgrp", &fd))
return NULL;
pgid = tcgetpgrp(fd);

View File

@ -1242,6 +1242,40 @@ PyNumber_AsSsize_t(PyObject *item, PyObject *err)
}
PyObject *
_PyNumber_ConvertIntegralToInt(PyObject *integral, const char* error_format)
{
static PyObject *int_name = NULL;
if (int_name == NULL) {
int_name = PyUnicode_InternFromString("__int__");
if (int_name == NULL)
return NULL;
}
if (integral && !PyLong_Check(integral)) {
/* Don't go through tp_as_number->nb_int to avoid
hitting the classic class fallback to __trunc__. */
PyObject *int_func = PyObject_GetAttr(integral, int_name);
if (int_func == NULL) {
PyErr_Clear(); /* Raise a different error. */
goto non_integral_error;
}
Py_DECREF(integral);
integral = PyEval_CallObject(int_func, NULL);
Py_DECREF(int_func);
if (integral && !PyLong_Check(integral)) {
goto non_integral_error;
}
}
return integral;
non_integral_error:
PyErr_Format(PyExc_TypeError, error_format, Py_TYPE(integral)->tp_name);
Py_DECREF(integral);
return NULL;
}
/* Add a check for embedded NULL-bytes in the argument. */
static PyObject *
long_from_string(const char *s, Py_ssize_t len)
@ -1265,9 +1299,17 @@ PyObject *
PyNumber_Long(PyObject *o)
{
PyNumberMethods *m;
static PyObject *trunc_name = NULL;
PyObject *trunc_func;
const char *buffer;
Py_ssize_t buffer_len;
if (trunc_name == NULL) {
trunc_name = PyUnicode_InternFromString("__trunc__");
if (trunc_name == NULL)
return NULL;
}
if (o == NULL)
return null_error();
if (PyLong_CheckExact(o)) {
@ -1287,6 +1329,7 @@ PyNumber_Long(PyObject *o)
return res;
}
if (m && m->nb_long) { /* This should include subclasses of long */
/* Classic classes always take this branch. */
PyObject *res = m->nb_long(o);
if (res && !PyLong_Check(res)) {
PyErr_Format(PyExc_TypeError,
@ -1299,6 +1342,27 @@ PyNumber_Long(PyObject *o)
}
if (PyLong_Check(o)) /* A long subclass without nb_long */
return _PyLong_Copy((PyLongObject *)o);
trunc_func = PyObject_GetAttr(o, trunc_name);
if (trunc_func) {
PyObject *truncated = PyEval_CallObject(trunc_func, NULL);
PyObject *int_instance;
Py_DECREF(trunc_func);
/* __trunc__ is specified to return an Integral type,
but long() needs to return a long. */
int_instance = _PyNumber_ConvertIntegralToInt(
truncated,
"__trunc__ returned non-Integral (type %.200s)");
return int_instance;
}
PyErr_Clear(); /* It's not an error if o.__trunc__ doesn't exist. */
if (PyString_Check(o))
/* need to do extra error checking that PyLong_FromString()
* doesn't do. In particular long('9.5') must raise an
* exception, not truncate the float.
*/
return long_from_string(PyString_AS_STRING(o),
PyString_GET_SIZE(o));
if (PyUnicode_Check(o))
/* The above check is done in PyLong_FromUnicode(). */
return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),

View File

@ -1540,17 +1540,15 @@ _PyFloat_Init(void)
}
void
PyFloat_Fini(void)
PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
{
PyFloatObject *p;
PyFloatBlock *list, *next;
unsigned i;
int bc, bf; /* block count, number of freed blocks */
int frem, fsum; /* remaining unfreed floats per block, total */
size_t bc = 0, bf = 0; /* block count, number of freed blocks */
size_t fsum = 0; /* total unfreed ints */
int frem; /* remaining unfreed ints per block */
bc = 0;
bf = 0;
fsum = 0;
list = block_list;
block_list = NULL;
free_list = NULL;
@ -1585,6 +1583,22 @@ PyFloat_Fini(void)
fsum += frem;
list = next;
}
*pbc = bc;
*pbf = bf;
*bsum = fsum;
}
void
PyFloat_Fini(void)
{
PyFloatObject *p;
PyFloatBlock *list;
unsigned i;
size_t bc, bf; /* block count, number of freed blocks */
size_t fsum; /* total unfreed floats per block */
PyFloat_CompactFreeList(&bc, &bf, &fsum);
if (!Py_VerboseFlag)
return;
fprintf(stderr, "# cleanup floats");
@ -1593,7 +1607,9 @@ PyFloat_Fini(void)
}
else {
fprintf(stderr,
": %d unfreed float%s in %d out of %d block%s\n",
": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
PY_FORMAT_SIZE_T "d out of %"
PY_FORMAT_SIZE_T "d block%s\n",
fsum, fsum == 1 ? "" : "s",
bc - bf, bc, bc == 1 ? "" : "s");
}

View File

@ -2173,7 +2173,8 @@ PySet_Discard(PyObject *set, PyObject *key)
int
PySet_Add(PyObject *anyset, PyObject *key)
{
if (!PyAnySet_Check(anyset)) {
if (!PySet_Check(anyset) &&
(!PyFrozenSet_Check(anyset) || Py_REFCNT(anyset) != 1)) {
PyErr_BadInternalCall();
return -1;
}
@ -2277,6 +2278,10 @@ test_c_api(PySetObject *so)
f = PyFrozenSet_New(dup);
assertRaises(PySet_Clear(f) == -1, PyExc_SystemError);
assertRaises(_PySet_Update(f, dup) == -1, PyExc_SystemError);
assert(PySet_Add(f, elem) == 0);
Py_INCREF(f);
assertRaises(PySet_Add(f, elem) == -1, PyExc_SystemError);
Py_DECREF(f);
Py_DECREF(f);
/* Exercise direct iteration */

View File

@ -730,17 +730,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\
10. Number of stack pops performed by call_function()"
);
static PyObject *
sys_cleartypecache(PyObject* self, PyObject* args)
{
PyType_ClearCache();
Py_RETURN_NONE;
}
PyDoc_STRVAR(cleartypecache_doc,
"_cleartypecache() -> None\n\
Clear the internal type lookup cache.");
#ifdef __cplusplus
extern "C" {
#endif
@ -759,12 +748,41 @@ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *);
}
#endif
static PyObject *
sys_clear_type_cache(PyObject* self, PyObject* args)
{
PyType_ClearCache();
Py_RETURN_NONE;
}
PyDoc_STRVAR(sys_clear_type_cache__doc__,
"_clear_type_cache() -> None\n\
Clear the internal type lookup cache.");
static PyObject *
sys_compact_freelists(PyObject* self, PyObject* args)
{
size_t fsum, fbc, fbf;
PyFloat_CompactFreeList(&fbc, &fbf, &fsum);
return Py_BuildValue("((kkk))", fsum, fbc, fbf);
}
PyDoc_STRVAR(sys_compact_freelists__doc__,
"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks),)\n\
Compact the free lists of floats.");
static PyMethodDef sys_methods[] = {
/* Might as well keep this in alphabetic order */
{"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS,
callstats_doc},
{"_cleartypecache", sys_cleartypecache, METH_NOARGS,
cleartypecache_doc},
{"_clear_type_cache", sys_clear_type_cache, METH_NOARGS,
sys_clear_type_cache__doc__},
{"_compact_freelists", sys_compact_freelists, METH_NOARGS,
sys_compact_freelists__doc__},
{"_current_frames", sys_current_frames, METH_NOARGS,
current_frames_doc},
{"displayhook", sys_displayhook, METH_O, displayhook_doc},