Merged revisions 60481,60485,60489-60492,60494-60496,60498-60499,60501-60503,60505-60506,60508-60509,60523-60524,60532,60543,60545,60547-60548,60552-60567 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r60553 | neal.norwitz | 2008-02-03 17:53:09 +0100 (Sun, 03 Feb 2008) | 1 line

  Ignore leaky warnings from test_asynchat
........
  r60555 | christian.heimes | 2008-02-03 20:51:13 +0100 (Sun, 03 Feb 2008) | 1 line

  Another int -> pid_t case
........
  r60560 | amaury.forgeotdarc | 2008-02-03 23:51:43 +0100 (Sun, 03 Feb 2008) | 6 lines

  Ensure that PySet_Add() operates on a newly created frozenset, like PyTuple_SetItem does.

  Add PyFrozenSet_Check(), which was not needed before; The list of Py*Set_Check* macros seems to be complete now.

  Add missing NEWS entries about all this.
........
  r60563 | amaury.forgeotdarc | 2008-02-04 00:14:32 +0100 (Mon, 04 Feb 2008) | 2 lines

  Nasty typo in setobject.h
........
  r60564 | amaury.forgeotdarc | 2008-02-04 00:15:32 +0100 (Mon, 04 Feb 2008) | 3 lines

  Correct test_mailbox on win32: since the test sets a custom 'colon' attribute
  to the main mailbox, copy it to secondary mailbox instances.
........
  r60565 | amaury.forgeotdarc | 2008-02-04 00:57:24 +0100 (Mon, 04 Feb 2008) | 2 lines

  Let test_socketserver pass on win32, which does not have AF_UNIX sockets.
........
  r60566 | jeffrey.yasskin | 2008-02-04 02:04:35 +0100 (Mon, 04 Feb 2008) | 2 lines

  Make int() and long() fall back to __trunc__(). See issue 2002.
........
  r60567 | christian.heimes | 2008-02-04 19:00:12 +0100 (Mon, 04 Feb 2008) | 3 lines

  Patch #1953
  I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers.
  The patch also renames sys._cleartypecache to sys._clear_type_cache
........

Doc/c-api/float.rst

@@ -72,3 +72,9 @@ Floating Point Objects
 .. cfunction:: double PyFloat_GetMin(void)
 
    Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
+
+.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
+
+   Compact the float free list. *bc* is the number of allocated blocks before
+   blocks are freed, *bf* is the number of freed blocks and *sum* is the number
+   of remaining objects in the blocks.

Doc/c-api/set.rst

@@ -56,6 +56,13 @@ the constructor functions work with any iterable Python object.
 
    .. versionadded:: 2.6
 
+.. cfunction:: int PyFrozenSet_Check(PyObject *p)
+
+   Return true if *p* is a :class:`frozenset` object or an instance of a
+   subtype.
+
+   .. versionadded:: 2.6
+
 .. cfunction:: int PyAnySet_Check(PyObject *p)
 
    Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an

Doc/library/sys.rst

@@ -54,9 +54,28 @@ always available.
    A string containing the copyright pertaining to the Python interpreter.
 
 
-.. function:: _cleartypecache()
+.. function:: _compact_freelists()
 
-   Clear the internal type lookup cache.
+   Compact the free list of floats by deallocating unused blocks.
+   It can reduce the memory usage of the Python process several tenth of
+   thousands of integers or floats have been allocated at once.
+
+   The return value is a tuple of tuples each containing three elements,
+   amount of used objects, total block count before the blocks are deallocated
+   and amount of freed blocks. 
+
+   This function should be used for specialized purposes only.
+
+   .. versionadded:: 2.6
+
+
+.. function:: _clear_type_cache()
+
+   Clear the internal type cache. The type cache is used to speed up attribute
+   and method lookups. Use the function *only* to drop unnecessary references
+   during reference leak debugging.
+
+   This function should be used for internal and specialized purposes only.
 
    .. versionadded:: 2.6
 

Include/abstract.h

@@ -792,6 +792,19 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 
      PyAPI_FUNC(Py_ssize_t) PyNumber_AsSsize_t(PyObject *o, PyObject *exc);
 
+       /*
+         Returns the Integral instance converted to an int. The
+         instance is expected to be int or long or have an __int__
+         method. Steals integral's reference. error_format will be
+         used to create the TypeError if integral isn't actually an
+         Integral instance. error_format should be a format string
+         that can accept a char* naming integral's type.
+       */
+
+     PyAPI_FUNC(PyObject *) _PyNumber_ConvertIntegralToInt(
+             PyObject *integral,
+             const char* error_format);
+
        /*
         Returns the object converted to Py_ssize_t by going through
         PyNumber_Index first.  If an overflow error occurs while

Include/floatobject.h

@@ -91,6 +91,9 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void);
 PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
 PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
 
+/* free list api */
+PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
+
 #ifdef __cplusplus
 }
 #endif

Include/setobject.h

@@ -75,7 +75,11 @@ PyAPI_DATA(PyTypeObject) PySetIter_Type;
 	  PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \
 	  PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
 #define PySet_Check(ob) \
-	(Py_TYPE(ob) == &PySet_Type || PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))
+	(Py_TYPE(ob) == &PySet_Type || \
+	PyType_IsSubtype(Py_TYPE(ob), &PySet_Type))
+#define   PyFrozenSet_Check(ob) \
+	(Py_TYPE(ob) == &PyFrozenSet_Type || \
+	  PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type))
 
 PyAPI_FUNC(PyObject *) PySet_New(PyObject *);
 PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *);

Lib/rational.py

@@ -406,8 +406,6 @@ class Rational(RationalAbc):
         else:
             return a.numerator // a.denominator
 
-    __int__ = __trunc__
-
     def __floor__(a):
         """Will be math.floor(a) in 3.0."""
         return a.numerator // a.denominator

Lib/test/regrtest.py

@@ -753,7 +753,7 @@ def dash_R_cleanup(fs, ps, pic, abcs):
     sys.path_importer_cache.update(pic)
 
     # clear type cache
-    sys._cleartypecache()
+    sys._clear_type_cache()
 
     # Clear ABC registries, restoring previously saved ABC registries.
     for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:

Lib/test/test_builtin.py

@@ -861,6 +861,14 @@ class BuiltinTest(unittest.TestCase):
 
     def test_intconversion(self):
         # Test __int__()
+        class ClassicMissingMethods:
+            pass
+        self.assertRaises(TypeError, int, ClassicMissingMethods())
+
+        class MissingMethods(object):
+            pass
+        self.assertRaises(TypeError, int, MissingMethods())
+
         class Foo0:
             def __int__(self):
                 return 42
@@ -892,6 +900,49 @@ class BuiltinTest(unittest.TestCase):
         self.assertEqual(int(Foo4()), 42)
         self.assertRaises(TypeError, int, Foo5())
 
+        class Classic:
+            pass
+        for base in (object, Classic):
+            class IntOverridesTrunc(base):
+                def __int__(self):
+                    return 42
+                def __trunc__(self):
+                    return -12
+            self.assertEqual(int(IntOverridesTrunc()), 42)
+
+            class JustTrunc(base):
+                def __trunc__(self):
+                    return 42
+            self.assertEqual(int(JustTrunc()), 42)
+
+            for trunc_result_base in (object, Classic):
+                class Integral(trunc_result_base):
+                    def __int__(self):
+                        return 42
+
+                class TruncReturnsNonInt(base):
+                    def __trunc__(self):
+                        return Integral()
+                self.assertEqual(int(TruncReturnsNonInt()), 42)
+
+                class NonIntegral(trunc_result_base):
+                    def __trunc__(self):
+                        # Check that we avoid infinite recursion.
+                        return NonIntegral()
+
+                class TruncReturnsNonIntegral(base):
+                    def __trunc__(self):
+                        return NonIntegral()
+                try:
+                    int(TruncReturnsNonIntegral())
+                except TypeError as e:
+                    self.assertEquals(str(e),
+                                      "__trunc__ returned non-Integral"
+                                      " (type NonIntegral)")
+                else:
+                    self.fail("Failed to raise TypeError with %s" %
+                              ((base, trunc_result_base),))
+
     def test_iter(self):
         self.assertRaises(TypeError, iter)
         self.assertRaises(TypeError, iter, 42, 42)
@@ -1095,7 +1146,6 @@ class BuiltinTest(unittest.TestCase):
         self.assertEqual(int('2br45qc', 35), 4294967297)
         self.assertEqual(int('1z141z5', 36), 4294967297)
 
-
     def test_longconversion(self):
         # Test __long__()
         class Foo0:

Lib/test/test_mailbox.py

@@ -517,6 +517,7 @@ class TestMaildir(TestMailbox):
         class FakeMessage(mailbox.MaildirMessage):
             pass
         box = mailbox.Maildir(self._path, factory=FakeMessage)
+        box.colon = self._box.colon
         msg2 = box.get_message(key)
         self.assert_(isinstance(msg2, FakeMessage))
 

Lib/test/test_socketserver.py

@@ -51,13 +51,14 @@ class MyDatagramHandler(MyMixinHandler,
     SocketServer.DatagramRequestHandler):
     pass
 
-class ForkingUnixStreamServer(SocketServer.ForkingMixIn,
+if HAVE_UNIX_SOCKETS:
-                              SocketServer.UnixStreamServer):
+    class ForkingUnixStreamServer(SocketServer.ForkingMixIn,
-    pass
+                                  SocketServer.UnixStreamServer):
+        pass
 
-class ForkingUnixDatagramServer(SocketServer.ForkingMixIn,
+    class ForkingUnixDatagramServer(SocketServer.ForkingMixIn,
-                                SocketServer.UnixDatagramServer):
+                                    SocketServer.UnixDatagramServer):
-    pass
+        pass
 
 
 class MyMixinServer:

Lib/test/test_sys.py

@@ -330,6 +330,20 @@ class SysModuleTest(unittest.TestCase):
             self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
         self.assert_(repr(sys.flags))
 
+    def test_clear_type_cache(self):
+        sys._clear_type_cache()
+
+    def test_compact_freelists(self):
+        sys._compact_freelists()
+        r = sys._compact_freelists()
+        # freed blocks shouldn't change
+        self.assertEqual(r[0][2], 0)
+        # fill freelists
+        floats = [float(i) for i in range(12000)]
+        del floats
+        # should free more than 200 blocks
+        r = sys._compact_freelists()
+        self.assert_(r[0][2] > 200, r[0][2])
 
 def test_main():
     test.test_support.run_unittest(SysModuleTest)

Misc/build.sh

@@ -67,7 +67,7 @@ REFLOG="build/reflog.txt.out"
 # Note: test_XXX (none currently) really leak, but are disabled
 # so we don't send spam.  Any test which really leaks should only 
 # be listed here if there are also test cases under Lib/test/leakers.
-LEAKY_TESTS="test_(cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)"
+LEAKY_TESTS="test_(asynchat|cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)"
 
 # These tests always fail, so skip them so we don't get false positives.
 _ALWAYS_SKIP=""

Modules/posixmodule.c

@@ -4626,7 +4626,8 @@ Return the process group associated with the terminal given by a fd.");
 static PyObject *
 posix_tcgetpgrp(PyObject *self, PyObject *args)
 {
-	int fd, pgid;
+	int fd;
+	pid_t pgid;
 	if (!PyArg_ParseTuple(args, "i:tcgetpgrp", &fd))
 		return NULL;
 	pgid = tcgetpgrp(fd);

Objects/abstract.c

@@ -1242,6 +1242,40 @@ PyNumber_AsSsize_t(PyObject *item, PyObject *err)
 }
 
 
+PyObject *
+_PyNumber_ConvertIntegralToInt(PyObject *integral, const char* error_format)
+{
+	static PyObject *int_name = NULL;
+	if (int_name == NULL) {
+		int_name = PyUnicode_InternFromString("__int__");
+		if (int_name == NULL)
+			return NULL;
+	}
+
+	if (integral && !PyLong_Check(integral)) { 
+		/* Don't go through tp_as_number->nb_int to avoid
+		   hitting the classic class fallback to __trunc__. */
+		PyObject *int_func = PyObject_GetAttr(integral, int_name);
+		if (int_func == NULL) {
+			PyErr_Clear(); /* Raise a different error. */
+			goto non_integral_error;
+		}
+		Py_DECREF(integral);
+		integral = PyEval_CallObject(int_func, NULL);
+		Py_DECREF(int_func);
+		if (integral && !PyLong_Check(integral)) { 
+			goto non_integral_error;
+		}
+	}
+	return integral;
+
+non_integral_error:
+	PyErr_Format(PyExc_TypeError, error_format, Py_TYPE(integral)->tp_name);
+	Py_DECREF(integral);
+	return NULL;
+}
+
+
 /* Add a check for embedded NULL-bytes in the argument. */
 static PyObject *
 long_from_string(const char *s, Py_ssize_t len)
@@ -1265,9 +1299,17 @@ PyObject *
 PyNumber_Long(PyObject *o)
 {
 	PyNumberMethods *m;
+	static PyObject *trunc_name = NULL;
+	PyObject *trunc_func;
 	const char *buffer;
 	Py_ssize_t buffer_len;
 
+	if (trunc_name == NULL) {
+		trunc_name = PyUnicode_InternFromString("__trunc__");
+		if (trunc_name == NULL)
+			return NULL;
+	}
+
 	if (o == NULL)
 		return null_error();
 	if (PyLong_CheckExact(o)) {
@@ -1287,6 +1329,7 @@ PyNumber_Long(PyObject *o)
 		return res;
 	}
 	if (m && m->nb_long) { /* This should include subclasses of long */
+		/* Classic classes always take this branch. */
 		PyObject *res = m->nb_long(o);
 		if (res && !PyLong_Check(res)) {
 			PyErr_Format(PyExc_TypeError,
@@ -1299,6 +1342,27 @@ PyNumber_Long(PyObject *o)
 	}
 	if (PyLong_Check(o)) /* A long subclass without nb_long */
 		return _PyLong_Copy((PyLongObject *)o);
+	trunc_func = PyObject_GetAttr(o, trunc_name);
+	if (trunc_func) {
+		PyObject *truncated = PyEval_CallObject(trunc_func, NULL);
+		PyObject *int_instance;
+		Py_DECREF(trunc_func);
+		/* __trunc__ is specified to return an Integral type,
+		   but long() needs to return a long. */
+		int_instance = _PyNumber_ConvertIntegralToInt(
+			truncated,
+			"__trunc__ returned non-Integral (type %.200s)");
+		return int_instance;
+	}
+	PyErr_Clear();  /* It's not an error if  o.__trunc__ doesn't exist. */
+
+	if (PyString_Check(o))
+		/* need to do extra error checking that PyLong_FromString()
+		 * doesn't do.  In particular long('9.5') must raise an
+		 * exception, not truncate the float.
+		 */
+		return long_from_string(PyString_AS_STRING(o),
+					PyString_GET_SIZE(o));
 	if (PyUnicode_Check(o))
 		/* The above check is done in PyLong_FromUnicode(). */
 		return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),

Objects/floatobject.c

@@ -1540,17 +1540,15 @@ _PyFloat_Init(void)
 }
 
 void
-PyFloat_Fini(void)
+PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
 {
 	PyFloatObject *p;
 	PyFloatBlock *list, *next;
 	unsigned i;
-	int bc, bf;	/* block count, number of freed blocks */
+	size_t bc = 0, bf = 0;	/* block count, number of freed blocks */
-	int frem, fsum;	/* remaining unfreed floats per block, total */
+	size_t fsum = 0;	/* total unfreed ints */
+	int frem;		/* remaining unfreed ints per block */
 
-	bc = 0;
-	bf = 0;
-	fsum = 0;
 	list = block_list;
 	block_list = NULL;
 	free_list = NULL;
@@ -1585,6 +1583,22 @@ PyFloat_Fini(void)
 		fsum += frem;
 		list = next;
 	}
+	*pbc = bc;
+	*pbf = bf;
+	*bsum = fsum;
+}
+
+void
+PyFloat_Fini(void)
+{
+	PyFloatObject *p;
+	PyFloatBlock *list;
+	unsigned i;
+	size_t bc, bf;	/* block count, number of freed blocks */
+	size_t fsum;	/* total unfreed floats per block */
+
+	PyFloat_CompactFreeList(&bc, &bf, &fsum);
+
 	if (!Py_VerboseFlag)
 		return;
 	fprintf(stderr, "# cleanup floats");
@@ -1593,7 +1607,9 @@ PyFloat_Fini(void)
 	}
 	else {
 		fprintf(stderr,
-			": %d unfreed float%s in %d out of %d block%s\n",
+			": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
+			PY_FORMAT_SIZE_T "d out of %"
+			PY_FORMAT_SIZE_T "d block%s\n",
 			fsum, fsum == 1 ? "" : "s",
 			bc - bf, bc, bc == 1 ? "" : "s");
 	}

Objects/setobject.c

@@ -2173,7 +2173,8 @@ PySet_Discard(PyObject *set, PyObject *key)
 int
 PySet_Add(PyObject *anyset, PyObject *key)
 {
-	if (!PyAnySet_Check(anyset)) {
+	if (!PySet_Check(anyset) && 
+	    (!PyFrozenSet_Check(anyset) || Py_REFCNT(anyset) != 1)) {
 		PyErr_BadInternalCall();
 		return -1;
 	}
@@ -2277,6 +2278,10 @@ test_c_api(PySetObject *so)
 	f = PyFrozenSet_New(dup);
 	assertRaises(PySet_Clear(f) == -1, PyExc_SystemError);
 	assertRaises(_PySet_Update(f, dup) == -1, PyExc_SystemError);
+	assert(PySet_Add(f, elem) == 0);
+	Py_INCREF(f);
+	assertRaises(PySet_Add(f, elem) == -1, PyExc_SystemError);
+	Py_DECREF(f);
 	Py_DECREF(f);
 
 	/* Exercise direct iteration */

Python/sysmodule.c

@@ -730,17 +730,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\
 10. Number of stack pops performed by call_function()"
 );
 
-static PyObject *
-sys_cleartypecache(PyObject* self, PyObject* args)
-{
-	PyType_ClearCache();
-	Py_RETURN_NONE;
-}
-
-PyDoc_STRVAR(cleartypecache_doc,
-"_cleartypecache() -> None\n\
-Clear the internal type lookup cache.");
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -759,12 +748,41 @@ extern PyObject *_Py_GetDXProfile(PyObject *,  PyObject *);
 }
 #endif
 
+static PyObject *
+sys_clear_type_cache(PyObject* self, PyObject* args)
+{
+	PyType_ClearCache();
+	Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(sys_clear_type_cache__doc__,
+"_clear_type_cache() -> None\n\
+Clear the internal type lookup cache.");
+
+
+static PyObject *
+sys_compact_freelists(PyObject* self, PyObject* args)
+{
+	size_t fsum, fbc, fbf;
+
+	PyFloat_CompactFreeList(&fbc, &fbf, &fsum);
+
+	return Py_BuildValue("((kkk))", fsum, fbc, fbf);
+
+}
+
+PyDoc_STRVAR(sys_compact_freelists__doc__,
+"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks),)\n\
+Compact the free lists of floats.");
+
 static PyMethodDef sys_methods[] = {
 	/* Might as well keep this in alphabetic order */
 	{"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS,
 	 callstats_doc},
-	{"_cleartypecache", sys_cleartypecache, METH_NOARGS,
+	{"_clear_type_cache",	sys_clear_type_cache,	  METH_NOARGS,
-	 cleartypecache_doc},
+	 sys_clear_type_cache__doc__},
+	{"_compact_freelists",	sys_compact_freelists,	  METH_NOARGS,
+	 sys_compact_freelists__doc__},
 	{"_current_frames", sys_current_frames, METH_NOARGS,
 	 current_frames_doc},
 	{"displayhook",	sys_displayhook, METH_O, displayhook_doc},