Fix a couple of bugs exposed by the new __index__ code. The 64-bit buildbots

were failing due to inappropriate clipping of numbers larger than 2**31
with new-style classes. (typeobject.c)  In reviewing the code for classic
classes, there were 2 problems.  Any negative value return could be returned.
Always return -1 if there was an error.  Also make the checks similar
with the new-style classes.  I believe this is correct for 32 and 64 bit
boxes, including Windows64.

Add a test of classic classes too.

Lib/test/test_index.py

@@ -181,8 +181,8 @@ class OverflowTestCase(unittest.TestCase):
         self.assertEqual(self.pos.__index__(), self.pos)
         self.assertEqual(self.neg.__index__(), self.neg)
 
-    def test_getitem(self):
-        class GetItem(object):
+    def _getitem_helper(self, base):
+        class GetItem(base):
             def __len__(self):
                 return maxint
             def __getitem__(self, key):
@@ -195,6 +195,13 @@ class OverflowTestCase(unittest.TestCase):
         self.assertEqual(x[self.neg:self.pos], (-1, maxint))
         self.assertEqual(x[self.neg:self.pos:1].indices(maxint), (0, maxint, 1))
 
+    def test_getitem(self):
+        self._getitem_helper(object)
+
+    def test_getitem_classic(self):
+        class Empty: pass
+        self._getitem_helper(Empty)
+
     def test_sequence_repeat(self):
         self.failUnlessRaises(OverflowError, lambda: "a" * self.pos)
         self.failUnlessRaises(OverflowError, lambda: "a" * self.neg)

Misc/NEWS

@@ -12,6 +12,12 @@ What's New in Python 2.5 release candidate 1?
 Core and builtins
 -----------------
 
+- Fix bug related to __len__ functions using values > 2**32 on 64-bit machines
+  with new-style classes.
+  
+- Fix bug related to __len__ functions returning negative values with
+  classic classes.
+  
 - Patch #1538606, Fix __index__() clipping.  There were some problems
   discovered with the API and how integers that didn't fit into Py_ssize_t
   were handled.  This patch attempts to provide enough alternatives

Objects/classobject.c

@@ -974,24 +974,25 @@ instance_length(PyInstanceObject *inst)
 	if (res == NULL)
 		return -1;
 	if (PyInt_Check(res)) {
-		Py_ssize_t temp = PyInt_AsSsize_t(res);
-		if (temp == -1 && PyErr_Occurred()) {
+		outcome = PyInt_AsSsize_t(res);
+		if (outcome == -1 && PyErr_Occurred()) {
 			Py_DECREF(res);
 			return -1;
 		}
-		outcome = (Py_ssize_t)temp;
-#if SIZEOF_SIZE_T < SIZEOF_LONG
+#if SIZEOF_SIZE_T < SIZEOF_INT
 		/* Overflow check -- range of PyInt is more than C int */
-		if (outcome != temp) {
+		if (outcome != (int)outcome) {
 			PyErr_SetString(PyExc_OverflowError,
 			 "__len__() should return 0 <= outcome < 2**31");
 			outcome = -1;
 		}
 		else
 #endif
-		if (outcome < 0)
+		if (outcome < 0) {
 			PyErr_SetString(PyExc_ValueError,
 					"__len__() should return >= 0");
+			outcome = -1;
+		}
 	}
 	else {
 		PyErr_SetString(PyExc_TypeError,

Objects/typeobject.c

@@ -4110,19 +4110,17 @@ slot_sq_length(PyObject *self)
 {
 	static PyObject *len_str;
 	PyObject *res = call_method(self, "__len__", &len_str, "()");
-	Py_ssize_t temp;
 	Py_ssize_t len;
 
 	if (res == NULL)
 		return -1;
-	temp = PyInt_AsSsize_t(res);
-	len = (int)temp;
+	len = PyInt_AsSsize_t(res);
 	Py_DECREF(res);
 	if (len == -1 && PyErr_Occurred())
 		return -1;
-#if SIZEOF_SIZE_T < SIZEOF_LONG
+#if SIZEOF_SIZE_T < SIZEOF_INT
 	/* Overflow check -- range of PyInt is more than C ssize_t */
-	if (len != temp) {
+	if (len != (int)len) {
 		PyErr_SetString(PyExc_OverflowError,
 			"__len__() should return 0 <= outcome < 2**31");
 		return -1;