Bug 1003935: xrange overflows

Added XXX comment about why the undocumented PyRange_New() API function
is too broken to be worth the considerable pain of repairing.

Changed range_new() to stop using PyRange_New().  This fixes a variety
of bogus errors.  Nothing in the core uses PyRange_New() now.

Documented that xrange() is intended to be simple and fast, and that
CPython restricts its arguments, and length of its result sequence, to
native C longs.

Added some tests that failed before the patch, and repaired a test that
relied on a bogus OverflowError getting raised.

Doc/lib/libfuncs.tex

@@ -79,7 +79,7 @@ def my_import(name):
 
   \indexii{Boolean}{type}
   \versionadded{2.2.1}
-  \versionchanged[If no argument is given, this function returns 
+  \versionchanged[If no argument is given, this function returns
                   \constant{False}]{2.3}
 \end{funcdesc}
 
@@ -379,7 +379,7 @@ class C:
   that differentiate between binary and text files (else it is
   ignored).  If the file cannot be opened, \exception{IOError} is
   raised.
-  
+
   In addition to the standard \cfunction{fopen()} values \var{mode}
   may be \code{'U'} or \code{'rU'}. If Python is built with universal
   newline support (the default) the file is opened as a text file, but
@@ -392,7 +392,7 @@ class C:
   \var{mode} \code{'U'} is the same as normal text mode.  Note that
   file objects so opened also have an attribute called
   \member{newlines} which has a value of \code{None} (if no newlines
-  have yet been seen), \code{'\e n'}, \code{'\e r'}, \code{'\e r\e n'}, 
+  have yet been seen), \code{'\e n'}, \code{'\e r'}, \code{'\e r\e n'},
   or a tuple containing all the newline types seen.
 
   If \var{mode} is omitted, it defaults to \code{'r'}.  When opening a
@@ -459,7 +459,7 @@ class C:
   the inner sets should also be \class{frozenset} objects.  If
   \var{iterable} is not specified, returns a new empty set,
   \code{frozenset([])}.
-  \versionadded{2.4}  
+  \versionadded{2.4}
 \end{funcdesc}
 
 \begin{funcdesc}{getattr}{object, name\optional{, default}}
@@ -659,7 +659,7 @@ class C:
 \end{funcdesc}
 
 \begin{funcdesc}{object}{}
-  Return a new featureless object.  \function{object()} is a base 
+  Return a new featureless object.  \function{object()} is a base
   for all new style classes.  It has the methods that are common
   to all instances of new style classes.
   \versionadded{2.2}
@@ -901,7 +901,7 @@ except NameError:
   must be immutable.  To represent sets of sets, the inner sets should
   be \class{frozenset} objects.  If \var{iterable} is not specified,
   returns a new empty set, \code{set([])}.
-  \versionadded{2.4}  
+  \versionadded{2.4}
 \end{funcdesc}
 
 \begin{funcdesc}{setattr}{object, name, value}
@@ -931,7 +931,7 @@ except NameError:
   Return a new sorted list from the items in \var{iterable}.
   The optional arguments \var{cmp}, \var{key}, and \var{reverse}
   have the same meaning as those for the \method{list.sort()} method.
-  \versionadded{2.4}    
+  \versionadded{2.4}
 \end{funcdesc}
 
 \begin{funcdesc}{staticmethod}{function}
@@ -1099,6 +1099,12 @@ It's a function
   them) except when a very large range is used on a memory-starved
   machine or when all of the range's elements are never used (such as
   when the loop is usually terminated with \keyword{break}).
+
+  \note{\function{xrange()} is intended to be simple and fast.
+        Implementations may impose restrictions to achieve this.
+        The C implementation of Python restricts all arguments to
+        native C longs ("short" Python integers), and also requires
+        that that number of elements fit in a native C long.}
 \end{funcdesc}
 
 \begin{funcdesc}{zip}{\optional{seq1, \moreargs}}
@@ -1114,11 +1120,11 @@ It's a function
 
   \versionchanged[Formerly, \function{zip()} required at least one argument
   and \code{zip()} raised a \exception{TypeError} instead of returning
-  an empty list.]{2.4} 
+  an empty list.]{2.4}
 \end{funcdesc}
 
 
-% ---------------------------------------------------------------------------	
+% ---------------------------------------------------------------------------
 
 
 \section{Non-essential Built-in Functions \label{non-essential-built-in-funcs}}

Lib/test/test_xrange.py

@@ -48,10 +48,15 @@ class XrangeTest(unittest.TestCase):
         self.assertRaises(TypeError, xrange, 0, "spam")
         self.assertRaises(TypeError, xrange, 0, 42, "spam")
 
-        self.assertRaises(OverflowError, xrange, 0, sys.maxint, sys.maxint-1)
+        self.assertEqual(len(xrange(0, sys.maxint, sys.maxint-1)), 2)
+
         self.assertRaises(OverflowError, xrange, -sys.maxint, sys.maxint)
         self.assertRaises(OverflowError, xrange, 0, 2*sys.maxint)
 
+        self.assertEqual(len(xrange(-sys.maxint, sys.maxint, 2)),
+                         sys.maxint)
+        self.assertRaises(OverflowError, xrange, -sys.maxint-1, sys.maxint, 2)
+
 def test_main():
     test.test_support.run_unittest(XrangeTest)
 

Objects/rangeobject.c

@@ -9,6 +9,13 @@ typedef struct {
 	long	len;
 } rangeobject;
 
+/* XXX PyRange_New should be deprecated.  It's not documented.  It's not
+ * used in the core.  Its error-checking is akin to Swiss cheese:  accepts
+ * step == 0; accepts len < 0; ignores that (len - 1) * step may overflow;
+ * raises a baffling "integer addition" exception if it thinks the last
+ * item is "too big"; and doesn't compute whether "last item is too big"
+ * correctly even if the multiplication doesn't overflow.
+ */
 PyObject *
 PyRange_New(long start, long len, long step, int reps)
 {
@@ -79,6 +86,7 @@ get_len_of_range(long lo, long hi, long step)
 static PyObject *
 range_new(PyTypeObject *type, PyObject *args, PyObject *kw)
 {
+	rangeobject *obj;
 	long ilow = 0, ihigh = 0, istep = 1;
 	long n;
 
@@ -107,7 +115,14 @@ range_new(PyTypeObject *type, PyObject *args, PyObject *kw)
 				"xrange() result has too many items");
 		return NULL;
 	}
-	return PyRange_New(ilow, n, istep, 1);
+
+	obj = PyObject_New(rangeobject, &PyRange_Type);
+	if (obj == NULL)
+		return NULL;
+	obj->start = ilow;
+	obj->len   = n;
+	obj->step  = istep;
+	return (PyObject *) obj;
 }
 
 PyDoc_STRVAR(range_doc,