From 97ff04789de3e37af585648de70260a54a29bd47 Mon Sep 17 00:00:00 2001 From: Armin Rigo Date: Wed, 9 Aug 2006 15:37:26 +0000 Subject: [PATCH] Concatenation on a long string breaks (SF #1526585). --- Lib/test/regrtest.py | 4 ++- Lib/test/test_bigaddrspace.py | 46 +++++++++++++++++++++++++++++++++++ Lib/test/test_support.py | 21 +++++++++++++++- Python/ceval.c | 12 ++++++--- 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 Lib/test/test_bigaddrspace.py diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 26c4d0409fa..4553838afcd 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -66,7 +66,9 @@ reports are written to. These parameters all have defaults (5, 4 and -M runs tests that require an exorbitant amount of memory. These tests typically try to ascertain containers keep working when containing more than -2 bilion objects, and only work on 64-bit systems. The passed-in memlimit, +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, which is a string in the form of '2.5Gb', determines howmuch memory the tests will limit themselves to (but they may go slightly over.) The number shouldn't be more memory than the machine has (including swap memory). You diff --git a/Lib/test/test_bigaddrspace.py b/Lib/test/test_bigaddrspace.py new file mode 100644 index 00000000000..8c215fe09ad --- /dev/null +++ b/Lib/test/test_bigaddrspace.py @@ -0,0 +1,46 @@ +from test import test_support +from test.test_support import bigaddrspacetest, MAX_Py_ssize_t + +import unittest +import operator +import sys + + +class StrTest(unittest.TestCase): + + @bigaddrspacetest + def test_concat(self): + s1 = 'x' * MAX_Py_ssize_t + self.assertRaises(OverflowError, operator.add, s1, '?') + + @bigaddrspacetest + def test_optimized_concat(self): + x = 'x' * MAX_Py_ssize_t + try: + x = x + '?' # this statement uses a fast path in ceval.c + except OverflowError: + pass + else: + self.fail("should have raised OverflowError") + try: + x += '?' # this statement uses a fast path in ceval.c + except OverflowError: + pass + else: + self.fail("should have raised OverflowError") + self.assertEquals(len(x), MAX_Py_ssize_t) + + ### the following test is pending a patch + # (http://mail.python.org/pipermail/python-dev/2006-July/067774.html) + #@bigaddrspacetest + #def test_repeat(self): + # self.assertRaises(OverflowError, operator.mul, 'x', MAX_Py_ssize_t + 1) + + +def test_main(): + test_support.run_unittest(StrTest) + +if __name__ == '__main__': + if len(sys.argv) > 1: + test_support.set_memlimit(sys.argv[1]) + test_main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 6532c79a997..a9d5dabcc76 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -314,6 +314,12 @@ _1M = 1024*1024 _1G = 1024 * _1M _2G = 2 * _1G +# Hack to get at the maximum value an internal index can take. +class _Dummy: + def __getslice__(self, i, j): + return j +MAX_Py_ssize_t = _Dummy()[:] + def set_memlimit(limit): import re global max_memuse @@ -328,7 +334,9 @@ def set_memlimit(limit): if m is None: raise ValueError('Invalid memory limit %r' % (limit,)) memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) - if memlimit < 2.5*_1G: + if memlimit > MAX_Py_ssize_t: + memlimit = MAX_Py_ssize_t + if memlimit < _2G - 1: raise ValueError('Memory limit %r too low to be useful' % (limit,)) max_memuse = memlimit @@ -371,6 +379,17 @@ def bigmemtest(minsize, memuse, overhead=5*_1M): return wrapper return decorator +def bigaddrspacetest(f): + """Decorator for tests that fill the address space.""" + def wrapper(self): + if max_memuse < MAX_Py_ssize_t: + if verbose: + sys.stderr.write("Skipping %s because of memory " + "constraint\n" % (f.__name__,)) + else: + return f(self) + return wrapper + #======================================================================= # Preliminary PyUNIT integration. diff --git a/Python/ceval.c b/Python/ceval.c index ef484d9b70f..a0e8b30c052 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -4225,6 +4225,14 @@ string_concatenate(PyObject *v, PyObject *w, { /* This function implements 'variable += expr' when both arguments are strings. */ + Py_ssize_t v_len = PyString_GET_SIZE(v); + Py_ssize_t w_len = PyString_GET_SIZE(w); + Py_ssize_t new_len = v_len + w_len; + if (new_len < 0) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + return NULL; + } if (v->ob_refcnt == 2) { /* In the common case, there are 2 references to the value @@ -4269,9 +4277,7 @@ string_concatenate(PyObject *v, PyObject *w, /* Now we own the last reference to 'v', so we can resize it * in-place. */ - Py_ssize_t v_len = PyString_GET_SIZE(v); - Py_ssize_t w_len = PyString_GET_SIZE(w); - if (_PyString_Resize(&v, v_len + w_len) != 0) { + if (_PyString_Resize(&v, new_len) != 0) { /* XXX if _PyString_Resize() fails, 'v' has been * deallocated so it cannot be put back into 'variable'. * The MemoryError is raised when there is no value in