From 31b9410654d73c9d1121ae9eb0d1bb14e6d54448 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 3 Dec 2015 01:02:03 +0200 Subject: [PATCH 1/2] Issue #25709: Fixed problem with in-place string concatenation and utf-8 cache. --- Lib/test/test_unicode.py | 17 +++++++++++++++++ Misc/NEWS | 5 +++++ Objects/unicodeobject.c | 5 +++++ 3 files changed, 27 insertions(+) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index f359082267d..ef0fd1c5f2f 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2303,6 +2303,23 @@ class UnicodeTest(string_tests.CommonTest, self.assertNotEqual(abc, abcdef) self.assertEqual(abcdef.decode('unicode_internal'), text) + @support.cpython_only + def test_pep393_utf8_caching_bug(self): + # Issue #25709: Problem with string concatenation and utf-8 cache + from _testcapi import getargs_s_hash + for k in 0x24, 0xa4, 0x20ac, 0x1f40d: + s = '' + for i in range(5): + # Due to CPython specific optimization the 's' string can be + # resized in-place. + s += chr(k) + # Parsing with the "s#" format code calls indirectly + # PyUnicode_AsUTF8AndSize() which creates the UTF-8 + # encoded string cached in the Unicode object. + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + # Check that the second call returns the same result + self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Misc/NEWS b/Misc/NEWS index 183ddd0819c..1f87a82fa18 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,8 +10,13 @@ What's New in Python 3.3.7? Core and Builtins ----------------- +- Issue #25709: Fixed problem with in-place string concatenation and utf-8 cache. + - Issue #24407: Fix crash when dict is mutated while being updated. +- Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside + __getattr__. + - Issue #24096: Make warnings.warn_explicit more robust against mutation of the warnings.filters list. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2bc34c57307..cf4f8421eb6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -679,6 +679,11 @@ resize_compact(PyObject *unicode, Py_ssize_t length) } new_size = (struct_size + (length + 1) * char_size); + if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { + PyObject_DEL(_PyUnicode_UTF8(unicode)); + _PyUnicode_UTF8(unicode) = NULL; + _PyUnicode_UTF8_LENGTH(unicode) = 0; + } _Py_DEC_REFTOTAL; _Py_ForgetReference(unicode); From 188c118ba55394b700e4cce9014ec4b09094e7b3 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 6 Jul 2016 15:27:35 -0400 Subject: [PATCH 2/2] Switch to the new upload url for PyPI --- Lib/distutils/config.py | 2 +- Lib/distutils/tests/test_config.py | 4 ++-- Lib/distutils/tests/test_upload.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py index 106e1465988..9b06d7699e9 100644 --- a/Lib/distutils/config.py +++ b/Lib/distutils/config.py @@ -21,7 +21,7 @@ password:%s class PyPIRCCommand(Command): """Base command that knows how to handle the .pypirc file """ - DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi' + DEFAULT_REPOSITORY = 'https://upload.pypi.io/legacy/' DEFAULT_REALM = 'pypi' repository = None realm = None diff --git a/Lib/distutils/tests/test_config.py b/Lib/distutils/tests/test_config.py index 4de825a81ed..c37381550d2 100644 --- a/Lib/distutils/tests/test_config.py +++ b/Lib/distutils/tests/test_config.py @@ -87,7 +87,7 @@ class PyPIRCCommandTestCase(support.TempdirManager, config = list(sorted(config.items())) waited = [('password', 'secret'), ('realm', 'pypi'), - ('repository', 'https://pypi.python.org/pypi'), + ('repository', 'https://upload.pypi.io/legacy/'), ('server', 'server1'), ('username', 'me')] self.assertEqual(config, waited) @@ -96,7 +96,7 @@ class PyPIRCCommandTestCase(support.TempdirManager, config = cmd._read_pypirc() config = list(sorted(config.items())) waited = [('password', 'secret'), ('realm', 'pypi'), - ('repository', 'https://pypi.python.org/pypi'), + ('repository', 'https://upload.pypi.io/legacy/'), ('server', 'server-login'), ('username', 'tarek')] self.assertEqual(config, waited) diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index 8532369aa50..5a462e5a94c 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -127,7 +127,7 @@ class uploadTestCase(PyPIRCCommandTestCase): self.assertTrue(headers['Content-type'].startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') self.assertEqual(self.last_open.req.get_full_url(), - 'https://pypi.python.org/pypi') + 'https://upload.pypi.io/legacy/') self.assertIn(b'xxx', self.last_open.req.data) # The PyPI response body was echoed