Merge Python 3.4.0rc1 release branch.

.hgtags

@@ -118,6 +118,8 @@ d047928ae3f6314a13b6137051315453d0ae89b6 v3.3.2
 fd53c500f8b80f54f3ecedec9da2e8c7e52a6888 v3.3.3rc1
 d32442c0e60dfbd71234e807d3d1dedd227495a9 v3.3.3rc2
 c3896275c0f61b2510a6c7e6c458a750359a91b8 v3.3.3
+fa92f5f940c6c0d839d7f0611e4b717606504a3c v3.3.4rc1
+7ff62415e4263c432c8acf6e424224209211eadb v3.3.4
 46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1
 9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2
 dd9cdf90a5073510877e9dd5112f8e6cf20d5e89 v3.4.0a3

Doc/tools/sphinxext/susp-ignored.csv

@@ -282,3 +282,4 @@ whatsnew/changelog,,:PythonCmd,"With Tk < 8.5 _tkinter.c:PythonCmd() raised Unic
 whatsnew/changelog,,::,": Fix FTP tests for IPv6, bind to ""::1"" instead of ""localhost""."
 whatsnew/changelog,,::,": Use ""127.0.0.1"" or ""::1"" instead of ""localhost"" as much as"
 whatsnew/changelog,,:password,user:password
+whatsnew/changelog,,:gz,w:gz

Doc/whatsnew/3.4.rst

@@ -364,6 +364,9 @@ Some smaller changes made to the core Python language are:
   Contributed by Victor Stinner, Kang-Hao (Kenny) Lu and Serhiy Storchaka in
   :issue:`12892`.
 
+* :class:`bytes`.join() and :class:`bytearray`.join() now accept arbitrary
+  buffer objects as arguments.  (Contributed by Antoine Pitrou in
+  :issue:`15958`.)
 
 
 New Modules
@@ -1609,6 +1612,19 @@ Porting to Python 3.4
 This section lists previously described changes and other bugfixes
 that may require changes to your code.
 
+
+Changes in 'python' command behavior
+------------------------------------
+
+* In a posix shell, setting the :envvar:`PATH` environment variable to
+  an empty value is equivalent to not setting it at all.  However, setting
+  :envvar:`PYTHONPATH` to an empty value was *not* equivalent to not setting it
+  at all: setting :envvar:`PYTHONPATH` to an empty value was equivalent to
+  setting it to ``.``, which leads to confusion when reasoning by analogy to
+  how :envvar:`PATH` works.  The behavior now conforms to the posix convention
+  for :envvar:`PATH`.
+
+
 Changes in the Python API
 -------------------------
 
@@ -1704,6 +1720,11 @@ Changes in the Python API
   informative :exc:`ValueError` rather than the previous more mysterious
   :exc:`AttributeError` (:issue:`9177`).
 
+* :meth:`slice.indices` no longer produces an :exc:`OverflowError` for huge
+  values.  As a consequence of this fix, :meth:`slice.indices` now raises a
+  :exc:`ValueError` if given a negative length; previously it returned nonsense
+  values (:issue:`14794`).
+
 
 Changes in the C API
 --------------------

Include/moduleobject.h

@@ -25,6 +25,7 @@ PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *);
 PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyModule_Clear(PyObject *);
+PyAPI_FUNC(void) _PyModule_ClearDict(PyObject *);
 #endif
 PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*);
 PyAPI_FUNC(void*) PyModule_GetState(PyObject*);

Include/pystate.h

@@ -33,7 +33,6 @@ typedef struct _is {
     int codecs_initialized;
     int fscodec_initialized;
 
-
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif
@@ -41,6 +40,7 @@ typedef struct _is {
     int tscdump;
 #endif
 
+    PyObject *builtins_copy;
 } PyInterpreterState;
 #endif
 

Lib/asyncio/base_events.py

@@ -96,6 +96,7 @@ class BaseEventLoop(events.AbstractEventLoop):
         self._default_executor = None
         self._internal_fds = 0
         self._running = False
+        self._clock_resolution = time.get_clock_info('monotonic').resolution
 
     def _make_socket_transport(self, sock, protocol, waiter=None, *,
                                extra=None, server=None):
@@ -643,10 +644,10 @@ class BaseEventLoop(events.AbstractEventLoop):
         self._process_events(event_list)
 
         # Handle 'later' callbacks that are ready.
-        now = self.time()
+        end_time = self.time() + self._clock_resolution
         while self._scheduled:
             handle = self._scheduled[0]
-            if handle._when > now:
+            if handle._when >= end_time:
                 break
             handle = heapq.heappop(self._scheduled)
             self._ready.append(handle)

Lib/asyncio/test_utils.py

@@ -191,6 +191,7 @@ class TestLoop(base_events.BaseEventLoop):
         self._gen = gen()
         next(self._gen)
         self._time = 0
+        self._clock_resolution = 1e-9
         self._timers = []
         self._selector = TestSelector()
 

Lib/idlelib/MultiCall.py

@@ -111,6 +111,8 @@ class _SimpleBinder:
             except tkinter.TclError as e:
                 if e.args[0] == APPLICATION_GONE:
                     pass
+                else:
+                    raise
 
 # An int in range(1 << len(_modifiers)) represents a combination of modifiers
 # (if the least significent bit is on, _modifiers[0] is on, and so on).
@@ -244,6 +246,8 @@ class _ComplexBinder:
             except tkinter.TclError as e:
                 if e.args[0] == APPLICATION_GONE:
                     break
+                else:
+                    raise
 
 # define the list of event types to be handled by MultiEvent. the order is
 # compatible with the definition of event type constants.
@@ -411,6 +415,8 @@ def MultiCallCreator(widget):
                         except tkinter.TclError as e:
                             if e.args[0] == APPLICATION_GONE:
                                 break
+                            else:
+                                raise
 
     _multicall_dict[widget] = MultiCall
     return MultiCall

Lib/idlelib/NEWS.txt

@@ -875,4 +875,3 @@ What's New in IDLEfork 0.9 Alpha 1?
 --------------------------------------------------------------------
 Refer to HISTORY.txt for additional information on earlier releases.
 --------------------------------------------------------------------
-

Lib/smtplib.py

@@ -62,6 +62,7 @@ SMTP_PORT = 25
 SMTP_SSL_PORT = 465
 CRLF = "\r\n"
 bCRLF = b"\r\n"
+_MAXLINE = 8192 # more than 8 times larger than RFC 821, 4.5.3
 
 OLDSTYLE_AUTH = re.compile(r"auth=(.*)", re.I)
 
@@ -365,7 +366,7 @@ class SMTP:
             self.file = self.sock.makefile('rb')
         while 1:
             try:
-                line = self.file.readline()
+                line = self.file.readline(_MAXLINE + 1)
             except OSError as e:
                 self.close()
                 raise SMTPServerDisconnected("Connection unexpectedly closed: "
@@ -375,6 +376,8 @@ class SMTP:
                 raise SMTPServerDisconnected("Connection unexpectedly closed")
             if self.debuglevel > 0:
                 print('reply:', repr(line), file=stderr)
+            if len(line) > _MAXLINE:
+                raise SMTPResponseException(500, "Line too long.")
             resp.append(line[4:].strip(b' \t\r\n'))
             code = line[:3]
             # Check that the error code is syntactically correct.

Lib/subprocess.py

@@ -738,6 +738,9 @@ _PLATFORM_DEFAULT_CLOSE_FDS = object()
 
 
 class Popen(object):
+
+    _child_created = False  # Set here since __del__ checks it
+
     def __init__(self, args, bufsize=-1, executable=None,
                  stdin=None, stdout=None, stderr=None,
                  preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
@@ -748,7 +751,6 @@ class Popen(object):
         """Create new Popen instance."""
         _cleanup()
 
-        self._child_created = False
         self._input = None
         self._communication_started = False
         if bufsize is None:
@@ -890,11 +892,8 @@ class Popen(object):
         # Wait for the process to terminate, to avoid zombies.
         self.wait()
 
-    def __del__(self, _maxsize=sys.maxsize, _active=_active):
-        # If __init__ hasn't had a chance to execute (e.g. if it
-        # was passed an undeclared keyword argument), we don't
-        # have a _child_created attribute at all.
-        if not getattr(self, '_child_created', False):
+    def __del__(self, _maxsize=sys.maxsize):
+        if not self._child_created:
             # We didn't get to successfully create a child process.
             return
         # In case the child hasn't been waited on, check if it's done.
@@ -1446,7 +1445,7 @@ class Popen(object):
                 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
                 _WEXITSTATUS=os.WEXITSTATUS):
             # This method is called (indirectly) by __del__, so it cannot
-            # refer to anything outside of its local scope."""
+            # refer to anything outside of its local scope.
             if _WIFSIGNALED(sts):
                 self.returncode = -_WTERMSIG(sts)
             elif _WIFEXITED(sts):

Lib/test/mock_socket.py

@@ -21,8 +21,13 @@ class MockFile:
     """
     def __init__(self, lines):
         self.lines = lines
-    def readline(self):
-        return self.lines.pop(0) + b'\r\n'
+    def readline(self, limit=-1):
+        result = self.lines.pop(0) + b'\r\n'
+        if limit >= 0:
+            # Re-insert the line, removing the \r\n we added.
+            self.lines.insert(0, result[limit:-2])
+            result = result[:limit]
+        return result
     def close(self):
         pass
 

Lib/test/test_asyncio/test_events.py

@@ -1176,15 +1176,17 @@ class EventLoopTestsMixin:
             loop = self.loop
             yield from asyncio.sleep(1e-2, loop=loop)
             yield from asyncio.sleep(1e-4, loop=loop)
+            yield from asyncio.sleep(1e-6, loop=loop)
+            yield from asyncio.sleep(1e-8, loop=loop)
+            yield from asyncio.sleep(1e-10, loop=loop)
 
         self.loop.run_until_complete(wait())
-        # The ideal number of call is 6, but on some platforms, the selector
+        # The ideal number of call is 12, but on some platforms, the selector
         # may sleep at little bit less than timeout depending on the resolution
-        # of the clock used by the kernel. Tolerate 2 useless calls on these
-        # platforms.
-        self.assertLessEqual(self.loop._run_once_counter, 8,
-            {'time_info': time.get_clock_info('time'),
-             'monotonic_info': time.get_clock_info('monotonic'),
+        # of the clock used by the kernel. Tolerate a few useless calls on
+        # these platforms.
+        self.assertLessEqual(self.loop._run_once_counter, 20,
+            {'clock_resolution': self.loop._clock_resolution,
              'selector': self.loop._selector.__class__.__name__})
 
 

Lib/test/test_builtin.py

@@ -16,6 +16,7 @@ import unittest
 import warnings
 from operator import neg
 from test.support import TESTFN, unlink,  run_unittest, check_warnings
+from test.script_helper import assert_python_ok
 try:
     import pty, signal
 except ImportError:
@@ -1592,6 +1593,34 @@ class TestSorted(unittest.TestCase):
         data = 'The quick Brown fox Jumped over The lazy Dog'.split()
         self.assertRaises(TypeError, sorted, data, None, lambda x,y: 0)
 
+
+class ShutdownTest(unittest.TestCase):
+
+    def test_cleanup(self):
+        # Issue #19255: builtins are still available at shutdown
+        code = """if 1:
+            import builtins
+            import sys
+
+            class C:
+                def __del__(self):
+                    print("before")
+                    # Check that builtins still exist
+                    len(())
+                    print("after")
+
+            c = C()
+            # Make this module survive until builtins and sys are cleaned
+            builtins.here = sys.modules[__name__]
+            sys.here = sys.modules[__name__]
+            # Create a reference loop so that this module needs to go
+            # through a GC phase.
+            here = sys.modules[__name__]
+            """
+        rc, out, err = assert_python_ok("-c", code)
+        self.assertEqual(["before", "after"], out.decode().splitlines())
+
+
 def load_tests(loader, tests, pattern):
     from doctest import DocTestSuite
     tests.addTest(DocTestSuite(builtins))

Lib/test/test_io.py

@@ -2691,7 +2691,8 @@ class CTextIOWrapperTest(TextIOWrapperTest):
 
 class PyTextIOWrapperTest(TextIOWrapperTest):
     io = pyio
-    shutdown_error = "LookupError: unknown encoding: ascii"
+    #shutdown_error = "LookupError: unknown encoding: ascii"
+    shutdown_error = "TypeError: 'NoneType' object is not iterable"
 
 
 class IncrementalNewlineDecoderTest(unittest.TestCase):

Lib/test/test_smtplib.py

@@ -563,6 +563,33 @@ class BadHELOServerTests(unittest.TestCase):
                             HOST, self.port, 'localhost', 3)
 
 
+@unittest.skipUnless(threading, 'Threading required for this test.')
+class TooLongLineTests(unittest.TestCase):
+    respdata = b'250 OK' + (b'.' * smtplib._MAXLINE * 2) + b'\n'
+
+    def setUp(self):
+        self.old_stdout = sys.stdout
+        self.output = io.StringIO()
+        sys.stdout = self.output
+
+        self.evt = threading.Event()
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.sock.settimeout(15)
+        self.port = support.bind_port(self.sock)
+        servargs = (self.evt, self.respdata, self.sock)
+        threading.Thread(target=server, args=servargs).start()
+        self.evt.wait()
+        self.evt.clear()
+
+    def tearDown(self):
+        self.evt.wait()
+        sys.stdout = self.old_stdout
+
+    def testLineTooLong(self):
+        self.assertRaises(smtplib.SMTPResponseException, smtplib.SMTP,
+                          HOST, self.port, 'localhost', 3)
+
+
 sim_users = {'Mr.A@somewhere.com':'John A',
              'Ms.B@xn--fo-fka.com':'Sally B',
              'Mrs.C@somewhereesle.com':'Ruth C',
@@ -888,7 +915,8 @@ class SMTPSimTests(unittest.TestCase):
 def test_main(verbose=None):
     support.run_unittest(GeneralTests, DebuggingServerTests,
                               NonConnectingTests,
-                              BadHELOServerTests, SMTPSimTests)
+                              BadHELOServerTests, SMTPSimTests,
+                              TooLongLineTests)
 
 if __name__ == '__main__':
     test_main()

Misc/NEWS

@@ -22,6 +22,11 @@ Release date: 2014-02-10
 Core and Builtins
 -----------------
 
+- Issue #19255: The builtins module is restored to initial value before
+  cleaning other modules.  The sys and builtins modules are cleaned last.
+
+- Issue #20588: Make Python-ast.c C89 compliant.
+
 - Issue #20437: Fixed 22 potential bugs when deleting objects references.
 
 - Issue #20500: Displaying an exception at interpreter shutdown no longer

Objects/moduleobject.c

@@ -298,6 +298,14 @@ PyModule_GetState(PyObject* m)
 
 void
 _PyModule_Clear(PyObject *m)
+{
+    PyObject *d = ((PyModuleObject *)m)->md_dict;
+    if (d != NULL)
+        _PyModule_ClearDict(d);
+}
+
+void
+_PyModule_ClearDict(PyObject *d)
 {
     /* To make the execution order of destructors for global
        objects a bit more predictable, we first zap all objects
@@ -308,11 +316,6 @@ _PyModule_Clear(PyObject *m)
 
     Py_ssize_t pos;
     PyObject *key, *value;
-    PyObject *d;
-
-    d = ((PyModuleObject *)m)->md_dict;
-    if (d == NULL)
-        return;
 
     /* First, clear only names starting with a single underscore */
     pos = 0;

Parser/asdl_c.py

@@ -1203,10 +1203,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
 {
     mod_ty res;
-    PyObject *req_type[] = {(PyObject*)Module_type, (PyObject*)Expression_type,
-                            (PyObject*)Interactive_type};
+    PyObject *req_type[3];
     char *req_name[] = {"Module", "Expression", "Interactive"};
     int isinstance;
+
+    req_type[0] = (PyObject*)Module_type;
+    req_type[1] = (PyObject*)Expression_type;
+    req_type[2] = (PyObject*)Interactive_type;
+
     assert(0 <= mode && mode <= 2);
 
     if (!init_types())

Python/Python-ast.c

@@ -7023,10 +7023,14 @@ PyObject* PyAST_mod2obj(mod_ty t)
 mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
 {
     mod_ty res;
-    PyObject *req_type[] = {(PyObject*)Module_type, (PyObject*)Expression_type,
-                            (PyObject*)Interactive_type};
+    PyObject *req_type[3];
     char *req_name[] = {"Module", "Expression", "Interactive"};
     int isinstance;
+
+    req_type[0] = (PyObject*)Module_type;
+    req_type[1] = (PyObject*)Expression_type;
+    req_type[2] = (PyObject*)Interactive_type;
+
     assert(0 <= mode && mode <= 2);
 
     if (!init_types())

Python/import.c

@@ -49,9 +49,13 @@ class fs_unicode_converter(CConverter):
 void
 _PyImport_Init(void)
 {
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
     initstr = PyUnicode_InternFromString("__init__");
     if (initstr == NULL)
         Py_FatalError("Can't initialize import variables");
+    interp->builtins_copy = PyDict_Copy(interp->builtins);
+    if (interp->builtins_copy == NULL)
+        Py_FatalError("Can't backup builtins dict");
 }
 
 void
@@ -397,8 +401,8 @@ PyImport_Cleanup(void)
     PyObject *key, *value, *dict;
     PyInterpreterState *interp = PyThreadState_GET()->interp;
     PyObject *modules = interp->modules;
-    PyObject *builtins = interp->builtins;
     PyObject *weaklist = NULL;
+    char **p;
 
     if (modules == NULL)
         return; /* Already done */
@@ -411,31 +415,22 @@ PyImport_Cleanup(void)
 
     /* XXX Perhaps these precautions are obsolete. Who knows? */
 
-    value = PyDict_GetItemString(modules, "builtins");
-    if (value != NULL && PyModule_Check(value)) {
-        dict = PyModule_GetDict(value);
+    if (Py_VerboseFlag)
+        PySys_WriteStderr("# clear builtins._\n");
+    PyDict_SetItemString(interp->builtins, "_", Py_None);
+
+    for (p = sys_deletes; *p != NULL; p++) {
         if (Py_VerboseFlag)
-            PySys_WriteStderr("# clear builtins._\n");
-        PyDict_SetItemString(dict, "_", Py_None);
+            PySys_WriteStderr("# clear sys.%s\n", *p);
+        PyDict_SetItemString(interp->sysdict, *p, Py_None);
     }
-    value = PyDict_GetItemString(modules, "sys");
-    if (value != NULL && PyModule_Check(value)) {
-        char **p;
-        PyObject *v;
-        dict = PyModule_GetDict(value);
-        for (p = sys_deletes; *p != NULL; p++) {
-            if (Py_VerboseFlag)
-                PySys_WriteStderr("# clear sys.%s\n", *p);
-            PyDict_SetItemString(dict, *p, Py_None);
-        }
-        for (p = sys_files; *p != NULL; p+=2) {
-            if (Py_VerboseFlag)
-                PySys_WriteStderr("# restore sys.%s\n", *p);
-            v = PyDict_GetItemString(dict, *(p+1));
-            if (v == NULL)
-                v = Py_None;
-            PyDict_SetItemString(dict, *p, v);
-        }
+    for (p = sys_files; *p != NULL; p+=2) {
+        if (Py_VerboseFlag)
+            PySys_WriteStderr("# restore sys.%s\n", *p);
+        value = PyDict_GetItemString(interp->sysdict, *(p+1));
+        if (value == NULL)
+            value = Py_None;
+        PyDict_SetItemString(interp->sysdict, *p, value);
     }
 
     /* We prepare a list which will receive (name, weakref) tuples of
@@ -473,11 +468,15 @@ PyImport_Cleanup(void)
 
     /* Clear the modules dict. */
     PyDict_Clear(modules);
-    /* Replace the interpreter's reference to builtins with an empty dict
-       (module globals still have a reference to the original builtins). */
-    builtins = interp->builtins;
-    interp->builtins = PyDict_New();
-    Py_DECREF(builtins);
+    /* Restore the original builtins dict, to ensure that any
+       user data gets cleared. */
+    dict = PyDict_Copy(interp->builtins);
+    if (dict == NULL)
+        PyErr_Clear();
+    PyDict_Clear(interp->builtins);
+    if (PyDict_Update(interp->builtins, interp->builtins_copy))
+        PyErr_Clear();
+    Py_XDECREF(dict);
     /* Clear module dict copies stored in the interpreter state */
     _PyState_ClearModules();
     /* Collect references */
@@ -488,7 +487,15 @@ PyImport_Cleanup(void)
 
     /* Now, if there are any modules left alive, clear their globals to
        minimize potential leaks.  All C extension modules actually end
-       up here, since they are kept alive in the interpreter state. */
+       up here, since they are kept alive in the interpreter state.
+
+       The special treatment of "builtins" here is because even
+       when it's not referenced as a module, its dictionary is
+       referenced by almost every module's __builtins__.  Since
+       deleting a module clears its dictionary (even if there are
+       references left to it), we need to delete the "builtins"
+       module last.  Likewise, we don't delete sys until the very
+       end because it is implicitly referenced (e.g. by print). */
     if (weaklist != NULL) {
         Py_ssize_t i, n;
         n = PyList_GET_SIZE(weaklist);
@@ -498,17 +505,27 @@ PyImport_Cleanup(void)
             PyObject *mod = PyWeakref_GET_OBJECT(PyTuple_GET_ITEM(tup, 1));
             if (mod == Py_None)
                 continue;
-            Py_INCREF(mod);
             assert(PyModule_Check(mod));
+            dict = PyModule_GetDict(mod);
+            if (dict == interp->builtins || dict == interp->sysdict)
+                continue;
+            Py_INCREF(mod);
             if (Py_VerboseFlag && PyUnicode_Check(name))
-                PySys_FormatStderr("# cleanup[3] wiping %U\n",
-                                   name, mod);
+                PySys_FormatStderr("# cleanup[3] wiping %U\n", name);
             _PyModule_Clear(mod);
             Py_DECREF(mod);
         }
         Py_DECREF(weaklist);
     }
 
+    /* Next, delete sys and builtins (in that order) */
+    if (Py_VerboseFlag)
+        PySys_FormatStderr("# cleanup[3] wiping sys\n");
+    _PyModule_ClearDict(interp->sysdict);
+    if (Py_VerboseFlag)
+        PySys_FormatStderr("# cleanup[3] wiping builtins\n");
+    _PyModule_ClearDict(interp->builtins);
+
     /* Clear and delete the modules directory.  Actual modules will
        still be there only if imported during the execution of some
        destructor. */

Python/pystate.c

@@ -72,6 +72,7 @@ PyInterpreterState_New(void)
         interp->modules_by_index = NULL;
         interp->sysdict = NULL;
         interp->builtins = NULL;
+        interp->builtins_copy = NULL;
         interp->tstate_head = NULL;
         interp->codec_search_path = NULL;
         interp->codec_search_cache = NULL;
@@ -115,6 +116,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
     Py_CLEAR(interp->modules_by_index);
     Py_CLEAR(interp->sysdict);
     Py_CLEAR(interp->builtins);
+    Py_CLEAR(interp->builtins_copy);
     Py_CLEAR(interp->importlib);
 }