From 069456267e96c359ec7b4bd4073c3a027629d79a Mon Sep 17 00:00:00 2001
From: Ralf Schmitt <ralf@systemexit.de>
Date: Tue, 31 May 2011 17:10:03 -0500
Subject: [PATCH 01/13] disable ASDLGEN if hg won't work, or if python is not
 installed. This change makes configure check for - the existence of a hg
 repository - the hg executable itself - the python executable

Running $(srcdir)/Parser/asdl_c.py (i.e. ASDLGEN) will fail if any of
the above prerequisites is missing, so we now disable it instead.

closes #12225
---
 Makefile.pre.in |  2 +-
 configure.in    | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/Makefile.pre.in b/Makefile.pre.in
index b93e482b070..694faadbfc2 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -274,7 +274,7 @@ AST_ASDL=	$(srcdir)/Parser/Python.asdl
 
 ASDLGEN_FILES=	$(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py
 # XXX Note that a build now requires Python exist before the build starts
-ASDLGEN=	$(srcdir)/Parser/asdl_c.py
+ASDLGEN=	@DISABLE_ASDLGEN@ $(srcdir)/Parser/asdl_c.py
 
 ##########################################################################
 # Python
diff --git a/configure.in b/configure.in
index 524282fd539..8e89c8fc368 100644
--- a/configure.in
+++ b/configure.in
@@ -832,7 +832,13 @@ fi
 AC_SUBST(HGVERSION)
 AC_SUBST(HGTAG)
 AC_SUBST(HGBRANCH)
+
+if test -e $srcdir/.hg/00changelog.i
+then
 AC_CHECK_PROG(HAS_HG, hg, found, not-found)
+else
+HAS_HG=no-repository
+fi
 if test $HAS_HG = found
 then
     HGVERSION="hg id -i \$(srcdir)"
@@ -844,6 +850,15 @@ else
     HGBRANCH=""
 fi
 
+AC_SUBST(DISABLE_ASDLGEN)
+DISABLE_ASDLGEN=""
+AC_CHECK_PROG(HAS_PYTHON, python, found, not-found)
+if test $HAS_HG != found -o $HAS_PYTHON != found
+then
+    DISABLE_ASDLGEN="@echo hg: $HAS_HG, python: $HAS_PYTHON! cannot run \$(srcdir)/Parser/asdl_c.py #"
+fi
+
+
 case $MACHDEP in
 bsdos*|hp*|HP*)
 	# install -d does not work on BSDI or HP-UX

From 2ee61884fe350bc1efbd20bffb305d6eac5a1937 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=89ric=20Araujo?= <merwok@netwok.org>
Date: Sat, 2 Jul 2011 16:45:45 +0200
Subject: [PATCH 02/13] Clean up NEWS entry and tests for shutil.disk_usage
 (#12442)

---
 Doc/whatsnew/3.3.rst    |  2 +-
 Lib/test/test_shutil.py | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst
index a2f512b8bc1..e5e18051a51 100644
--- a/Doc/whatsnew/3.3.rst
+++ b/Doc/whatsnew/3.3.rst
@@ -206,7 +206,7 @@ handle NAT with non-secure FTP without opening fixed ports.
 shutil
 ------
 
-The :mod:`shutil` module has a new :func:`~shutil.disk_usage` providing total,
+The :mod:`shutil` module has a new :func:`~shutil.disk_usage` function providing total,
 used and free disk space statistics.
 
 (Contributed by Giampaolo Rodolà in :issue:`12442`)
diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py
index 20e9412aec3..b17ff3e20ea 100644
--- a/Lib/test/test_shutil.py
+++ b/Lib/test/test_shutil.py
@@ -732,11 +732,11 @@ class TestShutil(unittest.TestCase):
                          "disk_usage not available on this platform")
     def test_disk_usage(self):
         usage = shutil.disk_usage(os.getcwd())
-        self.assertTrue(usage.total > 0)
-        self.assertTrue(usage.used > 0)
-        self.assertTrue(usage.free >= 0)
-        self.assertTrue(usage.total >= usage.used)
-        self.assertTrue(usage.total > usage.free)
+        self.assertGreater(usage.total, 0)
+        self.assertGreater(usage.used, 0)
+        self.assertGreaterEqual(usage.free, 0)
+        self.assertGreaterEqual(usage.total, usage.used)
+        self.assertGreater(usage.total, usage.free)
 
 
 class TestMove(unittest.TestCase):

From 5bdae3bb7c7ab9e85453698972fa5fa926f012f3 Mon Sep 17 00:00:00 2001
From: Vinay Sajip <vinay_sajip@yahoo.co.uk>
Date: Sat, 2 Jul 2011 16:42:47 +0100
Subject: [PATCH 03/13] Closes #12291: Fixed bug which was found when doing 
 multiple loads from one stream.

---
 Lib/importlib/test/source/test_file_loader.py |   2 +-
 Lib/test/test_marshal.py                      |  24 ++
 Misc/NEWS                                     |   3 +
 Python/marshal.c                              | 215 +++++++++++++-----
 4 files changed, 183 insertions(+), 61 deletions(-)

diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py
index 0ffe78dfa10..20280927273 100644
--- a/Lib/importlib/test/source/test_file_loader.py
+++ b/Lib/importlib/test/source/test_file_loader.py
@@ -214,7 +214,7 @@ class BadBytecodeTest(unittest.TestCase):
                                                 lambda bc: bc[:8] + b'<test>',
                                                 del_source=del_source)
             file_path = mapping['_temp'] if not del_source else bytecode_path
-            with self.assertRaises(ValueError):
+            with self.assertRaises(EOFError):
                 self.import_(file_path, '_temp')
 
     def _test_bad_magic(self, test, *, del_source=False):
diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
index 81cf598402a..cd100f9be44 100644
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -211,6 +211,30 @@ class BugsTestCase(unittest.TestCase):
         invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00'
         self.assertRaises(ValueError, marshal.loads, invalid_string)
 
+    def test_multiple_dumps_and_loads(self):
+        # Issue 12291: marshal.load() should be callable multiple times
+        # with interleaved data written by non-marshal code
+        # Adapted from a patch by Engelbert Gruber.
+        data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c']))
+        for interleaved in (b'', b'0123'):
+            ilen = len(interleaved)
+            positions = []
+            try:
+                with open(support.TESTFN, 'wb') as f:
+                    for d in data:
+                        marshal.dump(d, f)
+                        if ilen:
+                            f.write(interleaved)
+                        positions.append(f.tell())
+                with open(support.TESTFN, 'rb') as f:
+                    for i, d in enumerate(data):
+                        self.assertEqual(d, marshal.load(f))
+                        if ilen:
+                            f.read(ilen)
+                        self.assertEqual(positions[i], f.tell())
+            finally:
+                support.unlink(support.TESTFN)
+
 
 def test_main():
     support.run_unittest(IntTestCase,
diff --git a/Misc/NEWS b/Misc/NEWS
index 423d7ec3e02..2013559be88 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@ What's New in Python 3.2.1 release candidate 2?
 Core and Builtins
 -----------------
 
+- Issue #12291: You can now load multiple marshalled objects from a stream,
+  with other data interleaved between marshalled objects.
+
 - Issue #12084: os.stat on Windows now works properly with relative symbolic
   links when called from any directory.
 
diff --git a/Python/marshal.c b/Python/marshal.c
index 73d4f374cd0..396e05c6381 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -57,6 +57,7 @@ typedef struct {
     int error;  /* see WFERR_* values */
     int depth;
     /* If fp == NULL, the following are valid: */
+    PyObject * readable;    /* Stream-like object being read from */
     PyObject *str;
     char *ptr;
     char *end;
@@ -466,27 +467,75 @@ typedef WFILE RFILE; /* Same struct with different invariants */
 
 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
 
-#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
-
 static int
 r_string(char *s, int n, RFILE *p)
 {
-    if (p->fp != NULL)
-        /* The result fits into int because it must be <=n. */
-        return (int)fread(s, 1, n, p->fp);
-    if (p->end - p->ptr < n)
-        n = (int)(p->end - p->ptr);
-    memcpy(s, p->ptr, n);
-    p->ptr += n;
-    return n;
+    char * ptr;
+    int read, left;
+
+    if (!p->readable) {
+        if (p->fp != NULL)
+            /* The result fits into int because it must be <=n. */
+            read = (int) fread(s, 1, n, p->fp);
+        else {
+            left = (int)(p->end - p->ptr);
+            read = (left < n) ? left : n;
+            memcpy(s, p->ptr, read);
+            p->ptr += read;
+        }
+    }
+    else {
+        PyObject *data = PyObject_CallMethod(p->readable, "read", "i", n);
+        read = 0;
+        if (data != NULL) {
+            if (!PyBytes_Check(data)) {
+                PyErr_Format(PyExc_TypeError,
+                             "f.read() returned not bytes but %.100s",
+                             data->ob_type->tp_name);
+            }
+            else {
+                read = PyBytes_GET_SIZE(data);
+                if (read > 0) {
+                    ptr = PyBytes_AS_STRING(data);
+                    memcpy(s, ptr, read);
+                }
+            }
+            Py_DECREF(data);
+        }
+    }
+    if (!PyErr_Occurred() && (read < n)) {
+        PyErr_SetString(PyExc_EOFError, "EOF read where not expected");
+    }
+    return read;
+}
+
+
+static int
+r_byte(RFILE *p)
+{
+    int c = EOF;
+    unsigned char ch;
+    int n;
+
+    if (!p->readable)
+        c = p->fp ? getc(p->fp) : rs_byte(p);
+    else {
+        n = r_string((char *) &ch, 1, p);
+        if (n > 0)
+            c = ch;
+    }
+    return c;
 }
 
 static int
 r_short(RFILE *p)
 {
     register short x;
-    x = r_byte(p);
-    x |= r_byte(p) << 8;
+    unsigned char buffer[2];
+
+    r_string((char *) buffer, 2, p);
+    x = buffer[0];
+    x |= buffer[1] << 8;
     /* Sign-extension, in case short greater than 16 bits */
     x |= -(x & 0x8000);
     return x;
@@ -496,19 +545,13 @@ static long
 r_long(RFILE *p)
 {
     register long x;
-    register FILE *fp = p->fp;
-    if (fp) {
-        x = getc(fp);
-        x |= (long)getc(fp) << 8;
-        x |= (long)getc(fp) << 16;
-        x |= (long)getc(fp) << 24;
-    }
-    else {
-        x = rs_byte(p);
-        x |= (long)rs_byte(p) << 8;
-        x |= (long)rs_byte(p) << 16;
-        x |= (long)rs_byte(p) << 24;
-    }
+    unsigned char buffer[4];
+
+    r_string((char *) buffer, 4, p);
+    x = buffer[0];
+    x |= (long)buffer[1] << 8;
+    x |= (long)buffer[2] << 16;
+    x |= (long)buffer[3] << 24;
 #if SIZEOF_LONG > 4
     /* Sign extension for 64-bit machines */
     x |= -(x & 0x80000000L);
@@ -526,25 +569,30 @@ r_long(RFILE *p)
 static PyObject *
 r_long64(RFILE *p)
 {
+    PyObject * result = NULL;
     long lo4 = r_long(p);
     long hi4 = r_long(p);
+
+    if (!PyErr_Occurred()) {
 #if SIZEOF_LONG > 4
-    long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
-    return PyLong_FromLong(x);
+        long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
+        result = PyLong_FromLong(x);
 #else
-    unsigned char buf[8];
-    int one = 1;
-    int is_little_endian = (int)*(char*)&one;
-    if (is_little_endian) {
-        memcpy(buf, &lo4, 4);
-        memcpy(buf+4, &hi4, 4);
-    }
-    else {
-        memcpy(buf, &hi4, 4);
-        memcpy(buf+4, &lo4, 4);
-    }
-    return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
+        unsigned char buf[8];
+        int one = 1;
+        int is_little_endian = (int)*(char*)&one;
+        if (is_little_endian) {
+            memcpy(buf, &lo4, 4);
+            memcpy(buf+4, &hi4, 4);
+        }
+        else {
+            memcpy(buf, &hi4, 4);
+            memcpy(buf+4, &lo4, 4);
+        }
+        result = _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
 #endif
+    }
+    return result;
 }
 
 static PyObject *
@@ -556,6 +604,8 @@ r_PyLong(RFILE *p)
     digit d;
 
     n = r_long(p);
+    if (PyErr_Occurred())
+        return NULL;
     if (n == 0)
         return (PyObject *)_PyLong_New(0);
     if (n < -INT_MAX || n > INT_MAX) {
@@ -575,6 +625,8 @@ r_PyLong(RFILE *p)
         d = 0;
         for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
             md = r_short(p);
+            if (PyErr_Occurred())
+                break;
             if (md < 0 || md > PyLong_MARSHAL_BASE)
                 goto bad_digit;
             d += (digit)md << j*PyLong_MARSHAL_SHIFT;
@@ -584,6 +636,8 @@ r_PyLong(RFILE *p)
     d = 0;
     for (j=0; j < shorts_in_top_digit; j++) {
         md = r_short(p);
+        if (PyErr_Occurred())
+            break;
         if (md < 0 || md > PyLong_MARSHAL_BASE)
             goto bad_digit;
         /* topmost marshal digit should be nonzero */
@@ -595,6 +649,10 @@ r_PyLong(RFILE *p)
         }
         d += (digit)md << j*PyLong_MARSHAL_SHIFT;
     }
+    if (PyErr_Occurred()) {
+        Py_DECREF(ob);
+        return NULL;
+    }
     /* top digit should be nonzero, else the resulting PyLong won't be
        normalized */
     ob->ob_digit[size-1] = d;
@@ -663,7 +721,8 @@ r_object(RFILE *p)
         break;
 
     case TYPE_INT:
-        retval = PyLong_FromLong(r_long(p));
+        n = r_long(p);
+        retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n);
         break;
 
     case TYPE_INT64:
@@ -773,6 +832,10 @@ r_object(RFILE *p)
 
     case TYPE_STRING:
         n = r_long(p);
+        if (PyErr_Occurred()) {
+            retval = NULL;
+            break;
+        }
         if (n < 0 || n > INT_MAX) {
             PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
             retval = NULL;
@@ -798,6 +861,10 @@ r_object(RFILE *p)
         char *buffer;
 
         n = r_long(p);
+        if (PyErr_Occurred()) {
+            retval = NULL;
+            break;
+        }
         if (n < 0 || n > INT_MAX) {
             PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
             retval = NULL;
@@ -823,6 +890,10 @@ r_object(RFILE *p)
 
     case TYPE_TUPLE:
         n = r_long(p);
+        if (PyErr_Occurred()) {
+            retval = NULL;
+            break;
+        }
         if (n < 0 || n > INT_MAX) {
             PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
             retval = NULL;
@@ -850,6 +921,10 @@ r_object(RFILE *p)
 
     case TYPE_LIST:
         n = r_long(p);
+        if (PyErr_Occurred()) {
+            retval = NULL;
+            break;
+        }
         if (n < 0 || n > INT_MAX) {
             PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
             retval = NULL;
@@ -902,6 +977,10 @@ r_object(RFILE *p)
     case TYPE_SET:
     case TYPE_FROZENSET:
         n = r_long(p);
+        if (PyErr_Occurred()) {
+            retval = NULL;
+            break;
+        }
         if (n < 0 || n > INT_MAX) {
             PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
             retval = NULL;
@@ -955,10 +1034,20 @@ r_object(RFILE *p)
 
             /* XXX ignore long->int overflows for now */
             argcount = (int)r_long(p);
+            if (PyErr_Occurred())
+                goto code_error;
             kwonlyargcount = (int)r_long(p);
+            if (PyErr_Occurred())
+                goto code_error;
             nlocals = (int)r_long(p);
+            if (PyErr_Occurred())
+                goto code_error;
             stacksize = (int)r_long(p);
+            if (PyErr_Occurred())
+                goto code_error;
             flags = (int)r_long(p);
+            if (PyErr_Occurred())
+                goto code_error;
             code = r_object(p);
             if (code == NULL)
                 goto code_error;
@@ -1040,6 +1129,7 @@ PyMarshal_ReadShortFromFile(FILE *fp)
 {
     RFILE rf;
     assert(fp);
+    rf.readable = NULL;
     rf.fp = fp;
     rf.strings = NULL;
     rf.end = rf.ptr = NULL;
@@ -1051,6 +1141,7 @@ PyMarshal_ReadLongFromFile(FILE *fp)
 {
     RFILE rf;
     rf.fp = fp;
+    rf.readable = NULL;
     rf.strings = NULL;
     rf.ptr = rf.end = NULL;
     return r_long(&rf);
@@ -1112,6 +1203,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp)
     RFILE rf;
     PyObject *result;
     rf.fp = fp;
+    rf.readable = NULL;
     rf.strings = PyList_New(0);
     rf.depth = 0;
     rf.ptr = rf.end = NULL;
@@ -1126,6 +1218,7 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
     RFILE rf;
     PyObject *result;
     rf.fp = NULL;
+    rf.readable = NULL;
     rf.ptr = str;
     rf.end = str + len;
     rf.strings = PyList_New(0);
@@ -1142,6 +1235,7 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
     PyObject *res = NULL;
 
     wf.fp = NULL;
+    wf.readable = NULL;
     wf.str = PyBytes_FromStringAndSize((char *)NULL, 50);
     if (wf.str == NULL)
         return NULL;
@@ -1219,33 +1313,33 @@ The version argument indicates the data format that dump should use.");
 static PyObject *
 marshal_load(PyObject *self, PyObject *f)
 {
-    /* XXX Quick hack -- need to do this differently */
     PyObject *data, *result;
     RFILE rf;
-    data = PyObject_CallMethod(f, "read", "");
+    char *p;
+    int n;
+
+    /*
+     * Make a call to the read method, but read zero bytes.
+     * This is to ensure that the object passed in at least
+     * has a read method which returns bytes.
+     */
+    data = PyObject_CallMethod(f, "read", "i", 0);
     if (data == NULL)
         return NULL;
-    rf.fp = NULL;
-    if (PyBytes_Check(data)) {
-        rf.ptr = PyBytes_AS_STRING(data);
-        rf.end = rf.ptr + PyBytes_GET_SIZE(data);
-    }
-    else if (PyBytes_Check(data)) {
-        rf.ptr = PyBytes_AS_STRING(data);
-        rf.end = rf.ptr + PyBytes_GET_SIZE(data);
+    if (!PyBytes_Check(data)) {
+        PyErr_Format(PyExc_TypeError,
+                     "f.read() returned not bytes but %.100s",
+                     data->ob_type->tp_name);
+        result = NULL;
     }
     else {
-        PyErr_Format(PyExc_TypeError,
-                     "f.read() returned neither string "
-                     "nor bytes but %.100s",
-                     data->ob_type->tp_name);
-        Py_DECREF(data);
-        return NULL;
+        rf.strings = PyList_New(0);
+        rf.depth = 0;
+        rf.fp = NULL;
+        rf.readable = f;
+        result = read_object(&rf);
+        Py_DECREF(rf.strings);
     }
-    rf.strings = PyList_New(0);
-    rf.depth = 0;
-    result = read_object(&rf);
-    Py_DECREF(rf.strings);
     Py_DECREF(data);
     return result;
 }
@@ -1296,6 +1390,7 @@ marshal_loads(PyObject *self, PyObject *args)
     s = p.buf;
     n = p.len;
     rf.fp = NULL;
+    rf.readable = NULL;
     rf.ptr = s;
     rf.end = s + n;
     rf.strings = PyList_New(0);

From 32322843914c144f8c741470c5450c0cb7d9f7df Mon Sep 17 00:00:00 2001
From: Vinay Sajip <vinay_sajip@yahoo.co.uk>
Date: Sat, 2 Jul 2011 17:19:51 +0100
Subject: [PATCH 04/13] Removed breaking typo accidentally introduced during
 merge with 3.2.

---
 Python/marshal.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/Python/marshal.c b/Python/marshal.c
index b8d06ad2864..c749bb33576 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1320,8 +1320,6 @@ marshal_load(PyObject *self, PyObject *f)
 {
     PyObject *data, *result;
     RFILE rf;
-    char *p;
-    int n;
 
     /*
      * Make a call to the read method, but read zero bytes.
@@ -1338,12 +1336,10 @@ marshal_load(PyObject *self, PyObject *f)
         result = NULL;
     }
     else {
-        rf.strings = PyList_New(0);
         rf.depth = 0;
         rf.fp = NULL;
         rf.readable = f;
         result = read_object(&rf);
-        Py_DECREF(rf.strings);
     }
     Py_DECREF(data);
     return result;

From 623e8b86af8d8b34589478f7c85de49054e07a6d Mon Sep 17 00:00:00 2001
From: Vinay Sajip <vinay_sajip@yahoo.co.uk>
Date: Sat, 2 Jul 2011 17:21:37 +0100
Subject: [PATCH 05/13] Removed some unused local variables.

---
 Python/marshal.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/Python/marshal.c b/Python/marshal.c
index 396e05c6381..76d5690438d 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1315,8 +1315,6 @@ marshal_load(PyObject *self, PyObject *f)
 {
     PyObject *data, *result;
     RFILE rf;
-    char *p;
-    int n;
 
     /*
      * Make a call to the read method, but read zero bytes.

From aac0f75b3b28513c45116534720e66b0262e2e72 Mon Sep 17 00:00:00 2001
From: Vinay Sajip <vinay_sajip@yahoo.co.uk>
Date: Sat, 2 Jul 2011 18:42:21 +0100
Subject: [PATCH 06/13] Correct uninitialized data problem in marshal code.

---
 Python/marshal.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Python/marshal.c b/Python/marshal.c
index c749bb33576..35fcd3afb3d 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -1339,6 +1339,7 @@ marshal_load(PyObject *self, PyObject *f)
         rf.depth = 0;
         rf.fp = NULL;
         rf.readable = f;
+        rf.current_filename = NULL;
         result = read_object(&rf);
     }
     Py_DECREF(data);

From 020436b0d4ae271638ed5d0881c1fa7f7c0a1b09 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <solipsis@pitrou.net>
Date: Sat, 2 Jul 2011 21:20:25 +0200
Subject: [PATCH 07/13] Issue #12456: fix a possible hang on shutdown of a
 concurrent.futures.ProcessPoolExecutor.

---
 Lib/concurrent/futures/process.py   | 30 ++++++++++++++++++++---------
 Lib/test/test_concurrent_futures.py |  7 +++++++
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index c2331e7da97..689f9ba99c5 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -50,7 +50,7 @@ import os
 from concurrent.futures import _base
 import queue
 import multiprocessing
-from multiprocessing.queues import SimpleQueue, SentinelReady
+from multiprocessing.queues import SimpleQueue, SentinelReady, Full
 import threading
 import weakref
 
@@ -195,6 +195,10 @@ def _queue_management_worker(executor_reference,
         result_queue: A multiprocessing.Queue of _ResultItems generated by the
             process workers.
     """
+    executor = None
+
+    def shutting_down():
+        return _shutdown or executor is None or executor._shutdown_thread
 
     def shutdown_worker():
         # This is an upper bound
@@ -202,8 +206,7 @@ def _queue_management_worker(executor_reference,
         for i in range(0, nb_children_alive):
             call_queue.put(None)
         # If .join() is not called on the created processes then
-        # some multiprocessing.Queue methods may deadlock on Mac OS
-        # X.
+        # some multiprocessing.Queue methods may deadlock on Mac OS X.
         for p in processes.values():
             p.join()
 
@@ -222,7 +225,7 @@ def _queue_management_worker(executor_reference,
             if executor is not None:
                 executor._broken = True
                 executor._shutdown_thread = True
-                del executor
+                executor = None
             # All futures in flight must be marked failed
             for work_id, work_item in pending_work_items.items():
                 work_item.future.set_exception(
@@ -242,7 +245,11 @@ def _queue_management_worker(executor_reference,
         if isinstance(result_item, int):
             # Clean shutdown of a worker using its PID
             # (avoids marking the executor broken)
+            assert shutting_down()
             del processes[result_item]
+            if not processes:
+                shutdown_worker()
+                return
         elif result_item is not None:
             work_item = pending_work_items.pop(result_item.work_id, None)
             # work_item can be None if another process terminated (see above)
@@ -257,16 +264,21 @@ def _queue_management_worker(executor_reference,
         #   - The interpreter is shutting down OR
         #   - The executor that owns this worker has been collected OR
         #   - The executor that owns this worker has been shutdown.
-        if _shutdown or executor is None or executor._shutdown_thread:
+        if shutting_down():
             # Since no new work items can be added, it is safe to shutdown
             # this thread if there are no pending work items.
-            if not pending_work_items:
+            if not pending_work_items and call_queue.qsize() == 0:
                 shutdown_worker()
                 return
-            else:
+            try:
                 # Start shutting down by telling a process it can exit.
-                call_queue.put(None)
-        del executor
+                call_queue.put_nowait(None)
+            except Full:
+                # This is not a problem: we will eventually be woken up (in
+                # result_queue.get()) and be able to send a sentinel again,
+                # if necessary.
+                pass
+        executor = None
 
 _system_limits_checked = False
 _system_limited = None
diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py
index 5968980ff1e..fda6f5b807e 100644
--- a/Lib/test/test_concurrent_futures.py
+++ b/Lib/test/test_concurrent_futures.py
@@ -367,6 +367,13 @@ class ExecutorTest(unittest.TestCase):
 
         self.assertEqual([None, None], results)
 
+    def test_shutdown_race_issue12456(self):
+        # Issue #12456: race condition at shutdown where trying to post a
+        # sentinel in the call queue blocks (the queue is full while processes
+        # have exited).
+        self.executor.map(str, [2] * (self.worker_count + 1))
+        self.executor.shutdown()
+
 
 class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest):
     def test_map_submits_without_iteration(self):

From ac4e5abc788dfd10474fe3b0a6c5c802d3159763 Mon Sep 17 00:00:00 2001
From: R David Murray <rdmurray@bitdance.com>
Date: Sat, 2 Jul 2011 21:03:19 -0400
Subject: [PATCH 08/13] #12147: make send_message correctly handle Sender and
 Resent- headers.

Original patch by Nicolas Estibals.  My tweaks to the patch were mostly
style/cosmetic, and adding more tests.
---
 Doc/library/smtplib.rst  |  27 +++++++---
 Lib/smtplib.py           |  52 +++++++++++++-----
 Lib/test/test_smtplib.py | 111 ++++++++++++++++++++++++++++++++++++++-
 Misc/ACKS                |   1 +
 Misc/NEWS                |   3 ++
 5 files changed, 172 insertions(+), 22 deletions(-)
 mode change 100755 => 100644 Lib/smtplib.py

diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst
index 531a64d73f2..5978a8fe9d6 100644
--- a/Doc/library/smtplib.rst
+++ b/Doc/library/smtplib.rst
@@ -323,21 +323,32 @@ An :class:`SMTP` instance has the following methods:
    .. versionchanged:: 3.2 *msg* may be a byte string.
 
 
-.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, mail_options=[], rcpt_options=[])
+.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \
+                              mail_options=[], rcpt_options=[])
 
    This is a convenience method for calling :meth:`sendmail` with the message
    represented by an :class:`email.message.Message` object.  The arguments have
    the same meaning as for :meth:`sendmail`, except that *msg* is a ``Message``
    object.
 
-   If *from_addr* is ``None``, ``send_message`` sets its value to the value of
-   the :mailheader:`From` header from *msg*.  If *to_addrs* is ``None``,
-   ``send_message`` combines the values (if any) of the :mailheader:`To`,
-   :mailheader:`CC`, and :mailheader:`Bcc` fields from *msg*.  Regardless of
-   the values of *from_addr* and *to_addrs*, ``send_message`` deletes any  Bcc
-   field from *msg*.  It then serializes *msg* using
+   If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills
+   those arguments with addresses extracted from the headers of *msg* as
+   specified in :rfc:`2822`\: *from_addr* is set to the :mailheader:`Sender`
+   field if it is present, and otherwise to the :mailheader:`From` field.
+   *to_adresses* combines the values (if any) of the :mailheader:`To`,
+   :mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*.  If exactly one
+   set of :mailheader:`Resent-*` headers appear in the message, the regular
+   headers are ignored and the :mailheader:`Resent-*` headers are used instead.
+   If the message contains more than one set of :mailheader:`Resent-*` headers,
+   a :exc:`ValueError` is raised, since there is no way to unambiguously detect
+   the most recent set of :mailheader:`Resent-` headers.
+
+   ``send_message`` serializes *msg* using
    :class:`~email.generator.BytesGenerator` with ``\r\n`` as the *linesep*, and
-   calls :meth:`sendmail` to transmit the resulting message.
+   calls :meth:`sendmail` to transmit the resulting message.  Regardless of the
+   values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any
+   :mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear
+   in *msg*.
 
    .. versionadded:: 3.2
 
diff --git a/Lib/smtplib.py b/Lib/smtplib.py
old mode 100755
new mode 100644
index ce71699d8de..9080e4ae570
--- a/Lib/smtplib.py
+++ b/Lib/smtplib.py
@@ -49,6 +49,7 @@ import email.message
 import email.generator
 import base64
 import hmac
+import copy
 from email.base64mime import body_encode as encode_base64
 from sys import stderr
 
@@ -674,7 +675,7 @@ class SMTP:
 
         msg may be a string containing characters in the ASCII range, or a byte
         string.  A string is encoded to bytes using the ascii codec, and lone
-        \r and \n characters are converted to \r\n characters.
+        \\r and \\n characters are converted to \\r\\n characters.
 
         If there has been no previous EHLO or HELO command this session, this
         method tries ESMTP EHLO first.  If the server does ESMTP, message size
@@ -757,24 +758,49 @@ class SMTP:
         """Converts message to a bytestring and passes it to sendmail.
 
         The arguments are as for sendmail, except that msg is an
-        email.message.Message object.  If from_addr is None, the from_addr is
-        taken from the 'From' header of the Message.  If to_addrs is None, its
-        value is composed from the addresses listed in the 'To', 'CC', and
-        'Bcc' fields.  Regardless of the values of from_addr and to_addr, any
-        Bcc field in the Message object is deleted.  The Message object is then
-        serialized using email.generator.BytesGenerator and sendmail is called
-        to transmit the message.
+        email.message.Message object.  If from_addr is None or to_addrs is
+        None, these arguments are taken from the headers of the Message as
+        described in RFC 2822 (a ValueError is raised if there is more than
+        one set of 'Resent-' headers).  Regardless of the values of from_addr and
+        to_addr, any Bcc field (or Resent-Bcc field, when the Message is a
+        resent) of the Message object won't be transmitted.  The Message
+        object is then serialized using email.generator.BytesGenerator and
+        sendmail is called to transmit the message.
+
         """
+        # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822
+        # Section 3.6.6). In such a case, we use the 'Resent-*' fields.  However,
+        # if there is more than one 'Resent-' block there's no way to
+        # unambiguously determine which one is the most recent in all cases,
+        # so rather than guess we raise a ValueError in that case.
+        #
+        # TODO implement heuristics to guess the correct Resent-* block with an
+        # option allowing the user to enable the heuristics.  (It should be
+        # possible to guess correctly almost all of the time.)
+        resent =msg.get_all('Resent-Date')
+        if resent is None:
+            header_prefix = ''
+        elif len(resent) == 1:
+            header_prefix = 'Resent-'
+        else:
+            raise ValueError("message has more than one 'Resent-' header block")
         if from_addr is None:
-            from_addr = msg['From']
+            # Prefer the sender field per RFC 2822:3.6.2.
+            from_addr = (msg[header_prefix+'Sender']
+                           if (header_prefix+'Sender') in msg
+                           else msg[header_prefix+'From'])
         if to_addrs is None:
-            addr_fields = [f for f in (msg['To'], msg['Bcc'], msg['CC'])
-                            if f is not None]
+            addr_fields = [f for f in (msg[header_prefix+'To'],
+                                       msg[header_prefix+'Bcc'],
+                                       msg[header_prefix+'Cc']) if f is not None]
             to_addrs = [a[1] for a in email.utils.getaddresses(addr_fields)]
-        del msg['Bcc']
+        # Make a local copy so we can delete the bcc headers.
+        msg_copy = copy.copy(msg)
+        del msg_copy['Bcc']
+        del msg_copy['Resent-Bcc']
         with io.BytesIO() as bytesmsg:
             g = email.generator.BytesGenerator(bytesmsg)
-            g.flatten(msg, linesep='\r\n')
+            g.flatten(msg_copy, linesep='\r\n')
             flatmsg = bytesmsg.getvalue()
         return self.sendmail(from_addr, to_addrs, flatmsg, mail_options,
                              rcpt_options)
diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py
index dd920447add..bacfbdfe514 100644
--- a/Lib/test/test_smtplib.py
+++ b/Lib/test/test_smtplib.py
@@ -320,13 +320,16 @@ class DebuggingServerTests(unittest.TestCase):
         # XXX (see comment in testSend)
         time.sleep(0.01)
         smtp.quit()
+        # make sure the Bcc header is still in the message.
+        self.assertEqual(m['Bcc'], 'John Root <root@localhost>, "Dinsdale" '
+                                    '<warped@silly.walks.com>')
 
         self.client_evt.set()
         self.serv_evt.wait()
         self.output.flush()
         # Add the X-Peer header that DebuggingServer adds
         m['X-Peer'] = socket.gethostbyname('localhost')
-        # The Bcc header is deleted before serialization.
+        # The Bcc header should not be transmitted.
         del m['Bcc']
         mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
         self.assertEqual(self.output.getvalue(), mexpect)
@@ -365,6 +368,112 @@ class DebuggingServerTests(unittest.TestCase):
                                  re.MULTILINE)
             self.assertRegex(debugout, to_addr)
 
+    def testSendMessageWithSpecifiedAddresses(self):
+        # Make sure addresses specified in call override those in message.
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'foo@bar.com'
+        m['To'] = 'John, Dinsdale'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        smtp.send_message(m, from_addr='joe@example.com', to_addrs='foo@example.net')
+        # XXX (see comment in testSend)
+        time.sleep(0.01)
+        smtp.quit()
+
+        self.client_evt.set()
+        self.serv_evt.wait()
+        self.output.flush()
+        # Add the X-Peer header that DebuggingServer adds
+        m['X-Peer'] = socket.gethostbyname('localhost')
+        mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
+        self.assertEqual(self.output.getvalue(), mexpect)
+        debugout = smtpd.DEBUGSTREAM.getvalue()
+        sender = re.compile("^sender: joe@example.com$", re.MULTILINE)
+        self.assertRegex(debugout, sender)
+        for addr in ('John', 'Dinsdale'):
+            to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr),
+                                 re.MULTILINE)
+            self.assertNotRegex(debugout, to_addr)
+        recip = re.compile(r"^recips: .*'foo@example.net'.*$", re.MULTILINE)
+        self.assertRegex(debugout, recip)
+
+    def testSendMessageWithMultipleFrom(self):
+        # Sender overrides To
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'Bernard, Bianca'
+        m['Sender'] = 'the_rescuers@Rescue-Aid-Society.com'
+        m['To'] = 'John, Dinsdale'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        smtp.send_message(m)
+        # XXX (see comment in testSend)
+        time.sleep(0.01)
+        smtp.quit()
+
+        self.client_evt.set()
+        self.serv_evt.wait()
+        self.output.flush()
+        # Add the X-Peer header that DebuggingServer adds
+        m['X-Peer'] = socket.gethostbyname('localhost')
+        mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
+        self.assertEqual(self.output.getvalue(), mexpect)
+        debugout = smtpd.DEBUGSTREAM.getvalue()
+        sender = re.compile("^sender: the_rescuers@Rescue-Aid-Society.com$", re.MULTILINE)
+        self.assertRegex(debugout, sender)
+        for addr in ('John', 'Dinsdale'):
+            to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr),
+                                 re.MULTILINE)
+            self.assertRegex(debugout, to_addr)
+
+    def testSendMessageResent(self):
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'foo@bar.com'
+        m['To'] = 'John'
+        m['CC'] = 'Sally, Fred'
+        m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>'
+        m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000'
+        m['Resent-From'] = 'holy@grail.net'
+        m['Resent-To'] = 'Martha <my_mom@great.cooker.com>, Jeff'
+        m['Resent-Bcc'] = 'doe@losthope.net'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        smtp.send_message(m)
+        # XXX (see comment in testSend)
+        time.sleep(0.01)
+        smtp.quit()
+
+        self.client_evt.set()
+        self.serv_evt.wait()
+        self.output.flush()
+        # The Resent-Bcc headers are deleted before serialization.
+        del m['Bcc']
+        del m['Resent-Bcc']
+        # Add the X-Peer header that DebuggingServer adds
+        m['X-Peer'] = socket.gethostbyname('localhost')
+        mexpect = '%s%s\n%s' % (MSG_BEGIN, m.as_string(), MSG_END)
+        self.assertEqual(self.output.getvalue(), mexpect)
+        debugout = smtpd.DEBUGSTREAM.getvalue()
+        sender = re.compile("^sender: holy@grail.net$", re.MULTILINE)
+        self.assertRegex(debugout, sender)
+        for addr in ('my_mom@great.cooker.com', 'Jeff', 'doe@losthope.net'):
+            to_addr = re.compile(r"^recips: .*'{}'.*$".format(addr),
+                                 re.MULTILINE)
+            self.assertRegex(debugout, to_addr)
+
+    def testSendMessageMultipleResentRaises(self):
+        m = email.mime.text.MIMEText('A test message')
+        m['From'] = 'foo@bar.com'
+        m['To'] = 'John'
+        m['CC'] = 'Sally, Fred'
+        m['Bcc'] = 'John Root <root@localhost>, "Dinsdale" <warped@silly.walks.com>'
+        m['Resent-Date'] = 'Thu, 1 Jan 1970 17:42:00 +0000'
+        m['Resent-From'] = 'holy@grail.net'
+        m['Resent-To'] = 'Martha <my_mom@great.cooker.com>, Jeff'
+        m['Resent-Bcc'] = 'doe@losthope.net'
+        m['Resent-Date'] = 'Thu, 2 Jan 1970 17:42:00 +0000'
+        m['Resent-To'] = 'holy@grail.net'
+        m['Resent-From'] = 'Martha <my_mom@great.cooker.com>, Jeff'
+        smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3)
+        with self.assertRaises(ValueError):
+            smtp.send_message(m)
+        smtp.close()
 
 class NonConnectingTests(unittest.TestCase):
 
diff --git a/Misc/ACKS b/Misc/ACKS
index 74409f76730..45b4042fa9a 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -265,6 +265,7 @@ Michael Ernst
 Ben Escoto
 Andy Eskilsson
 Stefan Esser
+Nicolas Estibals
 Stephen D Evans
 Carey Evans
 Tim Everett
diff --git a/Misc/NEWS b/Misc/NEWS
index 2013559be88..89542ec41e1 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -28,6 +28,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #12147: Adjust the new-in-3.2 smtplib.send_message method for better
+  conformance to the RFCs:  correctly handle Sender and Resent- headers.
+
 - Issue #12352: Fix a deadlock in multiprocessing.Heap when a block is freed by
   the garbage collector while the Heap lock is held.
 

From 187c111a550e6b32c732601b9f22ced3341e7c2c Mon Sep 17 00:00:00 2001
From: Georg Brandl <georg@python.org>
Date: Sun, 3 Jul 2011 09:23:20 +0200
Subject: [PATCH 09/13] Regenerate configure.

---
 configure | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/configure b/configure
index 8e204f0e397..36723f614ad 100755
--- a/configure
+++ b/configure
@@ -644,6 +644,8 @@ LN
 INSTALL_DATA
 INSTALL_SCRIPT
 INSTALL_PROGRAM
+HAS_PYTHON
+DISABLE_ASDLGEN
 HAS_HG
 HGBRANCH
 HGTAG
@@ -5204,6 +5206,9 @@ fi
 
 
 
+
+if test -e $srcdir/.hg/00changelog.i
+then
 # Extract the first word of "hg", so it can be a program name with args.
 set dummy hg; ac_word=$2
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
@@ -5242,6 +5247,9 @@ $as_echo "no" >&6; }
 fi
 
 
+else
+HAS_HG=no-repository
+fi
 if test $HAS_HG = found
 then
     HGVERSION="hg id -i \$(srcdir)"
@@ -5253,6 +5261,52 @@ else
     HGBRANCH=""
 fi
 
+
+DISABLE_ASDLGEN=""
+# Extract the first word of "python", so it can be a program name with args.
+set dummy python; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_HAS_PYTHON+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$HAS_PYTHON"; then
+  ac_cv_prog_HAS_PYTHON="$HAS_PYTHON" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if { test -f "$as_dir/$ac_word$ac_exec_ext" && $as_test_x "$as_dir/$ac_word$ac_exec_ext"; }; then
+    ac_cv_prog_HAS_PYTHON="found"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_prog_HAS_PYTHON" && ac_cv_prog_HAS_PYTHON="not-found"
+fi
+fi
+HAS_PYTHON=$ac_cv_prog_HAS_PYTHON
+if test -n "$HAS_PYTHON"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HAS_PYTHON" >&5
+$as_echo "$HAS_PYTHON" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+if test $HAS_HG != found -o $HAS_PYTHON != found
+then
+    DISABLE_ASDLGEN="@echo hg: $HAS_HG, python: $HAS_PYTHON! cannot run \$(srcdir)/Parser/asdl_c.py #"
+fi
+
+
 case $MACHDEP in
 bsdos*|hp*|HP*)
 	# install -d does not work on BSDI or HP-UX

From 4334d740ed1ee6bbdad2e6df88ac11453d5f8142 Mon Sep 17 00:00:00 2001
From: Vinay Sajip <vinay_sajip@yahoo.co.uk>
Date: Sun, 3 Jul 2011 10:35:41 +0100
Subject: [PATCH 10/13] Issue #12406: Added upates for packaging's .exe files,
 command_template, and sysconfig.cfg.

---
 Tools/msi/msi.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py
index de1a6233238..ff490bea192 100644
--- a/Tools/msi/msi.py
+++ b/Tools/msi/msi.py
@@ -119,6 +119,7 @@ pythondll_uuid = {
     "30":"{6953bc3b-6768-4291-8410-7914ce6e2ca8}",
     "31":"{4afcba0b-13e4-47c3-bebe-477428b46913}",
     "32":"{3ff95315-1096-4d31-bd86-601d5438ad5e}",
+    "33":"{f7581ca4-d368-4eea-8f82-d48c64c4f047}",
     } [major+minor]
 
 # Compute the name that Sphinx gives to the docfile
@@ -1010,6 +1011,8 @@ def add_files(db):
             lib.remove_pyc()
         # package READMEs if present
         lib.glob("README")
+        if dir=='Lib':
+            lib.add_file("sysconfig.cfg")
         if dir=='test' and parent.physical=='Lib':
             lib.add_file("185test.db")
             lib.add_file("audiotest.au")
@@ -1045,7 +1048,7 @@ def add_files(db):
         if dir=="Icons":
             lib.glob("*.gif")
             lib.add_file("idle.icns")
-        if dir=="command" and parent.physical=="distutils":
+        if dir=="command" and parent.physical in ("distutils", "packaging"):
             lib.glob("wininst*.exe")
             lib.add_file("command_template")
         if dir=="lib2to3":
@@ -1156,6 +1159,8 @@ def add_files(db):
             lib.add_file("README.txt", src="README")
         if f == 'Scripts':
             lib.add_file("2to3.py", src="2to3")
+            lib.add_file("pydoc3.py", src="pydoc3")
+            lib.add_file("pysetup3.py", src="pysetup3")
             if have_tcl:
                 lib.start_component("pydocgui.pyw", tcltk, keyfile="pydocgui.pyw")
                 lib.add_file("pydocgui.pyw")

From 1c405b3e60d3237de8a4f6f338d42fdd0b84f5e7 Mon Sep 17 00:00:00 2001
From: Antoine Pitrou <solipsis@pitrou.net>
Date: Sun, 3 Jul 2011 13:17:06 +0200
Subject: [PATCH 11/13] Followup to 51c1f2cedb96 (and issue #12456): qsize()
 raises NotImplementedError on OS X, don't use it.

---
 Lib/concurrent/futures/process.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py
index 689f9ba99c5..41e132020e1 100644
--- a/Lib/concurrent/futures/process.py
+++ b/Lib/concurrent/futures/process.py
@@ -204,7 +204,7 @@ def _queue_management_worker(executor_reference,
         # This is an upper bound
         nb_children_alive = sum(p.is_alive() for p in processes.values())
         for i in range(0, nb_children_alive):
-            call_queue.put(None)
+            call_queue.put_nowait(None)
         # If .join() is not called on the created processes then
         # some multiprocessing.Queue methods may deadlock on Mac OS X.
         for p in processes.values():
@@ -265,18 +265,18 @@ def _queue_management_worker(executor_reference,
         #   - The executor that owns this worker has been collected OR
         #   - The executor that owns this worker has been shutdown.
         if shutting_down():
-            # Since no new work items can be added, it is safe to shutdown
-            # this thread if there are no pending work items.
-            if not pending_work_items and call_queue.qsize() == 0:
-                shutdown_worker()
-                return
             try:
-                # Start shutting down by telling a process it can exit.
-                call_queue.put_nowait(None)
+                # Since no new work items can be added, it is safe to shutdown
+                # this thread if there are no pending work items.
+                if not pending_work_items:
+                    shutdown_worker()
+                    return
+                else:
+                    # Start shutting down by telling a process it can exit.
+                    call_queue.put_nowait(None)
             except Full:
                 # This is not a problem: we will eventually be woken up (in
-                # result_queue.get()) and be able to send a sentinel again,
-                # if necessary.
+                # result_queue.get()) and be able to send a sentinel again.
                 pass
         executor = None
 

From 946eb865a3d163dc5aa242827d562933c89c9c5b Mon Sep 17 00:00:00 2001
From: Senthil Kumaran <senthil@uthcode.com>
Date: Sun, 3 Jul 2011 10:17:22 -0700
Subject: [PATCH 12/13] reST indentation fix in sqlite3 docs. rst uses 3 space
 indentation.

---
 Doc/library/sqlite3.rst | 75 +++++++++++++++++++++--------------------
 1 file changed, 38 insertions(+), 37 deletions(-)

diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 73676744303..32ae7244e39 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -599,43 +599,43 @@ Row Objects
 
 Let's assume we initialize a table as in the example given above::
 
-    conn = sqlite3.connect(":memory:")
-    c = conn.cursor()
-    c.execute('''create table stocks
-    (date text, trans text, symbol text,
-     qty real, price real)''')
-    c.execute("""insert into stocks
-              values ('2006-01-05','BUY','RHAT',100,35.14)""")
-    conn.commit()
-    c.close()
+   conn = sqlite3.connect(":memory:")
+   c = conn.cursor()
+   c.execute('''create table stocks
+   (date text, trans text, symbol text,
+    qty real, price real)''')
+   c.execute("""insert into stocks
+             values ('2006-01-05','BUY','RHAT',100,35.14)""")
+   conn.commit()
+   c.close()
 
 Now we plug :class:`Row` in::
 
-    >>> conn.row_factory = sqlite3.Row
-    >>> c = conn.cursor()
-    >>> c.execute('select * from stocks')
-    <sqlite3.Cursor object at 0x7f4e7dd8fa80>
-    >>> r = c.fetchone()
-    >>> type(r)
-    <class 'sqlite3.Row'>
-    >>> tuple(r)
-    ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14)
-    >>> len(r)
-    5
-    >>> r[2]
-    'RHAT'
-    >>> r.keys()
-    ['date', 'trans', 'symbol', 'qty', 'price']
-    >>> r['qty']
-    100.0
-    >>> for member in r:
-    ...     print(member)
-    ...
-    2006-01-05
-    BUY
-    RHAT
-    100.0
-    35.14
+   >>> conn.row_factory = sqlite3.Row
+   >>> c = conn.cursor()
+   >>> c.execute('select * from stocks')
+   <sqlite3.Cursor object at 0x7f4e7dd8fa80>
+   >>> r = c.fetchone()
+   >>> type(r)
+   <class 'sqlite3.Row'>
+   >>> tuple(r)
+   ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14)
+   >>> len(r)
+   5
+   >>> r[2]
+   'RHAT'
+   >>> r.keys()
+   ['date', 'trans', 'symbol', 'qty', 'price']
+   >>> r['qty']
+   100.0
+   >>> for member in r:
+   ...     print(member)
+   ...
+   2006-01-05
+   BUY
+   RHAT
+   100.0
+   35.14
 
 
 .. _sqlite3-types:
@@ -886,6 +886,7 @@ only makes sense to call from a different thread.
 .. rubric:: Footnotes
 
 .. [#f1] The sqlite3 module is not built with loadable extension support by
-  default, because some platforms (notably Mac OS X) have SQLite libraries which
-  are compiled without this feature. To get loadable extension support, you must
-  pass --enable-loadable-sqlite-extensions to configure.
+   default, because some platforms (notably Mac OS X) have SQLite
+   libraries which are compiled without this feature. To get loadable
+   extension support, you must pass --enable-loadable-sqlite-extensions to
+   configure.

From aeaeefa88b323324f7ceaff2823881d86b98a280 Mon Sep 17 00:00:00 2001
From: Georg Brandl <georg@python.org>
Date: Sun, 3 Jul 2011 19:22:42 +0200
Subject: [PATCH 13/13] Remove mention of medical condition from the test
 suite.

---
 Lib/test/test_csv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py
index 8ca1e62c4ae..2cca0e964c5 100644
--- a/Lib/test/test_csv.py
+++ b/Lib/test/test_csv.py
@@ -459,20 +459,20 @@ class TestDialectExcel(TestCsvBase):
                                  '5', '6']])
 
     def test_quoted_quote(self):
-        self.readerAssertEqual('1,2,3,"""I see,"" said the blind man","as he picked up his hammer and saw"',
+        self.readerAssertEqual('1,2,3,"""I see,"" said the happy man","as he picked up his hammer and saw"',
                                [['1', '2', '3',
-                                 '"I see," said the blind man',
+                                 '"I see," said the happy man',
                                  'as he picked up his hammer and saw']])
 
     def test_quoted_nl(self):
         input = '''\
 1,2,3,"""I see,""
-said the blind man","as he picked up his
+said the happy man","as he picked up his
 hammer and saw"
 9,8,7,6'''
         self.readerAssertEqual(input,
                                [['1', '2', '3',
-                                   '"I see,"\nsaid the blind man',
+                                   '"I see,"\nsaid the happy man',
                                    'as he picked up his\nhammer and saw'],
                                 ['9','8','7','6']])