From 6aa2d1fec7debd8995b7ba0f3f27b6e532a91270 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 12 Aug 2008 08:35:52 +0000 Subject: [PATCH] Merged revisions 65459,65472,65481,65518,65536,65581,65609,65637,65641,65644-65645 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r65459 | gregory.p.smith | 2008-08-04 00:13:29 +0000 (Mon, 04 Aug 2008) | 4 lines - Issue #1857: subprocess.Popen.poll gained an additional _deadstate keyword argument in python 2.5, this broke code that subclassed Popen to include its own poll method. Fixed my moving _deadstate to an _internal_poll method. ........ r65472 | andrew.kuchling | 2008-08-04 01:43:43 +0000 (Mon, 04 Aug 2008) | 3 lines Bug 3228: Explicitly supply the file mode to avoid creating executable files, and add corresponding tests. Possible 2.5 backport candidate ........ r65481 | gregory.p.smith | 2008-08-04 07:33:37 +0000 (Mon, 04 Aug 2008) | 22 lines Adds a sanity check to avoid a *very rare* infinite loop due to a corrupt tls key list data structure in the thread startup path. This change is a companion to r60148 which already successfully dealt with a similar issue on thread shutdown. In particular this loop has been observed happening from this call path: #0 in find_key () #1 in PyThread_set_key_value () #2 in _PyGILState_NoteThreadState () #3 in PyThreadState_New () #4 in t_bootstrap () #5 in pthread_start_thread () I don't know how this happens but it does, *very* rarely. On more than one hardware platform. I have not been able to reproduce it manually. (A flaky mutex implementation on the system in question is one hypothesis). As with r60148, the spinning we managed to observe in the wild was due to a single list element pointing back upon itself. ........ r65518 | mark.dickinson | 2008-08-04 21:30:09 +0000 (Mon, 04 Aug 2008) | 7 lines Issue #1481296: (again!) Make conversion of a float NaN to an int or long raise ValueError instead of returning 0. Also, change the error message for conversion of an infinity to an integer, replacing 'long' by 'integer', so that it's appropriate for both long(float('inf')) and int(float('inf')). ........ r65536 | andrew.kuchling | 2008-08-05 01:00:57 +0000 (Tue, 05 Aug 2008) | 1 line Bug 3228: take a test from Niels Gustaebel's patch, and based on his patch, check for having os.stat available ........ r65581 | guido.van.rossum | 2008-08-07 18:51:38 +0000 (Thu, 07 Aug 2008) | 3 lines Patch by Ian Charnas from issue 3517. Add F_FULLFSYNC if it exists (OS X only so far). ........ r65609 | antoine.pitrou | 2008-08-09 17:22:25 +0000 (Sat, 09 Aug 2008) | 3 lines #3205: bz2 iterator fails silently on MemoryError ........ r65637 | georg.brandl | 2008-08-11 09:07:59 +0000 (Mon, 11 Aug 2008) | 3 lines - Issue #3537: Fix an assertion failure when an empty but presized dict object was stored in the freelist. ........ r65641 | jesse.noller | 2008-08-11 14:28:07 +0000 (Mon, 11 Aug 2008) | 2 lines Remove the fqdn call for issue 3270 ........ r65644 | antoine.pitrou | 2008-08-11 17:21:36 +0000 (Mon, 11 Aug 2008) | 3 lines #3134: shutil referenced undefined WindowsError symbol ........ r65645 | jesse.noller | 2008-08-11 19:00:15 +0000 (Mon, 11 Aug 2008) | 2 lines Fix the connection refused error part of issue 3419, use errno module instead of a static list of possible connection refused messages. ........ --- Lib/mailbox.py | 5 +-- Lib/multiprocessing/connection.py | 8 ++--- Lib/shutil.py | 14 ++++++--- Lib/subprocess.py | 12 +++++--- Lib/test/test_dict.py | 11 +++++++ Lib/test/test_long.py | 3 +- Lib/test/test_mailbox.py | 51 ++++++++++++++++++++++++++++++- Modules/bz2module.c | 1 + Modules/fcntlmodule.c | 5 +++ Objects/dictobject.c | 4 +++ Objects/longobject.c | 6 ++-- Python/pystate.c | 4 +++ Python/thread.c | 12 +++++++- 13 files changed, 115 insertions(+), 21 deletions(-) diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 38ff0b1eb74..ab915edea74 100755 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -394,7 +394,8 @@ class Maildir(Mailbox): result = Maildir(path, factory=self._factory) maildirfolder_path = os.path.join(path, 'maildirfolder') if not os.path.exists(maildirfolder_path): - os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY)) + os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY, + 0o666)) return result def remove_folder(self, folder): @@ -1900,7 +1901,7 @@ def _unlock_file(f): def _create_carefully(path): """Create a file if it doesn't exist and open for reading and writing.""" - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR) + fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) try: return open(path, 'r+', newline='') finally: diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index b6f5bde6e6d..b962060615b 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -11,6 +11,7 @@ __all__ = [ 'Client', 'Listener', 'Pipe' ] import os import sys import socket +import errno import time import tempfile import itertools @@ -215,10 +216,7 @@ class SocketListener(object): self._socket = socket.socket(getattr(socket, family)) self._socket.bind(address) self._socket.listen(backlog) - address = self._socket.getsockname() - if type(address) is tuple: - address = (socket.getfqdn(address[0]),) + address[1:] - self._address = address + self._address = self._socket.getsockname() self._family = family self._last_accepted = None @@ -253,7 +251,7 @@ def SocketClient(address): try: s.connect(address) except socket.error as e: - if e.args[0] != 10061: # 10061 => connection refused + if e.args[0] != errno.ECONNREFUSED: # connection refused debug('failed to connect to address %s', address) raise time.sleep(0.01) diff --git a/Lib/shutil.py b/Lib/shutil.py index 56ea7ec3ede..9a5f78a50de 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -16,6 +16,11 @@ __all__ = ["copyfileobj","copyfile","copymode","copystat","copy","copy2", class Error(EnvironmentError): pass +try: + WindowsError +except NameError: + WindowsError = None + def copyfileobj(fsrc, fdst, length=16*1024): """copy data from file-like object fsrc to file-like object fdst""" while 1: @@ -162,11 +167,12 @@ def copytree(src, dst, symlinks=False, ignore=None): errors.extend(err.args[0]) try: copystat(src, dst) - except WindowsError: - # can't copy file access times on Windows - pass except OSError as why: - errors.extend((src, dst, str(why))) + if WindowsError is not None and isinstance(why, WindowsError): + # Copying file access times may fail on Windows + pass + else: + errors.extend((src, dst, str(why))) if errors: raise Error(errors) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index e94fc2c6fff..d3e96924221 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -375,7 +375,7 @@ _active = [] def _cleanup(): for inst in _active[:]: - res = inst.poll(_deadstate=sys.maxsize) + res = inst._internal_poll(_deadstate=sys.maxsize) if res is not None and res >= 0: try: _active.remove(inst) @@ -635,7 +635,7 @@ class Popen(object): # We didn't get to successfully create a child process. return # In case the child hasn't been waited on, check if it's done. - self.poll(_deadstate=sys.maxsize) + self._internal_poll(_deadstate=sys.maxsize) if self.returncode is None and _active is not None: # Child is still running, keep us alive until we can wait on it. _active.append(self) @@ -671,6 +671,10 @@ class Popen(object): return self._communicate(input) + def poll(self): + return self._internal_poll() + + if mswindows: # # Windows methods @@ -842,7 +846,7 @@ class Popen(object): errwrite.Close() - def poll(self, _deadstate=None): + def _internal_poll(self, _deadstate=None): """Check if child process has terminated. Returns returncode attribute.""" if self.returncode is None: @@ -1103,7 +1107,7 @@ class Popen(object): raise RuntimeError("Unknown child exit status!") - def poll(self, _deadstate=None): + def _internal_poll(self, _deadstate=None): """Check if child process has terminated. Returns returncode attribute.""" if self.returncode is None: diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b518a0bb02b..403d5ebf86c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -638,6 +638,17 @@ class DictTest(unittest.TestCase): resizing = True d[9] = 6 + def test_empty_presized_dict_in_freelist(self): + # Bug #3537: if an empty but presized dict with a size larger + # than 7 was in the freelist, it triggered an assertion failure + try: + d = {'a': 1/0, 'b': None, 'c': None, 'd': None, 'e': None, + 'f': None, 'g': None, 'h': None} + except ZeroDivisionError: + pass + d = {} + + from test import mapping_tests diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index c47587888af..dc04badf20a 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -768,7 +768,8 @@ class LongTest(unittest.TestCase): def test_nan_inf(self): self.assertRaises(OverflowError, int, float('inf')) - self.assertRaises(OverflowError, int, float('nan')) + self.assertRaises(OverflowError, int, float('-inf')) + self.assertRaises(ValueError, int, float('nan')) def test_true_division(self): huge = 1 << 40000 diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 783323fb76d..182838118d1 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -709,6 +709,38 @@ class TestMaildir(TestMailbox): for msg in self._box: pass + def test_file_permissions(self): + # Verify that message files are created without execute permissions + if not hasattr(os, "stat") or not hasattr(os, "umask"): + return + msg = mailbox.MaildirMessage(self._template % 0) + orig_umask = os.umask(0) + try: + key = self._box.add(msg) + finally: + os.umask(orig_umask) + path = os.path.join(self._path, self._box._lookup(key)) + mode = os.stat(path).st_mode + self.assert_(mode & 0o111 == 0) + + def test_folder_file_perms(self): + # From bug #3228, we want to verify that the file created inside a Maildir + # subfolder isn't marked as executable. + if not hasattr(os, "stat") or not hasattr(os, "umask"): + return + + orig_umask = os.umask(0) + try: + subfolder = self._box.add_folder('subfolder') + finally: + os.umask(orig_umask) + + path = os.path.join(subfolder._path, 'maildirfolder') + st = os.stat(path) + perms = st.st_mode + self.assertFalse((perms & 0o111)) # Execute bits should all be off. + + class _TestMboxMMDF(TestMailbox): def tearDown(self): @@ -800,11 +832,28 @@ class _TestMboxMMDF(TestMailbox): self._box.close() - class TestMbox(_TestMboxMMDF): _factory = lambda self, path, factory=None: mailbox.mbox(path, factory) + def test_file_perms(self): + # From bug #3228, we want to verify that the mailbox file isn't executable, + # even if the umask is set to something that would leave executable bits set. + # We only run this test on platforms that support umask. + if hasattr(os, 'umask') and hasattr(os, 'stat'): + try: + old_umask = os.umask(0o077) + self._box.close() + os.unlink(self._path) + self._box = mailbox.mbox(self._path, create=True) + self._box.add('') + self._box.close() + finally: + os.umask(old_umask) + + st = os.stat(self._path) + perms = st.st_mode + self.assertFalse((perms & 0o111)) # Execute bits should all be off. class TestMMDF(_TestMboxMMDF): diff --git a/Modules/bz2module.c b/Modules/bz2module.c index 562198be05b..c6819914153 100644 --- a/Modules/bz2module.c +++ b/Modules/bz2module.c @@ -317,6 +317,7 @@ Util_ReadAhead(BZ2FileObject *f, int bufsize) return 0; } if ((f->f_buf = PyMem_Malloc(bufsize)) == NULL) { + PyErr_NoMemory(); return -1; } Py_BEGIN_ALLOW_THREADS diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 353889c429d..46bf4d11722 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -530,6 +530,11 @@ all_ins(PyObject* d) if (ins(d, "F_SHLCK", (long)F_SHLCK)) return -1; #endif +/* OS X (and maybe others) let you tell the storage device to flush to physical media */ +#ifdef F_FULLFSYNC + if (ins(d, "F_FULLFSYNC", (long)F_FULLFSYNC)) return -1; +#endif + /* For F_{GET|SET}FL */ #ifdef FD_CLOEXEC if (ins(d, "FD_CLOEXEC", (long)FD_CLOEXEC)) return -1; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 828803c0912..6f3ba1bea2a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -242,6 +242,10 @@ PyDict_New(void) _Py_NewReference((PyObject *)mp); if (mp->ma_fill) { EMPTY_TO_MINSIZE(mp); + } else { + /* At least set ma_table and ma_mask; these are wrong + if an empty but presized dict is added to freelist */ + INIT_NONZERO_DICT_SLOTS(mp); } assert (mp->ma_used == 0); assert (mp->ma_table == mp->ma_smalltable); diff --git a/Objects/longobject.c b/Objects/longobject.c index ae62cfd423f..3aa518b9175 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -271,12 +271,12 @@ PyLong_FromDouble(double dval) neg = 0; if (Py_IS_INFINITY(dval)) { PyErr_SetString(PyExc_OverflowError, - "cannot convert float infinity to int"); + "cannot convert float infinity to integer"); return NULL; } if (Py_IS_NAN(dval)) { - PyErr_SetString(PyExc_OverflowError, - "cannot convert float NaN to int"); + PyErr_SetString(PyExc_ValueError, + "cannot convert float NaN to integer"); return NULL; } if (dval < 0.0) { diff --git a/Python/pystate.c b/Python/pystate.c index c841eb31eee..94792567187 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -291,6 +291,10 @@ tstate_delete_common(PyThreadState *tstate) "PyThreadState_Delete: invalid tstate"); if (*p == tstate) break; + /* Sanity check. These states should never happen but if + * they do we must abort. Otherwise we'll end up spinning in + * in a tight loop with the lock held. A similar check is done + * in thread.c find_key(). */ if (*p == prev_p) Py_FatalError( "PyThreadState_Delete: small circular list(!)" diff --git a/Python/thread.c b/Python/thread.c index 41fa1e63139..d8cadba3670 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -260,15 +260,25 @@ static int nkeys = 0; /* PyThread_create_key() hands out nkeys+1 next */ static struct key * find_key(int key, void *value) { - struct key *p; + struct key *p, *prev_p; long id = PyThread_get_thread_ident(); if (!keymutex) return NULL; PyThread_acquire_lock(keymutex, 1); + prev_p = NULL; for (p = keyhead; p != NULL; p = p->next) { if (p->id == id && p->key == key) goto Done; + /* Sanity check. These states should never happen but if + * they do we must abort. Otherwise we'll end up spinning in + * in a tight loop with the lock held. A similar check is done + * in pystate.c tstate_delete_common(). */ + if (p == prev_p) + Py_FatalError("tls find_key: small circular list(!)"); + prev_p = p; + if (p->next == keyhead) + Py_FatalError("tls find_key: circular list(!)"); } if (value == NULL) { assert(p == NULL);