Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Fix issue2669: bsddb simple/legacy interface iteration silently fails 

when database changes size during iteration.

It now behaves like a dictionary, the next attempt to get a value from
the iterator after the database has changed size will raise a RuntimeError.
This commit is contained in:
Gregory P. Smith 2008-05-25 08:28:29 +00:00
parent e08e3d0686
commit 9e6468be1d
2 changed files with 122 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
Lib/bsddb/__init__.py
View File

@ -33,7 +33,7 @@
#---------------------------------------------------------------------- #----------------------------------------------------------------------
"""Support for Berkeley DB 3.3 through 4.6 with a simple interface. """Support for Berkeley DB 4.x with a simple interface.
For the full featured object oriented interface use the bsddb.db module For the full featured object oriented interface use the bsddb.db module
instead. It mirrors the Oracle Berkeley DB C API. instead. It mirrors the Oracle Berkeley DB C API.
@ -66,13 +66,8 @@ error = db.DBError # So bsddb.error will mean something...
import sys, os import sys, os
# for backwards compatibility with python versions older than 2.3, the import UserDict
# iterator interface is dynamically defined and added using a mixin from weakref import ref
# class. old python can't tokenize it due to the yield keyword.
if sys.version >= '2.3':
import UserDict
from weakref import ref
exec """
class _iter_mixin(UserDict.DictMixin): class _iter_mixin(UserDict.DictMixin):
def _make_iter_cursor(self): def _make_iter_cursor(self):
cur = _DeadlockWrap(self.db.cursor) cur = _DeadlockWrap(self.db.cursor)
@ -87,67 +82,80 @@ class _iter_mixin(UserDict.DictMixin):
return lambda ref: self._cursor_refs.pop(key, None) return lambda ref: self._cursor_refs.pop(key, None)
def __iter__(self): def __iter__(self):
self._kill_iteration = False
self._in_iter += 1
try: try:
cur = self._make_iter_cursor() try:
cur = self._make_iter_cursor()
# FIXME-20031102-greg: race condition. cursor could # FIXME-20031102-greg: race condition. cursor could
# be closed by another thread before this call. # be closed by another thread before this call.
# since we're only returning keys, we call the cursor # since we're only returning keys, we call the cursor
# methods with flags=0, dlen=0, dofs=0 # methods with flags=0, dlen=0, dofs=0
key = _DeadlockWrap(cur.first, 0,0,0)[0] key = _DeadlockWrap(cur.first, 0,0,0)[0]
yield key yield key
next = cur.next next = cur.next
while 1: while 1:
try: try:
key = _DeadlockWrap(next, 0,0,0)[0] key = _DeadlockWrap(next, 0,0,0)[0]
yield key yield key
except _bsddb.DBCursorClosedError: except _bsddb.DBCursorClosedError:
cur = self._make_iter_cursor() if self._kill_iteration:
# FIXME-20031101-greg: race condition. cursor could raise RuntimeError('Database changed size '
# be closed by another thread before this call. 'during iteration.')
_DeadlockWrap(cur.set, key,0,0,0) cur = self._make_iter_cursor()
next = cur.next # FIXME-20031101-greg: race condition. cursor could
except _bsddb.DBNotFoundError: # be closed by another thread before this call.
return _DeadlockWrap(cur.set, key,0,0,0)
except _bsddb.DBCursorClosedError: next = cur.next
# the database was modified during iteration. abort. except _bsddb.DBNotFoundError:
return pass
except _bsddb.DBCursorClosedError:
# the database was modified during iteration. abort.
pass
finally:
self._in_iter -= 1
def iteritems(self): def iteritems(self):
if not self.db: if not self.db:
return return
self._kill_iteration = False
self._in_iter += 1
try: try:
cur = self._make_iter_cursor() try:
cur = self._make_iter_cursor()
# FIXME-20031102-greg: race condition. cursor could # FIXME-20031102-greg: race condition. cursor could
# be closed by another thread before this call. # be closed by another thread before this call.
kv = _DeadlockWrap(cur.first) kv = _DeadlockWrap(cur.first)
key = kv[0] key = kv[0]
yield kv yield kv
next = cur.next next = cur.next
while 1: while 1:
try: try:
kv = _DeadlockWrap(next) kv = _DeadlockWrap(next)
key = kv[0] key = kv[0]
yield kv yield kv
except _bsddb.DBCursorClosedError: except _bsddb.DBCursorClosedError:
cur = self._make_iter_cursor() if self._kill_iteration:
# FIXME-20031101-greg: race condition. cursor could raise RuntimeError('Database changed size '
# be closed by another thread before this call. 'during iteration.')
_DeadlockWrap(cur.set, key,0,0,0) cur = self._make_iter_cursor()
next = cur.next # FIXME-20031101-greg: race condition. cursor could
except _bsddb.DBNotFoundError: # be closed by another thread before this call.
return _DeadlockWrap(cur.set, key,0,0,0)
except _bsddb.DBCursorClosedError: next = cur.next
# the database was modified during iteration. abort. except _bsddb.DBNotFoundError:
return pass
""" except _bsddb.DBCursorClosedError:
else: # the database was modified during iteration. abort.
class _iter_mixin: pass pass
finally:
self._in_iter -= 1
class _DBWithCursor(_iter_mixin): class _DBWithCursor(_iter_mixin):
@ -176,6 +184,8 @@ class _DBWithCursor(_iter_mixin):
# a collection of all DBCursor objects currently allocated # a collection of all DBCursor objects currently allocated
# by the _iter_mixin interface. # by the _iter_mixin interface.
self._cursor_refs = {} self._cursor_refs = {}
self._in_iter = 0
self._kill_iteration = False
def __del__(self): def __del__(self):
self.close() self.close()
@ -225,6 +235,8 @@ class _DBWithCursor(_iter_mixin):
def __setitem__(self, key, value): def __setitem__(self, key, value):
self._checkOpen() self._checkOpen()
self._closeCursors() self._closeCursors()
if self._in_iter and key not in self:
self._kill_iteration = True
def wrapF(): def wrapF():
self.db[key] = value self.db[key] = value
_DeadlockWrap(wrapF) # self.db[key] = value _DeadlockWrap(wrapF) # self.db[key] = value
@ -232,6 +244,8 @@ class _DBWithCursor(_iter_mixin):
def __delitem__(self, key): def __delitem__(self, key):
self._checkOpen() self._checkOpen()
self._closeCursors() self._closeCursors()
if self._in_iter and key in self:
self._kill_iteration = True
def wrapF(): def wrapF():
del self.db[key] del self.db[key]
_DeadlockWrap(wrapF) # del self.db[key] _DeadlockWrap(wrapF) # del self.db[key]

63
Lib/test/test_bsddb.py
View File

@ -66,9 +66,6 @@ class TestBSDDB(unittest.TestCase):
self.assertSetEquals(d.iteritems(), f.iteritems()) self.assertSetEquals(d.iteritems(), f.iteritems())
def test_iter_while_modifying_values(self): def test_iter_while_modifying_values(self):
if not hasattr(self.f, '__iter__'):
return
di = iter(self.d) di = iter(self.d)
while 1: while 1:
try: try:
@ -80,20 +77,62 @@ class TestBSDDB(unittest.TestCase):
# it should behave the same as a dict. modifying values # it should behave the same as a dict. modifying values
# of existing keys should not break iteration. (adding # of existing keys should not break iteration. (adding
# or removing keys should) # or removing keys should)
loops_left = len(self.f)
fi = iter(self.f) fi = iter(self.f)
while 1: while 1:
try: try:
key = fi.next() key = fi.next()
self.f[key] = 'modified '+key self.f[key] = 'modified '+key
loops_left -= 1
except StopIteration: except StopIteration:
break break
self.assertEqual(loops_left, 0)
self.test_mapping_iteration_methods() self.test_mapping_iteration_methods()
def test_iteritems_while_modifying_values(self): def test_iter_abort_on_changed_size(self):
if not hasattr(self.f, 'iteritems'): def DictIterAbort():
return di = iter(self.d)
while 1:
try:
di.next()
self.d['newkey'] = 'SPAM'
except StopIteration:
break
self.assertRaises(RuntimeError, DictIterAbort)
def DbIterAbort():
fi = iter(self.f)
while 1:
try:
fi.next()
self.f['newkey'] = 'SPAM'
except StopIteration:
break
self.assertRaises(RuntimeError, DbIterAbort)
def test_iteritems_abort_on_changed_size(self):
def DictIteritemsAbort():
di = self.d.iteritems()
while 1:
try:
di.next()
self.d['newkey'] = 'SPAM'
except StopIteration:
break
self.assertRaises(RuntimeError, DictIteritemsAbort)
def DbIteritemsAbort():
fi = self.f.iteritems()
while 1:
try:
key, value = fi.next()
del self.f[key]
except StopIteration:
break
self.assertRaises(RuntimeError, DbIteritemsAbort)
def test_iteritems_while_modifying_values(self):
di = self.d.iteritems() di = self.d.iteritems()
while 1: while 1:
try: try:
@ -105,13 +144,16 @@ class TestBSDDB(unittest.TestCase):
# it should behave the same as a dict. modifying values # it should behave the same as a dict. modifying values
# of existing keys should not break iteration. (adding # of existing keys should not break iteration. (adding
# or removing keys should) # or removing keys should)
loops_left = len(self.f)
fi = self.f.iteritems() fi = self.f.iteritems()
while 1: while 1:
try: try:
k, v = fi.next() k, v = fi.next()
self.f[k] = 'modified '+v self.f[k] = 'modified '+v
loops_left -= 1
except StopIteration: except StopIteration:
break break
self.assertEqual(loops_left, 0)
self.test_mapping_iteration_methods() self.test_mapping_iteration_methods()
@ -177,8 +219,8 @@ class TestBSDDB(unittest.TestCase):
# the database write and locking+threading support is enabled # the database write and locking+threading support is enabled
# the cursor's read lock will deadlock the write lock request.. # the cursor's read lock will deadlock the write lock request..
# test the iterator interface (if present) # test the iterator interface
if hasattr(self.f, 'iteritems'): if True:
if debug: print "D" if debug: print "D"
i = self.f.iteritems() i = self.f.iteritems()
k,v = i.next() k,v = i.next()
@ -213,10 +255,7 @@ class TestBSDDB(unittest.TestCase):
self.assert_(self.f[k], "be gone with ye deadlocks") self.assert_(self.f[k], "be gone with ye deadlocks")
def test_for_cursor_memleak(self): def test_for_cursor_memleak(self):
if not hasattr(self.f, 'iteritems'): # do the bsddb._DBWithCursor iterator internals leak cursors?
return
# do the bsddb._DBWithCursor _iter_mixin internals leak cursors?
nc1 = len(self.f._cursor_refs) nc1 = len(self.f._cursor_refs)
# create iterator # create iterator
i = self.f.iteritems() i = self.f.iteritems()