From 455d46f0d921337405ebd3a098c210f3614ae77c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Wed, 9 Jul 2003 04:45:59 +0000 Subject: [PATCH] bsddb 4.1.6: * Extended DB & DBEnv set_get_returns_none functionality to take a "level" instead of a boolean flag. The boolean 0 and 1 values still have the same effect. A value of 2 extends the "return None instead of raising an exception" behaviour to the DBCursor set methods. This will become the default behaviour in pybsddb 4.2. * Fixed a typo in DBCursor.join_item method that made it crash instead of returning a value. Obviously nobody uses it. Wrote a test case for join and join_item. --- Lib/bsddb/dbobj.py | 8 ++ Lib/bsddb/dbtables.py | 3 + Lib/bsddb/test/test_associate.py | 2 +- Lib/bsddb/test/test_basics.py | 63 +++++++++++-- Lib/bsddb/test/test_join.py | 108 ++++++++++++++++++++++ Modules/_bsddb.c | 151 ++++++++++++++++++++++--------- 6 files changed, 284 insertions(+), 51 deletions(-) diff --git a/Lib/bsddb/dbobj.py b/Lib/bsddb/dbobj.py index b2632a1a087..d23f533f347 100644 --- a/Lib/bsddb/dbobj.py +++ b/Lib/bsddb/dbobj.py @@ -15,6 +15,12 @@ # implied. # +# +# TODO it would be *really nice* to have an automatic shadow class populator +# so that new methods don't need to be added here manually after being +# added to _bsddb.c. +# + import db try: @@ -57,6 +63,8 @@ class DBEnv: return apply(self._cobj.set_lk_max_objects, args, kwargs) def set_mp_mmapsize(self, *args, **kwargs): return apply(self._cobj.set_mp_mmapsize, args, kwargs) + def set_timeout(self, *args, **kwargs): + return apply(self._cobj.set_timeout, args, kwargs) def set_tmp_dir(self, *args, **kwargs): return apply(self._cobj.set_tmp_dir, args, kwargs) def txn_begin(self, *args, **kwargs): diff --git a/Lib/bsddb/dbtables.py b/Lib/bsddb/dbtables.py index 85dbb4aa3b5..d052ca5b937 100644 --- a/Lib/bsddb/dbtables.py +++ b/Lib/bsddb/dbtables.py @@ -155,6 +155,9 @@ class bsdTableDB : if truncate: myflags |= DB_TRUNCATE self.db = DB(self.env) + # this code relies on DBCursor.set* methods to raise exceptions + # rather than returning None + self.db.set_get_returns_none(1) # allow duplicate entries [warning: be careful w/ metadata] self.db.set_flags(DB_DUP) self.db.open(filename, DB_BTREE, dbflags | myflags, mode) diff --git a/Lib/bsddb/test/test_associate.py b/Lib/bsddb/test/test_associate.py index 3061e4591a7..5fe41a93c41 100644 --- a/Lib/bsddb/test/test_associate.py +++ b/Lib/bsddb/test/test_associate.py @@ -1,5 +1,5 @@ """ -TestCases for multi-threaded access to a DB. +TestCases for DB.associate. """ import sys, os, string diff --git a/Lib/bsddb/test/test_basics.py b/Lib/bsddb/test/test_basics.py index 224aef40449..dbab2313b08 100644 --- a/Lib/bsddb/test/test_basics.py +++ b/Lib/bsddb/test/test_basics.py @@ -282,11 +282,11 @@ class BasicTestCase(unittest.TestCase): #---------------------------------------- - def test03_SimpleCursorStuff(self): + def test03_SimpleCursorStuff(self, get_raises_error=0, set_raises_error=1): if verbose: print '\n', '-=' * 30 - print "Running %s.test03_SimpleCursorStuff..." % \ - self.__class__.__name__ + print "Running %s.test03_SimpleCursorStuff (get_error %s, set_error %s)..." % \ + (self.__class__.__name__, get_raises_error, set_raises_error) if self.env and self.dbopenflags & db.DB_AUTO_COMMIT: txn = self.env.txn_begin() @@ -300,7 +300,15 @@ class BasicTestCase(unittest.TestCase): count = count + 1 if verbose and count % 100 == 0: print rec - rec = c.next() + try: + rec = c.next() + except db.DBNotFoundError, val: + if get_raises_error: + assert val[0] == db.DB_NOTFOUND + if verbose: print val + rec = None + else: + self.fail("unexpected DBNotFoundError") assert count == 1000 @@ -311,7 +319,15 @@ class BasicTestCase(unittest.TestCase): count = count + 1 if verbose and count % 100 == 0: print rec - rec = c.prev() + try: + rec = c.prev() + except db.DBNotFoundError, val: + if get_raises_error: + assert val[0] == db.DB_NOTFOUND + if verbose: print val + rec = None + else: + self.fail("unexpected DBNotFoundError") assert count == 1000 @@ -322,23 +338,29 @@ class BasicTestCase(unittest.TestCase): assert rec[1] == self.makeData('0505') try: - c.set('bad key') + n = c.set('bad key') except db.DBNotFoundError, val: assert val[0] == db.DB_NOTFOUND if verbose: print val else: - self.fail("expected exception") + if set_raises_error: + self.fail("expected exception") + if n != None: + self.fail("expected None: "+`n`) rec = c.get_both('0404', self.makeData('0404')) assert rec == ('0404', self.makeData('0404')) try: - c.get_both('0404', 'bad data') + n = c.get_both('0404', 'bad data') except db.DBNotFoundError, val: assert val[0] == db.DB_NOTFOUND if verbose: print val else: - self.fail("expected exception") + if get_raises_error: + self.fail("expected exception") + if n != None: + self.fail("expected None: "+`n`) if self.d.get_type() == db.DB_BTREE: rec = c.set_range('011') @@ -414,6 +436,29 @@ class BasicTestCase(unittest.TestCase): # SF pybsddb bug id 667343 del oldcursor + def test03b_SimpleCursorWithoutGetReturnsNone0(self): + # same test but raise exceptions instead of returning None + if verbose: + print '\n', '-=' * 30 + print "Running %s.test03b_SimpleCursorStuffWithoutGetReturnsNone..." % \ + self.__class__.__name__ + + old = self.d.set_get_returns_none(0) + assert old == 1 + self.test03_SimpleCursorStuff(get_raises_error=1, set_raises_error=1) + + def test03c_SimpleCursorGetReturnsNone2(self): + # same test but raise exceptions instead of returning None + if verbose: + print '\n', '-=' * 30 + print "Running %s.test03c_SimpleCursorStuffWithoutSetReturnsNone..." % \ + self.__class__.__name__ + + old = self.d.set_get_returns_none(2) + assert old == 1 + old = self.d.set_get_returns_none(2) + assert old == 2 + self.test03_SimpleCursorStuff(get_raises_error=0, set_raises_error=0) #---------------------------------------- diff --git a/Lib/bsddb/test/test_join.py b/Lib/bsddb/test/test_join.py index ab75ba19661..08784aceb15 100644 --- a/Lib/bsddb/test/test_join.py +++ b/Lib/bsddb/test/test_join.py @@ -1,9 +1,117 @@ """TestCases for using the DB.join and DBCursor.join_item methods. """ +import sys, os, string +import tempfile +import time +from pprint import pprint + +try: + from threading import Thread, currentThread + have_threads = 1 +except ImportError: + have_threads = 0 + import unittest +from test_all import verbose + +try: + # For Python 2.3 + from bsddb import db, dbshelve +except ImportError: + # For earlier Pythons w/distutils pybsddb + from bsddb3 import db, dbshelve + + +#---------------------------------------------------------------------- + +ProductIndex = [ + ('apple', "Convenience Store"), + ('blueberry', "Farmer's Market"), + ('shotgun', "S-Mart"), # Aisle 12 + ('pear', "Farmer's Market"), + ('chainsaw', "S-Mart"), # "Shop smart. Shop S-Mart!" + ('strawberry', "Farmer's Market"), +] + +ColorIndex = [ + ('blue', "blueberry"), + ('red', "apple"), + ('red', "chainsaw"), + ('red', "strawberry"), + ('yellow', "peach"), + ('yellow', "pear"), + ('black', "shotgun"), +] + +class JoinTestCase(unittest.TestCase): + keytype = '' + + def setUp(self): + self.filename = self.__class__.__name__ + '.db' + homeDir = os.path.join(os.path.dirname(sys.argv[0]), 'db_home') + self.homeDir = homeDir + try: os.mkdir(homeDir) + except os.error: pass + self.env = db.DBEnv() + self.env.open(homeDir, db.DB_CREATE | db.DB_INIT_MPOOL | db.DB_INIT_LOCK ) + + def tearDown(self): + self.env.close() + import glob + files = glob.glob(os.path.join(self.homeDir, '*')) + for file in files: + os.remove(file) + + def test01_join(self): + if verbose: + print '\n', '-=' * 30 + print "Running %s.test01_join..." % \ + self.__class__.__name__ + + # create and populate primary index + priDB = db.DB(self.env) + priDB.open(self.filename, "primary", db.DB_BTREE, db.DB_CREATE) + map(lambda t: apply(priDB.put, t), ProductIndex) + + # create and populate secondary index + secDB = db.DB(self.env) + secDB.set_flags(db.DB_DUP | db.DB_DUPSORT) + secDB.open(self.filename, "secondary", db.DB_BTREE, db.DB_CREATE) + map(lambda t: apply(secDB.put, t), ColorIndex) + + sCursor = None + jCursor = None + try: + # lets look up all of the red Products + sCursor = secDB.cursor() + assert sCursor.set('red') + + # FIXME: jCursor doesn't properly hold a reference to its + # cursors, if they are closed before jcursor is used it + # can cause a crash. + jCursor = priDB.join([sCursor]) + + if jCursor.get(0) != ('apple', "Convenience Store"): + self.fail("join cursor positioned wrong") + if jCursor.join_item() != 'chainsaw': + self.fail("DBCursor.join_item returned wrong item") + if jCursor.get(0)[0] != 'strawberry': + self.fail("join cursor returned wrong thing") + if jCursor.get(0): # there were only three red items to return + self.fail("join cursor returned too many items") + finally: + if jCursor: + jCursor.close() + if sCursor: + sCursor.close() + priDB.close() + secDB.close() def test_suite(): suite = unittest.TestSuite() + + suite.addTest(unittest.makeSuite(JoinTestCase)) + return suite diff --git a/Modules/_bsddb.c b/Modules/_bsddb.c index b74491ce323..227fe7bdf58 100644 --- a/Modules/_bsddb.c +++ b/Modules/_bsddb.c @@ -93,7 +93,7 @@ /* 40 = 4.0, 33 = 3.3; this will break if the second number is > 9 */ #define DBVER (DB_VERSION_MAJOR * 10 + DB_VERSION_MINOR) -#define PY_BSDDB_VERSION "4.1.5" +#define PY_BSDDB_VERSION "4.1.6" static char *rcs_id = "$Id$"; @@ -141,12 +141,6 @@ static PyInterpreterState* _db_interpreterState = NULL; #endif - -/* What is the default behaviour when DB->get or DBCursor->get returns a - DB_NOTFOUND error? Return None or raise an exception? */ -#define GET_RETURNS_NONE_DEFAULT 1 - - /* Should DB_INCOMPLETE be turned into a warning or an exception? */ #define INCOMPLETE_IS_WARNING 1 @@ -189,12 +183,24 @@ static PyObject* DBPermissionsError; /* EPERM */ /* --------------------------------------------------------------------- */ /* Structure definitions */ +struct behaviourFlags { + /* What is the default behaviour when DB->get or DBCursor->get returns a + DB_NOTFOUND error? Return None or raise an exception? */ + unsigned int getReturnsNone : 1; + /* What is the default behaviour for DBCursor.set* methods when DBCursor->get + * returns a DB_NOTFOUND error? Return None or raise an exception? */ + unsigned int cursorSetReturnsNone : 1; +}; + +#define DEFAULT_GET_RETURNS_NONE 1 +#define DEFAULT_CURSOR_SET_RETURNS_NONE 0 /* 0 in pybsddb < 4.2, python < 2.4 */ + typedef struct { PyObject_HEAD DB_ENV* db_env; u_int32_t flags; /* saved flags from open() */ int closed; - int getReturnsNone; + struct behaviourFlags moduleFlags; } DBEnvObject; @@ -205,7 +211,7 @@ typedef struct { u_int32_t flags; /* saved flags from open() */ u_int32_t setflags; /* saved flags from set_flags() */ int haveStat; - int getReturnsNone; + struct behaviourFlags moduleFlags; #if (DBVER >= 33) PyObject* associateCallback; int primaryDBType; @@ -595,7 +601,7 @@ static PyObject* _DBCursor_get(DBCursorObject* self, int extra_flags, err = self->dbc->c_get(self->dbc, &key, &data, flags); MYDB_END_ALLOW_THREADS; - if ((err == DB_NOTFOUND) && self->mydb->getReturnsNone) { + if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) { Py_INCREF(Py_None); retval = Py_None; } @@ -681,9 +687,10 @@ newDBObject(DBEnvObject* arg, int flags) } if (self->myenvobj) - self->getReturnsNone = self->myenvobj->getReturnsNone; + self->moduleFlags = self->myenvobj->moduleFlags; else - self->getReturnsNone = GET_RETURNS_NONE_DEFAULT; + self->moduleFlags.getReturnsNone = DEFAULT_GET_RETURNS_NONE; + self->moduleFlags.cursorSetReturnsNone = DEFAULT_CURSOR_SET_RETURNS_NONE; MYDB_BEGIN_ALLOW_THREADS; err = db_create(&self->db, db_env, flags); @@ -797,7 +804,8 @@ newDBEnvObject(int flags) self->closed = 1; self->flags = flags; - self->getReturnsNone = GET_RETURNS_NONE_DEFAULT; + self->moduleFlags.getReturnsNone = DEFAULT_GET_RETURNS_NONE; + self->moduleFlags.cursorSetReturnsNone = DEFAULT_CURSOR_SET_RETURNS_NONE; MYDB_BEGIN_ALLOW_THREADS; err = db_env_create(&self->db_env, flags); @@ -1182,7 +1190,7 @@ _DB_consume(DBObject* self, PyObject* args, PyObject* kwargs, int consume_flag) err = self->db->get(self->db, txn, &key, &data, flags|consume_flag); MYDB_END_ALLOW_THREADS; - if ((err == DB_NOTFOUND) && self->getReturnsNone) { + if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) { err = 0; Py_INCREF(Py_None); retval = Py_None; @@ -1324,7 +1332,7 @@ DB_get(DBObject* self, PyObject* args, PyObject* kwargs) Py_INCREF(dfltobj); retval = dfltobj; } - else if ((err == DB_NOTFOUND) && self->getReturnsNone) { + else if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) { err = 0; Py_INCREF(Py_None); retval = Py_None; @@ -1424,7 +1432,7 @@ DB_get_both(DBObject* self, PyObject* args, PyObject* kwargs) err = self->db->get(self->db, txn, &key, &data, flags); MYDB_END_ALLOW_THREADS; - if ((err == DB_NOTFOUND) && self->getReturnsNone) { + if ((err == DB_NOTFOUND) && self->moduleFlags.getReturnsNone) { err = 0; Py_INCREF(Py_None); retval = Py_None; @@ -1525,6 +1533,11 @@ DB_join(DBObject* self, PyObject* args) free(cursors); RETURN_IF_ERR(); + // FIXME: this is a buggy interface. The returned cursor + // contains internal references to the passed in cursors + // but does not hold python references to them or prevent + // them from being closed prematurely. This can cause + // python to crash when things are done in the wrong order. return (PyObject*) newDBCursorObject(dbc, self); } @@ -2156,14 +2169,18 @@ static PyObject* DB_set_get_returns_none(DBObject* self, PyObject* args) { int flags=0; - int oldValue; + int oldValue=0; if (!PyArg_ParseTuple(args,"i:set_get_returns_none", &flags)) return NULL; CHECK_DB_NOT_CLOSED(self); - oldValue = self->getReturnsNone; - self->getReturnsNone = flags; + if (self->moduleFlags.getReturnsNone) + ++oldValue; + if (self->moduleFlags.cursorSetReturnsNone) + ++oldValue; + self->moduleFlags.getReturnsNone = (flags >= 1); + self->moduleFlags.cursorSetReturnsNone = (flags >= 2); return PyInt_FromLong(oldValue); } @@ -2643,7 +2660,7 @@ DBC_get(DBCursorObject* self, PyObject* args, PyObject *kwargs) MYDB_END_ALLOW_THREADS; - if ((err == DB_NOTFOUND) && self->mydb->getReturnsNone) { + if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) { Py_INCREF(Py_None); retval = Py_None; } @@ -2790,7 +2807,11 @@ DBC_set(DBCursorObject* self, PyObject* args, PyObject *kwargs) MYDB_BEGIN_ALLOW_THREADS; err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET); MYDB_END_ALLOW_THREADS; - if (makeDBError(err)) { + if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) { + Py_INCREF(Py_None); + retval = Py_None; + } + else if (makeDBError(err)) { retval = NULL; } else { @@ -2848,7 +2869,11 @@ DBC_set_range(DBCursorObject* self, PyObject* args, PyObject* kwargs) MYDB_BEGIN_ALLOW_THREADS; err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET_RANGE); MYDB_END_ALLOW_THREADS; - if (makeDBError(err)) { + if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) { + Py_INCREF(Py_None); + retval = Py_None; + } + else if (makeDBError(err)) { retval = NULL; } else { @@ -2875,19 +2900,15 @@ DBC_set_range(DBCursorObject* self, PyObject* args, PyObject* kwargs) return retval; } - static PyObject* -DBC_get_both(DBCursorObject* self, PyObject* args) +_DBC_get_set_both(DBCursorObject* self, PyObject* keyobj, PyObject* dataobj, + int flags, unsigned int returnsNone) { - int err, flags=0; + int err; DBT key, data; - PyObject* retval, *keyobj, *dataobj; - - if (!PyArg_ParseTuple(args, "OO|i:get_both", &keyobj, &dataobj, &flags)) - return NULL; - - CHECK_CURSOR_NOT_CLOSED(self); + PyObject* retval; + // the caller did this: CHECK_CURSOR_NOT_CLOSED(self); if (!make_key_dbt(self->mydb, keyobj, &key, NULL)) return NULL; if (!make_dbt(dataobj, &data)) @@ -2896,7 +2917,11 @@ DBC_get_both(DBCursorObject* self, PyObject* args) MYDB_BEGIN_ALLOW_THREADS; err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_GET_BOTH); MYDB_END_ALLOW_THREADS; - if (makeDBError(err)) { + if ((err == DB_NOTFOUND) && returnsNone) { + Py_INCREF(Py_None); + retval = Py_None; + } + else if (makeDBError(err)) { retval = NULL; } else { @@ -2922,6 +2947,38 @@ DBC_get_both(DBCursorObject* self, PyObject* args) return retval; } +static PyObject* +DBC_get_both(DBCursorObject* self, PyObject* args) +{ + int flags=0; + PyObject *keyobj, *dataobj; + + if (!PyArg_ParseTuple(args, "OO|i:get_both", &keyobj, &dataobj, &flags)) + return NULL; + + // if the cursor is closed, self->mydb may be invalid + CHECK_CURSOR_NOT_CLOSED(self); + + return _DBC_get_set_both(self, keyobj, dataobj, flags, + self->mydb->moduleFlags.getReturnsNone); +} + +static PyObject* +DBC_set_both(DBCursorObject* self, PyObject* args) +{ + int flags=0; + PyObject *keyobj, *dataobj; + + if (!PyArg_ParseTuple(args, "OO|i:set_both", &keyobj, &dataobj, &flags)) + return NULL; + + // if the cursor is closed, self->mydb may be invalid + CHECK_CURSOR_NOT_CLOSED(self); + + return _DBC_get_set_both(self, keyobj, dataobj, flags, + self->mydb->moduleFlags.cursorSetReturnsNone); +} + static PyObject* DBC_set_recno(DBCursorObject* self, PyObject* args, PyObject *kwargs) @@ -2965,7 +3022,11 @@ DBC_set_recno(DBCursorObject* self, PyObject* args, PyObject *kwargs) MYDB_BEGIN_ALLOW_THREADS; err = self->dbc->c_get(self->dbc, &key, &data, flags|DB_SET_RECNO); MYDB_END_ALLOW_THREADS; - if (makeDBError(err)) { + if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.cursorSetReturnsNone) { + Py_INCREF(Py_None); + retval = Py_None; + } + else if (makeDBError(err)) { retval = NULL; } else { /* Can only be used for BTrees, so no need to return int key */ @@ -3010,11 +3071,11 @@ DBC_prev_nodup(DBCursorObject* self, PyObject* args, PyObject *kwargs) static PyObject* DBC_join_item(DBCursorObject* self, PyObject* args) { - int err; + int err, flags=0; DBT key, data; PyObject* retval; - if (!PyArg_ParseTuple(args, ":join_item")) + if (!PyArg_ParseTuple(args, "|i:join_item", &flags)) return NULL; CHECK_CURSOR_NOT_CLOSED(self); @@ -3027,9 +3088,13 @@ DBC_join_item(DBCursorObject* self, PyObject* args) } MYDB_BEGIN_ALLOW_THREADS; - err = self->dbc->c_get(self->dbc, &key, &data, DB_JOIN_ITEM); + err = self->dbc->c_get(self->dbc, &key, &data, flags | DB_JOIN_ITEM); MYDB_END_ALLOW_THREADS; - if (makeDBError(err)) { + if ((err == DB_NOTFOUND) && self->mydb->moduleFlags.getReturnsNone) { + Py_INCREF(Py_None); + retval = Py_None; + } + else if (makeDBError(err)) { retval = NULL; } else { @@ -3748,14 +3813,18 @@ static PyObject* DBEnv_set_get_returns_none(DBEnvObject* self, PyObject* args) { int flags=0; - int oldValue; + int oldValue=0; if (!PyArg_ParseTuple(args,"i:set_get_returns_none", &flags)) return NULL; CHECK_ENV_NOT_CLOSED(self); - oldValue = self->getReturnsNone; - self->getReturnsNone = flags; + if (self->moduleFlags.getReturnsNone) + ++oldValue; + if (self->moduleFlags.cursorSetReturnsNone) + ++oldValue; + self->moduleFlags.getReturnsNone = (flags >= 1); + self->moduleFlags.cursorSetReturnsNone = (flags >= 2); return PyInt_FromLong(oldValue); } @@ -3977,7 +4046,7 @@ static PyMethodDef DBCursor_methods[] = { {"set", (PyCFunction)DBC_set, METH_VARARGS|METH_KEYWORDS}, {"set_range", (PyCFunction)DBC_set_range, METH_VARARGS|METH_KEYWORDS}, {"get_both", (PyCFunction)DBC_get_both, METH_VARARGS}, - {"set_both", (PyCFunction)DBC_get_both, METH_VARARGS}, + {"set_both", (PyCFunction)DBC_set_both, METH_VARARGS}, {"set_recno", (PyCFunction)DBC_set_recno, METH_VARARGS|METH_KEYWORDS}, {"consume", (PyCFunction)DBC_consume, METH_VARARGS|METH_KEYWORDS}, {"next_dup", (PyCFunction)DBC_next_dup, METH_VARARGS|METH_KEYWORDS},