From ee475430d431814cbb6eb5e8a6c0ae51943349d4 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Fri, 15 Apr 2022 02:02:56 +0200 Subject: [PATCH] gh-69093: Support basic incremental I/O to blobs in `sqlite3` (GH-30680) Authored-by: Aviv Palivoda Co-authored-by: Erlend E. Aasland Co-authored-by: palaviv Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- Doc/includes/sqlite3/blob.py | 12 + Doc/library/sqlite3.rst | 66 ++++ Doc/whatsnew/3.11.rst | 4 + Lib/test/test_sqlite3/test_dbapi.py | 157 +++++++- .../2018-04-18-16-15-55.bpo-24905.jYqjYx.rst | 3 + Modules/_sqlite/blob.c | 351 ++++++++++++++++++ Modules/_sqlite/blob.h | 24 ++ Modules/_sqlite/clinic/blob.c.h | 165 ++++++++ Modules/_sqlite/clinic/connection.c.h | 106 +++++- Modules/_sqlite/connection.c | 87 ++++- Modules/_sqlite/connection.h | 5 +- Modules/_sqlite/module.c | 6 +- Modules/_sqlite/module.h | 1 + PCbuild/_sqlite3.vcxproj | 2 + PCbuild/_sqlite3.vcxproj.filters | 6 + setup.py | 1 + 16 files changed, 989 insertions(+), 7 deletions(-) create mode 100644 Doc/includes/sqlite3/blob.py create mode 100644 Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst create mode 100644 Modules/_sqlite/blob.c create mode 100644 Modules/_sqlite/blob.h create mode 100644 Modules/_sqlite/clinic/blob.c.h diff --git a/Doc/includes/sqlite3/blob.py b/Doc/includes/sqlite3/blob.py new file mode 100644 index 00000000000..61994fb82dd --- /dev/null +++ b/Doc/includes/sqlite3/blob.py @@ -0,0 +1,12 @@ +import sqlite3 + +con = sqlite3.connect(":memory:") +con.execute("create table test(blob_col blob)") +con.execute("insert into test(blob_col) values (zeroblob(10))") + +blob = con.blobopen("test", "blob_col", 1) +blob.write(b"Hello") +blob.write(b"World") +blob.seek(0) +print(blob.read()) # will print b"HelloWorld" +blob.close() diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 60dfbefd2e2..d0274fb7974 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -394,6 +394,20 @@ Connection Objects supplied, this must be a callable returning an instance of :class:`Cursor` or its subclasses. + .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") + + Open a :class:`Blob` handle to the :abbr:`BLOB (Binary Large OBject)` + located in row *row*, column *column*, table *table* of database *name*. + When *readonly* is :const:`True` the blob is opened without write + permissions. + + .. note:: + + The blob size cannot be changed using the :class:`Blob` class. + Use the SQL function ``zeroblob`` to create a blob with a fixed size. + + .. versionadded:: 3.11 + .. method:: commit() This method commits the current transaction. If you don't call this method, @@ -1088,6 +1102,58 @@ Exceptions transactions turned off. It is a subclass of :exc:`DatabaseError`. +.. _sqlite3-blob-objects: + +Blob Objects +------------ + +.. versionadded:: 3.11 + +.. class:: Blob + + A :class:`Blob` instance is a :term:`file-like object` that can read and write + data in an SQLite :abbr:`BLOB (Binary Large OBject)`. Call ``len(blob)`` to + get the size (number of bytes) of the blob. + + .. method:: close() + + Close the blob. + + The blob will be unusable from this point onward. An + :class:`~sqlite3.Error` (or subclass) exception will be raised if any + further operation is attempted with the blob. + + .. method:: read(length=-1, /) + + Read *length* bytes of data from the blob at the current offset position. + If the end of the blob is reached, the data up to + :abbr:`EOF (End of File)` will be returned. When *length* is not + specified, or is negative, :meth:`~Blob.read` will read until the end of + the blob. + + .. method:: write(data, /) + + Write *data* to the blob at the current offset. This function cannot + change the blob length. Writing beyond the end of the blob will raise + :exc:`ValueError`. + + .. method:: tell() + + Return the current access position of the blob. + + .. method:: seek(offset, origin=os.SEEK_SET, /) + + Set the current access position of the blob to *offset*. The *origin* + argument defaults to :data:`os.SEEK_SET` (absolute blob positioning). + Other values for *origin* are :data:`os.SEEK_CUR` (seek relative to the + current position) and :data:`os.SEEK_END` (seek relative to the blob’s + end). + + :class:`Blob` example: + + .. literalinclude:: ../includes/sqlite3/blob.py + + .. _sqlite3-types: SQLite and Python types diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index f8e86f6ba34..dba554cc834 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -393,6 +393,10 @@ sqlite3 :class:`sqlite3.Connection` for creating aggregate window functions. (Contributed by Erlend E. Aasland in :issue:`34916`.) +* Add :meth:`~sqlite3.Connection.blobopen` to :class:`sqlite3.Connection`. + :class:`sqlite3.Blob` allows incremental I/O operations on blobs. + (Contributed by Aviv Palivoda and Erlend E. Aasland in :issue:`24905`) + sys --- diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 2d2e58a3d44..faaa3713cb5 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -33,6 +33,8 @@ from test.support import ( check_disallow_instantiation, threading_helper, ) +from _testcapi import INT_MAX +from os import SEEK_SET, SEEK_CUR, SEEK_END from test.support.os_helper import TESTFN, unlink, temp_dir @@ -1041,11 +1043,163 @@ class CursorTests(unittest.TestCase): self.assertEqual(cu.fetchall(), [(1,)]) +class BlobTests(unittest.TestCase): + def setUp(self): + self.cx = sqlite.connect(":memory:") + self.cx.execute("create table test(b blob)") + self.data = b"this blob data string is exactly fifty bytes long!" + self.cx.execute("insert into test(b) values (?)", (self.data,)) + self.blob = self.cx.blobopen("test", "b", 1) + + def tearDown(self): + self.blob.close() + self.cx.close() + + def test_blob_seek_and_tell(self): + self.blob.seek(10) + self.assertEqual(self.blob.tell(), 10) + + self.blob.seek(10, SEEK_SET) + self.assertEqual(self.blob.tell(), 10) + + self.blob.seek(10, SEEK_CUR) + self.assertEqual(self.blob.tell(), 20) + + self.blob.seek(-10, SEEK_END) + self.assertEqual(self.blob.tell(), 40) + + def test_blob_seek_error(self): + msg_oor = "offset out of blob range" + msg_orig = "'origin' should be os.SEEK_SET, os.SEEK_CUR, or os.SEEK_END" + msg_of = "seek offset results in overflow" + + dataset = ( + (ValueError, msg_oor, lambda: self.blob.seek(1000)), + (ValueError, msg_oor, lambda: self.blob.seek(-10)), + (ValueError, msg_orig, lambda: self.blob.seek(10, -1)), + (ValueError, msg_orig, lambda: self.blob.seek(10, 3)), + ) + for exc, msg, fn in dataset: + with self.subTest(exc=exc, msg=msg, fn=fn): + self.assertRaisesRegex(exc, msg, fn) + + # Force overflow errors + self.blob.seek(1, SEEK_SET) + with self.assertRaisesRegex(OverflowError, msg_of): + self.blob.seek(INT_MAX, SEEK_CUR) + with self.assertRaisesRegex(OverflowError, msg_of): + self.blob.seek(INT_MAX, SEEK_END) + + def test_blob_read(self): + buf = self.blob.read() + self.assertEqual(buf, self.data) + + def test_blob_read_oversized(self): + buf = self.blob.read(len(self.data) * 2) + self.assertEqual(buf, self.data) + + def test_blob_read_advance_offset(self): + n = 10 + buf = self.blob.read(n) + self.assertEqual(buf, self.data[:n]) + self.assertEqual(self.blob.tell(), n) + + def test_blob_read_at_offset(self): + self.blob.seek(10) + self.assertEqual(self.blob.read(10), self.data[10:20]) + + def test_blob_read_error_row_changed(self): + self.cx.execute("update test set b='aaaa' where rowid=1") + with self.assertRaises(sqlite.OperationalError): + self.blob.read() + + def test_blob_write(self): + new_data = b"new data".ljust(50) + self.blob.write(new_data) + row = self.cx.execute("select b from test").fetchone() + self.assertEqual(row[0], new_data) + + def test_blob_write_at_offset(self): + new_data = b"c" * 25 + self.blob.seek(25) + self.blob.write(new_data) + row = self.cx.execute("select b from test").fetchone() + self.assertEqual(row[0], self.data[:25] + new_data) + + def test_blob_write_advance_offset(self): + self.blob.write(b"d"*10) + self.assertEqual(self.blob.tell(), 10) + + def test_blob_write_error_length(self): + with self.assertRaisesRegex(ValueError, "data longer than blob"): + self.blob.write(b"a" * 1000) + + def test_blob_write_error_row_changed(self): + self.cx.execute("update test set b='aaaa' where rowid=1") + with self.assertRaises(sqlite.OperationalError): + self.blob.write(b"aaa") + + def test_blob_write_error_readonly(self): + ro_blob = self.cx.blobopen("test", "b", 1, readonly=True) + with self.assertRaisesRegex(sqlite.OperationalError, "readonly"): + ro_blob.write(b"aaa") + ro_blob.close() + + def test_blob_open_error(self): + dataset = ( + (("test", "b", 1), {"name": "notexisting"}), + (("notexisting", "b", 1), {}), + (("test", "notexisting", 1), {}), + (("test", "b", 2), {}), + ) + regex = "no such" + for args, kwds in dataset: + with self.subTest(args=args, kwds=kwds): + with self.assertRaisesRegex(sqlite.OperationalError, regex): + self.cx.blobopen(*args, **kwds) + + def test_blob_sequence_not_supported(self): + with self.assertRaises(TypeError): + self.blob + self.blob + with self.assertRaises(TypeError): + self.blob * 5 + with self.assertRaises(TypeError): + b"a" in self.blob + + def test_blob_closed(self): + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test values (zeroblob(100))") + blob = cx.blobopen("test", "b", 1) + blob.close() + + msg = "Cannot operate on a closed blob" + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + blob.read() + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + blob.write(b"") + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + blob.seek(0) + with self.assertRaisesRegex(sqlite.ProgrammingError, msg): + blob.tell() + + def test_blob_closed_db_read(self): + with memory_database() as cx: + cx.execute("create table test(b blob)") + cx.execute("insert into test(b) values (zeroblob(100))") + blob = cx.blobopen("test", "b", 1) + cx.close() + self.assertRaisesRegex(sqlite.ProgrammingError, + "Cannot operate on a closed database", + blob.read) + + class ThreadTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") self.cur = self.con.cursor() - self.cur.execute("create table test(name text)") + self.cur.execute("create table test(name text, b blob)") + self.cur.execute("insert into test values('blob', zeroblob(1))") def tearDown(self): self.cur.close() @@ -1080,6 +1234,7 @@ class ThreadTests(unittest.TestCase): lambda: self.con.create_collation("foo", None), lambda: self.con.setlimit(sqlite.SQLITE_LIMIT_LENGTH, -1), lambda: self.con.getlimit(sqlite.SQLITE_LIMIT_LENGTH), + lambda: self.con.blobopen("test", "b", 1), ] if hasattr(sqlite.Connection, "serialize"): fns.append(lambda: self.con.serialize()) diff --git a/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst new file mode 100644 index 00000000000..0a57f90c123 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-04-18-16-15-55.bpo-24905.jYqjYx.rst @@ -0,0 +1,3 @@ +Add :meth:`~sqlite3.Connection.blobopen` to :class:`sqlite3.Connection`. +:class:`sqlite3.Blob` allows incremental I/O operations on blobs. +Patch by Aviv Palivoda and Erlend E. Aasland. diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c new file mode 100644 index 00000000000..821295cee81 --- /dev/null +++ b/Modules/_sqlite/blob.c @@ -0,0 +1,351 @@ +#include "blob.h" +#include "util.h" + +#define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) +#include "clinic/blob.c.h" +#undef clinic_state + +/*[clinic input] +module _sqlite3 +class _sqlite3.Blob "pysqlite_Blob *" "clinic_state()->BlobType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=908d3e16a45f8da7]*/ + +static void +close_blob(pysqlite_Blob *self) +{ + if (self->blob) { + sqlite3_blob *blob = self->blob; + self->blob = NULL; + + Py_BEGIN_ALLOW_THREADS + sqlite3_blob_close(blob); + Py_END_ALLOW_THREADS + } +} + +static int +blob_traverse(pysqlite_Blob *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->connection); + return 0; +} + +static int +blob_clear(pysqlite_Blob *self) +{ + Py_CLEAR(self->connection); + return 0; +} + +static void +blob_dealloc(pysqlite_Blob *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + + close_blob(self); + + if (self->in_weakreflist != NULL) { + PyObject_ClearWeakRefs((PyObject*)self); + } + tp->tp_clear((PyObject *)self); + tp->tp_free(self); + Py_DECREF(tp); +} + +// Return 1 if the blob object is usable, 0 if not. +static int +check_blob(pysqlite_Blob *self) +{ + if (!pysqlite_check_connection(self->connection) || + !pysqlite_check_thread(self->connection)) { + return 0; + } + if (self->blob == NULL) { + pysqlite_state *state = self->connection->state; + PyErr_SetString(state->ProgrammingError, + "Cannot operate on a closed blob."); + return 0; + } + return 1; +} + + +/*[clinic input] +_sqlite3.Blob.close as blob_close + +Close the blob. +[clinic start generated code]*/ + +static PyObject * +blob_close_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=848accc20a138d1b input=7bc178a402a40bd8]*/ +{ + if (!pysqlite_check_connection(self->connection) || + !pysqlite_check_thread(self->connection)) + { + return NULL; + } + close_blob(self); + Py_RETURN_NONE; +}; + +void +pysqlite_close_all_blobs(pysqlite_Connection *self) +{ + for (int i = 0; i < PyList_GET_SIZE(self->blobs); i++) { + PyObject *weakref = PyList_GET_ITEM(self->blobs, i); + PyObject *blob = PyWeakref_GetObject(weakref); + if (!Py_IsNone(blob)) { + close_blob((pysqlite_Blob *)blob); + } + } +} + +static void +blob_seterror(pysqlite_Blob *self, int rc) +{ + assert(self->connection != NULL); +#if SQLITE_VERSION_NUMBER < 3008008 + // SQLite pre 3.8.8 does not set this blob error on the connection + if (rc == SQLITE_ABORT) { + PyErr_SetString(self->connection->OperationalError, + "Cannot operate on an expired blob handle"); + return; + } +#endif + _pysqlite_seterror(self->connection->state, self->connection->db); +} + +static PyObject * +inner_read(pysqlite_Blob *self, int length, int offset) +{ + PyObject *buffer = PyBytes_FromStringAndSize(NULL, length); + if (buffer == NULL) { + return NULL; + } + + char *raw_buffer = PyBytes_AS_STRING(buffer); + int rc; + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_read(self->blob, raw_buffer, length, offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + Py_DECREF(buffer); + blob_seterror(self, rc); + return NULL; + } + return buffer; +} + + +/*[clinic input] +_sqlite3.Blob.read as blob_read + + length: int = -1 + Read length in bytes. + / + +Read data at the current offset position. + +If the end of the blob is reached, the data up to end of file will be returned. +When length is not specified, or is negative, Blob.read() will read until the +end of the blob. +[clinic start generated code]*/ + +static PyObject * +blob_read_impl(pysqlite_Blob *self, int length) +/*[clinic end generated code: output=1fc99b2541360dde input=f2e4aa4378837250]*/ +{ + if (!check_blob(self)) { + return NULL; + } + + /* Make sure we never read past "EOB". Also read the rest of the blob if a + * negative length is specified. */ + int blob_len = sqlite3_blob_bytes(self->blob); + int max_read_len = blob_len - self->offset; + if (length < 0 || length > max_read_len) { + length = max_read_len; + } + + PyObject *buffer = inner_read(self, length, self->offset); + if (buffer == NULL) { + return NULL; + } + self->offset += length; + return buffer; +}; + +static int +inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, int offset) +{ + int remaining_len = sqlite3_blob_bytes(self->blob) - self->offset; + if (len > remaining_len) { + PyErr_SetString(PyExc_ValueError, "data longer than blob length"); + return -1; + } + + int rc; + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_write(self->blob, buf, (int)len, offset); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + blob_seterror(self, rc); + return -1; + } + return 0; +} + + +/*[clinic input] +_sqlite3.Blob.write as blob_write + + data: Py_buffer + / + +Write data at the current offset. + +This function cannot change the blob length. Writing beyond the end of the +blob will result in an exception being raised. +[clinic start generated code]*/ + +static PyObject * +blob_write_impl(pysqlite_Blob *self, Py_buffer *data) +/*[clinic end generated code: output=b34cf22601b570b2 input=a84712f24a028e6d]*/ +{ + if (!check_blob(self)) { + return NULL; + } + + int rc = inner_write(self, data->buf, data->len, self->offset); + if (rc < 0) { + return NULL; + } + self->offset += (int)data->len; + Py_RETURN_NONE; +} + + +/*[clinic input] +_sqlite3.Blob.seek as blob_seek + + offset: int + origin: int = 0 + / + +Set the current access position to offset. + +The origin argument defaults to os.SEEK_SET (absolute blob positioning). +Other values for origin are os.SEEK_CUR (seek relative to the current position) +and os.SEEK_END (seek relative to the blob's end). +[clinic start generated code]*/ + +static PyObject * +blob_seek_impl(pysqlite_Blob *self, int offset, int origin) +/*[clinic end generated code: output=854c5a0e208547a5 input=5da9a07e55fe6bb6]*/ +{ + if (!check_blob(self)) { + return NULL; + } + + int blob_len = sqlite3_blob_bytes(self->blob); + switch (origin) { + case SEEK_SET: + break; + case SEEK_CUR: + if (offset > INT_MAX - self->offset) { + goto overflow; + } + offset += self->offset; + break; + case SEEK_END: + if (offset > INT_MAX - blob_len) { + goto overflow; + } + offset += blob_len; + break; + default: + PyErr_SetString(PyExc_ValueError, + "'origin' should be os.SEEK_SET, os.SEEK_CUR, or " + "os.SEEK_END"); + return NULL; + } + + if (offset < 0 || offset > blob_len) { + PyErr_SetString(PyExc_ValueError, "offset out of blob range"); + return NULL; + } + + self->offset = offset; + Py_RETURN_NONE; + +overflow: + PyErr_SetString(PyExc_OverflowError, "seek offset results in overflow"); + return NULL; +} + + +/*[clinic input] +_sqlite3.Blob.tell as blob_tell + +Return the current access position for the blob. +[clinic start generated code]*/ + +static PyObject * +blob_tell_impl(pysqlite_Blob *self) +/*[clinic end generated code: output=3d3ba484a90b3a99 input=7e34057aa303612c]*/ +{ + if (!check_blob(self)) { + return NULL; + } + return PyLong_FromLong(self->offset); +} + + +static PyMethodDef blob_methods[] = { + BLOB_CLOSE_METHODDEF + BLOB_READ_METHODDEF + BLOB_SEEK_METHODDEF + BLOB_TELL_METHODDEF + BLOB_WRITE_METHODDEF + {NULL, NULL} +}; + +static struct PyMemberDef blob_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(pysqlite_Blob, in_weakreflist), READONLY}, + {NULL}, +}; + +static PyType_Slot blob_slots[] = { + {Py_tp_dealloc, blob_dealloc}, + {Py_tp_traverse, blob_traverse}, + {Py_tp_clear, blob_clear}, + {Py_tp_methods, blob_methods}, + {Py_tp_members, blob_members}, + {0, NULL}, +}; + +static PyType_Spec blob_spec = { + .name = MODULE_NAME ".Blob", + .basicsize = sizeof(pysqlite_Blob), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = blob_slots, +}; + +int +pysqlite_blob_setup_types(PyObject *mod) +{ + PyObject *type = PyType_FromModuleAndSpec(mod, &blob_spec, NULL); + if (type == NULL) { + return -1; + } + pysqlite_state *state = pysqlite_get_state(mod); + state->BlobType = (PyTypeObject *)type; + return 0; +} diff --git a/Modules/_sqlite/blob.h b/Modules/_sqlite/blob.h new file mode 100644 index 00000000000..418ca03bdb5 --- /dev/null +++ b/Modules/_sqlite/blob.h @@ -0,0 +1,24 @@ +#ifndef PYSQLITE_BLOB_H +#define PYSQLITE_BLOB_H + +#include "Python.h" +#include "sqlite3.h" +#include "connection.h" + +#define BLOB_SEEK_START 0 +#define BLOB_SEEK_CUR 1 +#define BLOB_SEEK_END 2 + +typedef struct { + PyObject_HEAD + pysqlite_Connection *connection; + sqlite3_blob *blob; + int offset; + + PyObject *in_weakreflist; +} pysqlite_Blob; + +int pysqlite_blob_setup_types(PyObject *mod); +void pysqlite_close_all_blobs(pysqlite_Connection *self); + +#endif diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h new file mode 100644 index 00000000000..30b3e3c1947 --- /dev/null +++ b/Modules/_sqlite/clinic/blob.c.h @@ -0,0 +1,165 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(blob_close__doc__, +"close($self, /)\n" +"--\n" +"\n" +"Close the blob."); + +#define BLOB_CLOSE_METHODDEF \ + {"close", (PyCFunction)blob_close, METH_NOARGS, blob_close__doc__}, + +static PyObject * +blob_close_impl(pysqlite_Blob *self); + +static PyObject * +blob_close(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_close_impl(self); +} + +PyDoc_STRVAR(blob_read__doc__, +"read($self, length=-1, /)\n" +"--\n" +"\n" +"Read data at the current offset position.\n" +"\n" +" length\n" +" Read length in bytes.\n" +"\n" +"If the end of the blob is reached, the data up to end of file will be returned.\n" +"When length is not specified, or is negative, Blob.read() will read until the\n" +"end of the blob."); + +#define BLOB_READ_METHODDEF \ + {"read", (PyCFunction)(void(*)(void))blob_read, METH_FASTCALL, blob_read__doc__}, + +static PyObject * +blob_read_impl(pysqlite_Blob *self, int length); + +static PyObject * +blob_read(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int length = -1; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + length = _PyLong_AsInt(args[0]); + if (length == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = blob_read_impl(self, length); + +exit: + return return_value; +} + +PyDoc_STRVAR(blob_write__doc__, +"write($self, data, /)\n" +"--\n" +"\n" +"Write data at the current offset.\n" +"\n" +"This function cannot change the blob length. Writing beyond the end of the\n" +"blob will result in an exception being raised."); + +#define BLOB_WRITE_METHODDEF \ + {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, + +static PyObject * +blob_write_impl(pysqlite_Blob *self, Py_buffer *data); + +static PyObject * +blob_write(pysqlite_Blob *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (!PyBuffer_IsContiguous(&data, 'C')) { + _PyArg_BadArgument("write", "argument", "contiguous buffer", arg); + goto exit; + } + return_value = blob_write_impl(self, &data); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + +PyDoc_STRVAR(blob_seek__doc__, +"seek($self, offset, origin=0, /)\n" +"--\n" +"\n" +"Set the current access position to offset.\n" +"\n" +"The origin argument defaults to os.SEEK_SET (absolute blob positioning).\n" +"Other values for origin are os.SEEK_CUR (seek relative to the current position)\n" +"and os.SEEK_END (seek relative to the blob\'s end)."); + +#define BLOB_SEEK_METHODDEF \ + {"seek", (PyCFunction)(void(*)(void))blob_seek, METH_FASTCALL, blob_seek__doc__}, + +static PyObject * +blob_seek_impl(pysqlite_Blob *self, int offset, int origin); + +static PyObject * +blob_seek(pysqlite_Blob *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int offset; + int origin = 0; + + if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { + goto exit; + } + offset = _PyLong_AsInt(args[0]); + if (offset == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + origin = _PyLong_AsInt(args[1]); + if (origin == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = blob_seek_impl(self, offset, origin); + +exit: + return return_value; +} + +PyDoc_STRVAR(blob_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n" +"Return the current access position for the blob."); + +#define BLOB_TELL_METHODDEF \ + {"tell", (PyCFunction)blob_tell, METH_NOARGS, blob_tell__doc__}, + +static PyObject * +blob_tell_impl(pysqlite_Blob *self); + +static PyObject * +blob_tell(pysqlite_Blob *self, PyObject *Py_UNUSED(ignored)) +{ + return blob_tell_impl(self); +} +/*[clinic end generated code: output=d3a02b127f2cfa58 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 2b933f85224..d4597086f4c 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -145,6 +145,110 @@ exit: return return_value; } +PyDoc_STRVAR(blobopen__doc__, +"blobopen($self, table, column, row, /, *, readonly=False, name=\'main\')\n" +"--\n" +"\n" +"Open and return a BLOB object.\n" +"\n" +" table\n" +" Table name.\n" +" column\n" +" Column name.\n" +" row\n" +" Row index.\n" +" readonly\n" +" Open the BLOB without write permissions.\n" +" name\n" +" Database name."); + +#define BLOBOPEN_METHODDEF \ + {"blobopen", (PyCFunction)(void(*)(void))blobopen, METH_FASTCALL|METH_KEYWORDS, blobopen__doc__}, + +static PyObject * +blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, + int row, int readonly, const char *name); + +static PyObject * +blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"", "", "", "readonly", "name", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "blobopen", 0}; + PyObject *argsbuf[5]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; + const char *table; + const char *col; + int row; + int readonly = 0; + const char *name = "main"; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("blobopen", "argument 1", "str", args[0]); + goto exit; + } + Py_ssize_t table_length; + table = PyUnicode_AsUTF8AndSize(args[0], &table_length); + if (table == NULL) { + goto exit; + } + if (strlen(table) != (size_t)table_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("blobopen", "argument 2", "str", args[1]); + goto exit; + } + Py_ssize_t col_length; + col = PyUnicode_AsUTF8AndSize(args[1], &col_length); + if (col == NULL) { + goto exit; + } + if (strlen(col) != (size_t)col_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + row = _PyLong_AsInt(args[2]); + if (row == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + readonly = _PyLong_AsInt(args[3]); + if (readonly == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (!PyUnicode_Check(args[4])) { + _PyArg_BadArgument("blobopen", "argument 'name'", "str", args[4]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[4], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } +skip_optional_kwonly: + return_value = blobopen_impl(self, table, col, row, readonly, name); + +exit: + return return_value; +} + PyDoc_STRVAR(pysqlite_connection_close__doc__, "close($self, /)\n" "--\n" @@ -1041,4 +1145,4 @@ exit: #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=b9af1b52fda808bf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=be2f526e78fa65b1 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index d7c0a9e4616..85fb128fc7f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -26,6 +26,7 @@ #include "connection.h" #include "statement.h" #include "cursor.h" +#include "blob.h" #include "prepare_protocol.h" #include "util.h" @@ -234,10 +235,17 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, return -1; } - // Create list of weak references to cursors. + /* Create lists of weak references to cursors and blobs */ PyObject *cursors = PyList_New(0); if (cursors == NULL) { - Py_DECREF(statement_cache); + Py_XDECREF(statement_cache); + return -1; + } + + PyObject *blobs = PyList_New(0); + if (blobs == NULL) { + Py_XDECREF(statement_cache); + Py_XDECREF(cursors); return -1; } @@ -250,6 +258,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, self->thread_ident = PyThread_get_thread_ident(); self->statement_cache = statement_cache; self->cursors = cursors; + self->blobs = blobs; self->created_cursors = 0; self->row_factory = Py_NewRef(Py_None); self->text_factory = Py_NewRef(&PyUnicode_Type); @@ -291,6 +300,7 @@ connection_traverse(pysqlite_Connection *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); Py_VISIT(self->statement_cache); Py_VISIT(self->cursors); + Py_VISIT(self->blobs); Py_VISIT(self->row_factory); Py_VISIT(self->text_factory); VISIT_CALLBACK_CONTEXT(self->trace_ctx); @@ -314,6 +324,7 @@ connection_clear(pysqlite_Connection *self) { Py_CLEAR(self->statement_cache); Py_CLEAR(self->cursors); + Py_CLEAR(self->blobs); Py_CLEAR(self->row_factory); Py_CLEAR(self->text_factory); clear_callback_context(self->trace_ctx); @@ -429,6 +440,76 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } +/*[clinic input] +_sqlite3.Connection.blobopen as blobopen + + table: str + Table name. + column as col: str + Column name. + row: int + Row index. + / + * + readonly: bool(accept={int}) = False + Open the BLOB without write permissions. + name: str = "main" + Database name. + +Open and return a BLOB object. +[clinic start generated code]*/ + +static PyObject * +blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, + int row, int readonly, const char *name) +/*[clinic end generated code: output=0c8e2e58516d0b5c input=1e7052516acfc94d]*/ +{ + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + int rc; + sqlite3_blob *blob; + + Py_BEGIN_ALLOW_THREADS + rc = sqlite3_blob_open(self->db, name, table, col, row, !readonly, &blob); + Py_END_ALLOW_THREADS + + if (rc != SQLITE_OK) { + _pysqlite_seterror(self->state, self->db); + return NULL; + } + + pysqlite_Blob *obj = PyObject_GC_New(pysqlite_Blob, self->state->BlobType); + if (obj == NULL) { + goto error; + } + + obj->connection = (pysqlite_Connection *)Py_NewRef(self); + obj->blob = blob; + obj->offset = 0; + obj->in_weakreflist = NULL; + + PyObject_GC_Track(obj); + + // Add our blob to connection blobs list + PyObject *weakref = PyWeakref_NewRef((PyObject *)obj, NULL); + if (weakref == NULL) { + goto error; + } + rc = PyList_Append(self->blobs, weakref); + Py_DECREF(weakref); + if (rc < 0) { + goto error; + } + + return (PyObject *)obj; + +error: + Py_XDECREF(obj); + return NULL; +} + /*[clinic input] _sqlite3.Connection.close as pysqlite_connection_close @@ -451,6 +532,7 @@ pysqlite_connection_close_impl(pysqlite_Connection *self) return NULL; } + pysqlite_close_all_blobs(self); Py_CLEAR(self->statement_cache); connection_close(self); @@ -2257,6 +2339,7 @@ static PyMethodDef connection_methods[] = { SERIALIZE_METHODDEF DESERIALIZE_METHODDEF CREATE_WINDOW_FUNCTION_METHODDEF + BLOBOPEN_METHODDEF {NULL, NULL} }; diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 84f1f095cb3..2b946ff3c73 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -63,8 +63,9 @@ typedef struct PyObject *statement_cache; - /* Lists of weak references to statements and cursors used within this connection */ - PyObject* cursors; + /* Lists of weak references to cursors and blobs used within this connection */ + PyObject *cursors; + PyObject *blobs; /* Counters for how many cursors were created in the connection. May be * reset to 0 at certain intervals */ diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index ffda836d7a3..d355c2be37a 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -27,6 +27,7 @@ #include "prepare_protocol.h" #include "microprotocols.h" #include "row.h" +#include "blob.h" #if SQLITE_VERSION_NUMBER < 3007015 #error "SQLite 3.7.15 or higher required" @@ -582,6 +583,7 @@ module_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->Warning); // Types + Py_VISIT(state->BlobType); Py_VISIT(state->ConnectionType); Py_VISIT(state->CursorType); Py_VISIT(state->PrepareProtocolType); @@ -614,6 +616,7 @@ module_clear(PyObject *module) Py_CLEAR(state->Warning); // Types + Py_CLEAR(state->BlobType); Py_CLEAR(state->ConnectionType); Py_CLEAR(state->CursorType); Py_CLEAR(state->PrepareProtocolType); @@ -687,7 +690,8 @@ module_exec(PyObject *module) (pysqlite_cursor_setup_types(module) < 0) || (pysqlite_connection_setup_types(module) < 0) || (pysqlite_statement_setup_types(module) < 0) || - (pysqlite_prepare_protocol_setup_types(module) < 0) + (pysqlite_prepare_protocol_setup_types(module) < 0) || + (pysqlite_blob_setup_types(module) < 0) ) { goto error; } diff --git a/Modules/_sqlite/module.h b/Modules/_sqlite/module.h index fcea7096924..7deba22ffec 100644 --- a/Modules/_sqlite/module.h +++ b/Modules/_sqlite/module.h @@ -53,6 +53,7 @@ typedef struct { int BaseTypeAdapted; int enable_callback_tracebacks; + PyTypeObject *BlobType; PyTypeObject *ConnectionType; PyTypeObject *CursorType; PyTypeObject *PrepareProtocolType; diff --git a/PCbuild/_sqlite3.vcxproj b/PCbuild/_sqlite3.vcxproj index 9cff43f73e5..804aa07367a 100644 --- a/PCbuild/_sqlite3.vcxproj +++ b/PCbuild/_sqlite3.vcxproj @@ -106,6 +106,7 @@ + @@ -116,6 +117,7 @@ + diff --git a/PCbuild/_sqlite3.vcxproj.filters b/PCbuild/_sqlite3.vcxproj.filters index 79fc17b53fb..f4a265eba7d 100644 --- a/PCbuild/_sqlite3.vcxproj.filters +++ b/PCbuild/_sqlite3.vcxproj.filters @@ -36,6 +36,9 @@ Header Files + + Header Files + @@ -62,6 +65,9 @@ Source Files + + Source Files + diff --git a/setup.py b/setup.py index 1e1627ec392..60a45cf260f 100644 --- a/setup.py +++ b/setup.py @@ -1256,6 +1256,7 @@ class PyBuildExt(build_ext): def detect_sqlite(self): sources = [ + "_sqlite/blob.c", "_sqlite/connection.c", "_sqlite/cursor.c", "_sqlite/microprotocols.c",