Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-91602: Add iterdump() support for filtering database objects (#114501) 

Add optional 'filter' parameter to iterdump() that allows a "LIKE"
pattern for filtering database objects to dump.

Co-authored-by: Erlend E. Aasland <erlend@python.org>
This commit is contained in:
Mariusz Felisiak 2024-02-06 12:34:56 +01:00 committed by GitHub
parent 4bf41879d0
commit 1a10437a14
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 176 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Doc/library/sqlite3.rst
View File

@ -1137,12 +1137,19 @@ Connection objects
.. _Loading an Extension: https://www.sqlite.org/loadext.html#loading_an_extension_
.. method:: iterdump
.. method:: iterdump(*, filter=None)
Return an :term:`iterator` to dump the database as SQL source code.
Useful when saving an in-memory database for later restoration.
Similar to the ``.dump`` command in the :program:`sqlite3` shell.
:param filter:
An optional ``LIKE`` pattern for database objects to dump, e.g. ``prefix_%``.
If ``None`` (the default), all database objects will be included.
:type filter: str | None
Example:
.. testcode::
@ -1158,6 +1165,8 @@ Connection objects
:ref:`sqlite3-howto-encoding`
.. versionchanged:: 3.13
Added the *filter* parameter.
.. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250)

4
Doc/whatsnew/3.13.rst
View File

@ -438,6 +438,10 @@ sqlite3
object is not :meth:`closed <sqlite3.Connection.close>` explicitly.
(Contributed by Erlend E. Aasland in :gh:`105539`.)
* Add *filter* keyword-only parameter to :meth:`sqlite3.Connection.iterdump`
for filtering database objects to dump.
(Contributed by Mariusz Felisiak in :gh:`91602`.)
subprocess
----------

1
Include/internal/pycore_global_objects_fini_generated.h generated
View File

@ -940,6 +940,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fileno));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filepath));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fillvalue));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filter));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(filters));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(final));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(find_class));

1
Include/internal/pycore_global_strings.h
View File

@ -429,6 +429,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(fileno)
STRUCT_FOR_ID(filepath)
STRUCT_FOR_ID(fillvalue)
STRUCT_FOR_ID(filter)
STRUCT_FOR_ID(filters)
STRUCT_FOR_ID(final)
STRUCT_FOR_ID(find_class)

1
Include/internal/pycore_runtime_init_generated.h generated
View File

@ -938,6 +938,7 @@ extern "C" {
INIT_ID(fileno), \
INIT_ID(filepath), \
INIT_ID(fillvalue), \
INIT_ID(filter), \
INIT_ID(filters), \
INIT_ID(final), \
INIT_ID(find_class), \

3
Include/internal/pycore_unicodeobject_generated.h generated
View File

@ -1128,6 +1128,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
string = &_Py_ID(fillvalue);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(filter);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(filters);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);

19
Lib/sqlite3/dump.py
View File

@ -15,7 +15,7 @@ def _quote_value(value):
return "'{0}'".format(value.replace("'", "''"))
def _iterdump(connection):
def _iterdump(connection, *, filter=None):
"""
Returns an iterator to the dump of the database in an SQL text format.
@ -32,15 +32,23 @@ def _iterdump(connection):
yield('PRAGMA foreign_keys=OFF;')
yield('BEGIN TRANSACTION;')
if filter:
# Return database objects which match the filter pattern.
filter_name_clause = 'AND "name" LIKE ?'
params = [filter]
else:
filter_name_clause = ""
params = []
# sqlite_master table contains the SQL CREATE statements for the database.
q = """
q = f"""
SELECT "name", "type", "sql"
FROM "sqlite_master"
WHERE "sql" NOT NULL AND
"type" == 'table'
{filter_name_clause}
ORDER BY "name"
"""
schema_res = cu.execute(q)
schema_res = cu.execute(q, params)
sqlite_sequence = []
for table_name, type, sql in schema_res.fetchall():
if table_name == 'sqlite_sequence':
@ -82,13 +90,14 @@ def _iterdump(connection):
yield("{0};".format(row[0]))
# Now when the type is 'index', 'trigger', or 'view'
q = """
q = f"""
SELECT "name", "type", "sql"
FROM "sqlite_master"
WHERE "sql" NOT NULL AND
"type" IN ('index', 'trigger', 'view')
{filter_name_clause}
"""
schema_res = cu.execute(q)
schema_res = cu.execute(q, params)
for name, type, sql in schema_res.fetchall():
yield('{0};'.format(sql))

70
Lib/test/test_sqlite3/test_dump.py
View File

@ -54,6 +54,76 @@ class DumpTests(MemoryDatabaseMixin, unittest.TestCase):
[self.assertEqual(expected_sqls[i], actual_sqls[i])
for i in range(len(expected_sqls))]
def test_table_dump_filter(self):
all_table_sqls = [
"""CREATE TABLE "some_table_2" ("id_1" INTEGER);""",
"""INSERT INTO "some_table_2" VALUES(3);""",
"""INSERT INTO "some_table_2" VALUES(4);""",
"""CREATE TABLE "test_table_1" ("id_2" INTEGER);""",
"""INSERT INTO "test_table_1" VALUES(1);""",
"""INSERT INTO "test_table_1" VALUES(2);""",
]
all_views_sqls = [
"""CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""",
"""CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""",
]
# Create database structure.
for sql in [*all_table_sqls, *all_views_sqls]:
self.cu.execute(sql)
# %_table_% matches all tables.
dump_sqls = list(self.cx.iterdump(filter="%_table_%"))
self.assertEqual(
dump_sqls,
["BEGIN TRANSACTION;", *all_table_sqls, "COMMIT;"],
)
# view_% matches all views.
dump_sqls = list(self.cx.iterdump(filter="view_%"))
self.assertEqual(
dump_sqls,
["BEGIN TRANSACTION;", *all_views_sqls, "COMMIT;"],
)
# %_1 matches tables and views with the _1 suffix.
dump_sqls = list(self.cx.iterdump(filter="%_1"))
self.assertEqual(
dump_sqls,
[
"BEGIN TRANSACTION;",
"""CREATE TABLE "test_table_1" ("id_2" INTEGER);""",
"""INSERT INTO "test_table_1" VALUES(1);""",
"""INSERT INTO "test_table_1" VALUES(2);""",
"""CREATE VIEW "view_1" AS SELECT * FROM "some_table_2";""",
"COMMIT;"
],
)
# some_% matches some_table_2.
dump_sqls = list(self.cx.iterdump(filter="some_%"))
self.assertEqual(
dump_sqls,
[
"BEGIN TRANSACTION;",
"""CREATE TABLE "some_table_2" ("id_1" INTEGER);""",
"""INSERT INTO "some_table_2" VALUES(3);""",
"""INSERT INTO "some_table_2" VALUES(4);""",
"COMMIT;"
],
)
# Only single object.
dump_sqls = list(self.cx.iterdump(filter="view_2"))
self.assertEqual(
dump_sqls,
[
"BEGIN TRANSACTION;",
"""CREATE VIEW "view_2" AS SELECT * FROM "test_table_1";""",
"COMMIT;"
],
)
# % matches all objects.
dump_sqls = list(self.cx.iterdump(filter="%"))
self.assertEqual(
dump_sqls,
["BEGIN TRANSACTION;", *all_table_sqls, *all_views_sqls, "COMMIT;"],
)
def test_dump_autoincrement(self):
expected = [
'CREATE TABLE "t1" (id integer primary key autoincrement);',

3
Misc/NEWS.d/next/Library/2024-01-24-20-51-49.gh-issue-91602.8fOH8l.rst Normal file
View File

@ -0,0 +1,3 @@
Add *filter* keyword-only parameter to
:meth:`sqlite3.Connection.iterdump` for filtering database objects to dump.
Patch by Mariusz Felisiak.

60
Modules/_sqlite/clinic/connection.c.h generated
View File

@ -1204,21 +1204,67 @@ pysqlite_connection_interrupt(pysqlite_Connection *self, PyObject *Py_UNUSED(ign
}
PyDoc_STRVAR(pysqlite_connection_iterdump__doc__,
"iterdump($self, /)\n"
"iterdump($self, /, *, filter=None)\n"
"--\n"
"\n"
"Returns iterator to the dump of the database in an SQL text format.");
"Returns iterator to the dump of the database in an SQL text format.\n"
"\n"
" filter\n"
" An optional LIKE pattern for database objects to dump");
#define PYSQLITE_CONNECTION_ITERDUMP_METHODDEF \
{"iterdump", (PyCFunction)pysqlite_connection_iterdump, METH_NOARGS, pysqlite_connection_iterdump__doc__},
{"iterdump", _PyCFunction_CAST(pysqlite_connection_iterdump), METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_iterdump__doc__},
static PyObject *
pysqlite_connection_iterdump_impl(pysqlite_Connection *self);
pysqlite_connection_iterdump_impl(pysqlite_Connection *self,
PyObject *filter);
static PyObject *
pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *Py_UNUSED(ignored))
pysqlite_connection_iterdump(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
return pysqlite_connection_iterdump_impl(self);
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 1
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(filter), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
#else // !Py_BUILD_CORE
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"filter", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "iterdump",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
PyObject *argsbuf[1];
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
PyObject *filter = Py_None;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
if (!args) {
goto exit;
}
if (!noptargs) {
goto skip_optional_kwonly;
}
filter = args[0];
skip_optional_kwonly:
return_value = pysqlite_connection_iterdump_impl(self, filter);
exit:
return return_value;
}
PyDoc_STRVAR(pysqlite_connection_backup__doc__,
@ -1820,4 +1866,4 @@ exit:
#ifndef DESERIALIZE_METHODDEF
#define DESERIALIZE_METHODDEF
#endif /* !defined(DESERIALIZE_METHODDEF) */
/*[clinic end generated code: output=99299d3ee2c247ab input=a9049054013a1b77]*/
/*[clinic end generated code: output=3c6d0b748fac016f input=a9049054013a1b77]*/

20
Modules/_sqlite/connection.c
View File

@ -1979,12 +1979,17 @@ finally:
/*[clinic input]
_sqlite3.Connection.iterdump as pysqlite_connection_iterdump
*
filter: object = None
An optional LIKE pattern for database objects to dump
Returns iterator to the dump of the database in an SQL text format.
[clinic start generated code]*/
static PyObject *
pysqlite_connection_iterdump_impl(pysqlite_Connection *self)
/*[clinic end generated code: output=586997aaf9808768 input=1911ca756066da89]*/
pysqlite_connection_iterdump_impl(pysqlite_Connection *self,
PyObject *filter)
/*[clinic end generated code: output=fd81069c4bdeb6b0 input=4ae6d9a898f108df]*/
{
if (!pysqlite_check_connection(self)) {
return NULL;
@ -1998,9 +2003,16 @@ pysqlite_connection_iterdump_impl(pysqlite_Connection *self)
}
return NULL;
}
PyObject *retval = PyObject_CallOneArg(iterdump, (PyObject *)self);
PyObject *args[3] = {NULL, (PyObject *)self, filter};
PyObject *kwnames = Py_BuildValue("(s)", "filter");
if (!kwnames) {
Py_DECREF(iterdump);
return NULL;
}
Py_ssize_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET;
PyObject *retval = PyObject_Vectorcall(iterdump, args + 1, nargsf, kwnames);
Py_DECREF(iterdump);
Py_DECREF(kwnames);
return retval;
}