From 0da728387c99fe6c127b070f2d250dc5bdd62ee5 Mon Sep 17 00:00:00 2001 From: colorfulappl Date: Fri, 25 Nov 2022 03:56:50 +0800 Subject: [PATCH] gh-64490: Fix bugs in argument clinic varargs processing (#32092) --- .../pycore_global_objects_fini_generated.h | 5 + Include/internal/pycore_global_strings.h | 5 + .../internal/pycore_runtime_init_generated.h | 5 + .../internal/pycore_unicodeobject_generated.h | 10 + Lib/test/clinic.test | 7 +- Lib/test/test_clinic.py | 44 ++ ...2-08-11-09-58-15.gh-issue-64490.PjwhM4.rst | 7 + Modules/_testclinic.c | 119 ++++++ Modules/clinic/_testclinic.c.h | 390 +++++++++++++++++- Python/getargs.c | 20 +- Tools/clinic/clinic.py | 11 +- 11 files changed, 612 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 494bcf293cd..9951fa9951e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -986,6 +986,9 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(keyfile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kind)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw1)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(kw2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(lambda)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(last_node)); @@ -1084,6 +1087,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pid)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(policy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(print_file_and_line)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(progress)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index b0cb8365933..12144b02f45 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -472,6 +472,9 @@ struct _Py_global_strings { STRUCT_FOR_ID(keyfile) STRUCT_FOR_ID(keys) STRUCT_FOR_ID(kind) + STRUCT_FOR_ID(kw) + STRUCT_FOR_ID(kw1) + STRUCT_FOR_ID(kw2) STRUCT_FOR_ID(lambda) STRUCT_FOR_ID(last) STRUCT_FOR_ID(last_node) @@ -570,6 +573,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(pid) STRUCT_FOR_ID(policy) STRUCT_FOR_ID(pos) + STRUCT_FOR_ID(pos1) + STRUCT_FOR_ID(pos2) STRUCT_FOR_ID(print_file_and_line) STRUCT_FOR_ID(priority) STRUCT_FOR_ID(progress) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4b128da5455..87b0f2ed8df 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -978,6 +978,9 @@ extern "C" { INIT_ID(keyfile), \ INIT_ID(keys), \ INIT_ID(kind), \ + INIT_ID(kw), \ + INIT_ID(kw1), \ + INIT_ID(kw2), \ INIT_ID(lambda), \ INIT_ID(last), \ INIT_ID(last_node), \ @@ -1076,6 +1079,8 @@ extern "C" { INIT_ID(pid), \ INIT_ID(policy), \ INIT_ID(pos), \ + INIT_ID(pos1), \ + INIT_ID(pos2), \ INIT_ID(print_file_and_line), \ INIT_ID(priority), \ INIT_ID(progress), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 7ef1f7e94dd..80be342b5b3 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -850,6 +850,12 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(kind); PyUnicode_InternInPlace(&string); + string = &_Py_ID(kw); + PyUnicode_InternInPlace(&string); + string = &_Py_ID(kw1); + PyUnicode_InternInPlace(&string); + string = &_Py_ID(kw2); + PyUnicode_InternInPlace(&string); string = &_Py_ID(lambda); PyUnicode_InternInPlace(&string); string = &_Py_ID(last); @@ -1046,6 +1052,10 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(pos); PyUnicode_InternInPlace(&string); + string = &_Py_ID(pos1); + PyUnicode_InternInPlace(&string); + string = &_Py_ID(pos2); + PyUnicode_InternInPlace(&string); string = &_Py_ID(print_file_and_line); PyUnicode_InternInPlace(&string); string = &_Py_ID(priority); diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index f4842cc962f..0d844234d9d 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -3845,7 +3845,6 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; @@ -3864,7 +3863,7 @@ exit: static PyObject * test_vararg_impl(PyObject *module, PyObject *a, PyObject *args) -/*[clinic end generated code: output=6661f3ca97d85e8c input=81d33815ad1bae6e]*/ +/*[clinic end generated code: output=880365c61ae205d7 input=81d33815ad1bae6e]*/ /*[clinic input] test_vararg_with_default @@ -3918,7 +3917,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar }; #undef KWTUPLE PyObject *argsbuf[3]; - Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *a; PyObject *__clinic_args = NULL; int b = 0; @@ -3947,7 +3946,7 @@ exit: static PyObject * test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, int b) -/*[clinic end generated code: output=5fe3cfccb1bef781 input=6e110b54acd9b22d]*/ +/*[clinic end generated code: output=291e9a5a09831128 input=6e110b54acd9b22d]*/ /*[clinic input] test_vararg_with_only_defaults diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 890beeb9efe..4abf739cf52 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -730,6 +730,15 @@ foo.bar x: int """) + def test_parameters_no_more_than_one_vararg(self): + s = self.parse_function_should_fail(""" +module foo +foo.bar + *vararg1: object + *vararg2: object +""") + self.assertEqual(s, "Error on line 0:\nToo many var args\n") + def test_function_not_at_column_0(self): function = self.parse_function(""" module foo @@ -1222,6 +1231,13 @@ class ClinicFunctionalTest(unittest.TestCase): ac_tester.keyword_only_parameter(1) self.assertEqual(ac_tester.keyword_only_parameter(a=1), (1,)) + def test_posonly_vararg(self): + with self.assertRaises(TypeError): + ac_tester.posonly_vararg() + self.assertEqual(ac_tester.posonly_vararg(1, 2), (1, 2, ())) + self.assertEqual(ac_tester.posonly_vararg(1, b=2), (1, 2, ())) + self.assertEqual(ac_tester.posonly_vararg(1, 2, 3, 4), (1, 2, (3, 4))) + def test_vararg_and_posonly(self): with self.assertRaises(TypeError): ac_tester.vararg_and_posonly() @@ -1229,6 +1245,33 @@ class ClinicFunctionalTest(unittest.TestCase): ac_tester.vararg_and_posonly(1, b=2) self.assertEqual(ac_tester.vararg_and_posonly(1, 2, 3, 4), (1, (2, 3, 4))) + def test_vararg(self): + with self.assertRaises(TypeError): + ac_tester.vararg() + with self.assertRaises(TypeError): + ac_tester.vararg(1, b=2) + self.assertEqual(ac_tester.vararg(1, 2, 3, 4), (1, (2, 3, 4))) + + def test_vararg_with_default(self): + with self.assertRaises(TypeError): + ac_tester.vararg_with_default() + self.assertEqual(ac_tester.vararg_with_default(1, b=False), (1, (), False)) + self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4), (1, (2, 3, 4), False)) + self.assertEqual(ac_tester.vararg_with_default(1, 2, 3, 4, b=True), (1, (2, 3, 4), True)) + + def test_vararg_with_only_defaults(self): + self.assertEqual(ac_tester.vararg_with_only_defaults(), ((), None)) + self.assertEqual(ac_tester.vararg_with_only_defaults(b=2), ((), 2)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, b=2), ((1, ), 2)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4), ((1, 2, 3, 4), None)) + self.assertEqual(ac_tester.vararg_with_only_defaults(1, 2, 3, 4, b=5), ((1, 2, 3, 4), 5)) + + def test_gh_32092_oob(self): + ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) + + def test_gh_32092_kw_pass(self): + ac_tester.gh_32092_kw_pass(1, 2, 3) + def test_gh_99233_refcount(self): arg = '*A unique string is not referenced by anywhere else.*' arg_refcount_origin = sys.getrefcount(arg) @@ -1241,5 +1284,6 @@ class ClinicFunctionalTest(unittest.TestCase): with self.assertRaisesRegex(TypeError, expected_error): ac_tester.gh_99240_double_free('a', '\0b') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst new file mode 100644 index 00000000000..4a308a93060 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-08-11-09-58-15.gh-issue-64490.PjwhM4.rst @@ -0,0 +1,7 @@ +Argument Clinic varargs bugfixes + +* Fix out-of-bounds error in :c:func:`!_PyArg_UnpackKeywordsWithVararg`. +* Fix incorrect check which allowed more than one varargs in clinic.py. +* Fix miscalculation of ``noptargs`` in generated code. +* Do not generate ``noptargs`` when there is a vararg argument and no optional argument. + diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 56eddfd6fdb..91fdee24d32 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -950,6 +950,25 @@ keyword_only_parameter_impl(PyObject *module, PyObject *a) } +/*[clinic input] +posonly_vararg + + a: object + / + b: object + *args: object + +[clinic start generated code]*/ + +static PyObject * +posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args) +/*[clinic end generated code: output=ee6713acda6b954e input=783427fe7ec2b67a]*/ +{ + return pack_arguments_newref(3, a, b, args); +} + + /*[clinic input] vararg_and_posonly @@ -967,6 +986,100 @@ vararg_and_posonly_impl(PyObject *module, PyObject *a, PyObject *args) } +/*[clinic input] +vararg + + a: object + *args: object + +[clinic start generated code]*/ + +static PyObject * +vararg_impl(PyObject *module, PyObject *a, PyObject *args) +/*[clinic end generated code: output=91ab7a0efc52dd5e input=02c0f772d05f591e]*/ +{ + return pack_arguments_newref(2, a, args); +} + + +/*[clinic input] +vararg_with_default + + a: object + *args: object + b: bool = False + +[clinic start generated code]*/ + +static PyObject * +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b) +/*[clinic end generated code: output=182c01035958ce92 input=68cafa6a79f89e36]*/ +{ + PyObject *obj_b = b ? Py_True : Py_False; + return pack_arguments_newref(3, a, args, obj_b); +} + + +/*[clinic input] +vararg_with_only_defaults + + *args: object + b: object = None + +[clinic start generated code]*/ + +static PyObject * +vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b) +/*[clinic end generated code: output=c06b1826d91f2f7b input=678c069bc67550e1]*/ +{ + return pack_arguments_newref(2, args, b); +} + + + +/*[clinic input] +gh_32092_oob + + pos1: object + pos2: object + *varargs: object + kw1: object = None + kw2: object = None + +Proof-of-concept of GH-32092 OOB bug. + +[clinic start generated code]*/ + +static PyObject * +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2) +/*[clinic end generated code: output=ee259c130054653f input=46d15c881608f8ff]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +gh_32092_kw_pass + + pos: object + *args: object + kw: object = None + +Proof-of-concept of GH-32092 keyword args passing bug. + +[clinic start generated code]*/ + +static PyObject * +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw) +/*[clinic end generated code: output=4a2bbe4f7c8604e9 input=5c0bd5b9079a0cce]*/ +{ + Py_RETURN_NONE; +} + + /*[clinic input] gh_99233_refcount @@ -1046,7 +1159,13 @@ static PyMethodDef tester_methods[] = { POSONLY_KEYWORDS_OPT_KWONLY_OPT_METHODDEF POSONLY_OPT_KEYWORDS_OPT_KWONLY_OPT_METHODDEF KEYWORD_ONLY_PARAMETER_METHODDEF + POSONLY_VARARG_METHODDEF VARARG_AND_POSONLY_METHODDEF + VARARG_METHODDEF + VARARG_WITH_DEFAULT_METHODDEF + VARARG_WITH_ONLY_DEFAULTS_METHODDEF + GH_32092_OOB_METHODDEF + GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF {NULL, NULL} diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 9aad44566bc..21bde529470 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2326,6 +2326,66 @@ exit: return return_value; } +PyDoc_STRVAR(posonly_vararg__doc__, +"posonly_vararg($module, a, /, b, *args)\n" +"--\n" +"\n"); + +#define POSONLY_VARARG_METHODDEF \ + {"posonly_vararg", _PyCFunction_CAST(posonly_vararg), METH_FASTCALL|METH_KEYWORDS, posonly_vararg__doc__}, + +static PyObject * +posonly_vararg_impl(PyObject *module, PyObject *a, PyObject *b, + PyObject *args); + +static PyObject * +posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "posonly_vararg", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *a; + PyObject *b; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + b = args[1]; + __clinic_args = args[2]; + return_value = posonly_vararg_impl(module, a, b, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(vararg_and_posonly__doc__, "vararg_and_posonly($module, a, /, *args)\n" "--\n" @@ -2359,6 +2419,334 @@ exit: return return_value; } +PyDoc_STRVAR(vararg__doc__, +"vararg($module, /, a, *args)\n" +"--\n" +"\n"); + +#define VARARG_METHODDEF \ + {"vararg", _PyCFunction_CAST(vararg), METH_FASTCALL|METH_KEYWORDS, vararg__doc__}, + +static PyObject * +vararg_impl(PyObject *module, PyObject *a, PyObject *args); + +static PyObject * +vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "vararg", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *a; + PyObject *__clinic_args = NULL; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + return_value = vararg_impl(module, a, __clinic_args); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(vararg_with_default__doc__, +"vararg_with_default($module, /, a, *args, b=False)\n" +"--\n" +"\n"); + +#define VARARG_WITH_DEFAULT_METHODDEF \ + {"vararg_with_default", _PyCFunction_CAST(vararg_with_default), METH_FASTCALL|METH_KEYWORDS, vararg_with_default__doc__}, + +static PyObject * +vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args, + int b); + +static PyObject * +vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(a), &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"a", "b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "vararg_with_default", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *a; + PyObject *__clinic_args = NULL; + int b = 0; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + a = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = PyObject_IsTrue(args[2]); + if (b < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = vararg_with_default_impl(module, a, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(vararg_with_only_defaults__doc__, +"vararg_with_only_defaults($module, /, *args, b=None)\n" +"--\n" +"\n"); + +#define VARARG_WITH_ONLY_DEFAULTS_METHODDEF \ + {"vararg_with_only_defaults", _PyCFunction_CAST(vararg_with_only_defaults), METH_FASTCALL|METH_KEYWORDS, vararg_with_only_defaults__doc__}, + +static PyObject * +vararg_with_only_defaults_impl(PyObject *module, PyObject *args, PyObject *b); + +static PyObject * +vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(b), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"b", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "vararg_with_only_defaults", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = 0 + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *__clinic_args = NULL; + PyObject *b = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + b = args[1]; +skip_optional_kwonly: + return_value = vararg_with_only_defaults_impl(module, __clinic_args, b); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + +PyDoc_STRVAR(gh_32092_oob__doc__, +"gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" +"--\n" +"\n" +"Proof-of-concept of GH-32092 OOB bug."); + +#define GH_32092_OOB_METHODDEF \ + {"gh_32092_oob", _PyCFunction_CAST(gh_32092_oob), METH_FASTCALL|METH_KEYWORDS, gh_32092_oob__doc__}, + +static PyObject * +gh_32092_oob_impl(PyObject *module, PyObject *pos1, PyObject *pos2, + PyObject *varargs, PyObject *kw1, PyObject *kw2); + +static PyObject * +gh_32092_oob(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(pos1), &_Py_ID(pos2), &_Py_ID(kw1), &_Py_ID(kw2), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pos1", "pos2", "kw1", "kw2", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "gh_32092_oob", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[5]; + Py_ssize_t noptargs = Py_MIN(nargs, 2) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + PyObject *pos1; + PyObject *pos2; + PyObject *varargs = NULL; + PyObject *kw1 = Py_None; + PyObject *kw2 = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, 2, argsbuf); + if (!args) { + goto exit; + } + pos1 = args[0]; + pos2 = args[1]; + varargs = args[2]; + if (!noptargs) { + goto skip_optional_kwonly; + } + if (args[3]) { + kw1 = args[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + kw2 = args[4]; +skip_optional_kwonly: + return_value = gh_32092_oob_impl(module, pos1, pos2, varargs, kw1, kw2); + +exit: + Py_XDECREF(varargs); + return return_value; +} + +PyDoc_STRVAR(gh_32092_kw_pass__doc__, +"gh_32092_kw_pass($module, /, pos, *args, kw=None)\n" +"--\n" +"\n" +"Proof-of-concept of GH-32092 keyword args passing bug."); + +#define GH_32092_KW_PASS_METHODDEF \ + {"gh_32092_kw_pass", _PyCFunction_CAST(gh_32092_kw_pass), METH_FASTCALL|METH_KEYWORDS, gh_32092_kw_pass__doc__}, + +static PyObject * +gh_32092_kw_pass_impl(PyObject *module, PyObject *pos, PyObject *args, + PyObject *kw); + +static PyObject * +gh_32092_kw_pass(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(pos), &_Py_ID(kw), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pos", "kw", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "gh_32092_kw_pass", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *pos; + PyObject *__clinic_args = NULL; + PyObject *kw = Py_None; + + args = _PyArg_UnpackKeywordsWithVararg(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, 1, argsbuf); + if (!args) { + goto exit; + } + pos = args[0]; + __clinic_args = args[1]; + if (!noptargs) { + goto skip_optional_kwonly; + } + kw = args[2]; +skip_optional_kwonly: + return_value = gh_32092_kw_pass_impl(module, pos, __clinic_args, kw); + +exit: + Py_XDECREF(__clinic_args); + return return_value; +} + PyDoc_STRVAR(gh_99233_refcount__doc__, "gh_99233_refcount($module, /, *args)\n" "--\n" @@ -2423,4 +2811,4 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=49dced2c99bcd0fb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9a5ca5909c087102 input=a9049054013a1b77]*/ diff --git a/Python/getargs.c b/Python/getargs.c index 748209d7d71..0167dd753d8 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -2598,7 +2598,25 @@ _PyArg_UnpackKeywordsWithVararg(PyObject *const *args, Py_ssize_t nargs, current_arg = NULL; } - buf[i + vararg + 1] = current_arg; + /* If an arguments is passed in as a keyword argument, + * it should be placed before `buf[vararg]`. + * + * For example: + * def f(a, /, b, *args): + * pass + * f(1, b=2) + * + * This `buf` array should be: [1, 2, NULL]. + * In this case, nargs < vararg. + * + * Otherwise, we leave a place at `buf[vararg]` for vararg tuple + * so the index is `i + 1`. */ + if (nargs < vararg) { + buf[i] = current_arg; + } + else { + buf[i + 1] = current_arg; + } if (current_arg) { --nkwargs; diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 0117a50725d..0ece814e8f1 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -719,7 +719,7 @@ class CLanguage(Language): vararg = NO_VARARG pos_only = min_pos = max_pos = min_kw_only = pseudo_args = 0 for i, p in enumerate(parameters, 1): - if p.is_keyword_only() or vararg != NO_VARARG: + if p.is_keyword_only(): assert not p.is_positional_only() if not p.is_optional(): min_kw_only = i - max_pos @@ -1016,13 +1016,14 @@ class CLanguage(Language): parser_definition = parser_body(parser_prototype, *parser_code) else: - has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters)) + has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) if vararg == NO_VARARG: args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( min_pos, max_pos, min_kw_only ) + nargs = "nargs" else: args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( min_pos, @@ -1030,6 +1031,7 @@ class CLanguage(Language): min_kw_only, vararg ) + nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if not new_or_init: flags = "METH_FASTCALL|METH_KEYWORDS" parser_prototype = parser_prototype_fastcall_keywords @@ -1037,8 +1039,7 @@ class CLanguage(Language): declarations = declare_parser(f) declarations += "\nPyObject *argsbuf[%s];" % len(converters) if has_optional_kw: - pre_buffer = "0" if vararg != NO_VARARG else "nargs" - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (pre_buffer, min_pos + min_kw_only) + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) parser_code = [normalize_snippet(""" args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); if (!args) {{ @@ -1055,7 +1056,7 @@ class CLanguage(Language): declarations += "\nPyObject * const *fastargs;" declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (min_pos + min_kw_only) + declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) parser_code = [normalize_snippet(""" fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); if (!fastargs) {{