gh-108494: Argument Clinic: fix support of Limited C API (GH-108536)

Lib/test/test_call.py

@@ -65,7 +65,8 @@ class CFunctionCallsErrorMessages(unittest.TestCase):
         self.assertRaisesRegex(TypeError, msg, int.from_bytes, b'a', 'little', False)
 
     def test_varargs1min(self):
-        msg = r"get expected at least 1 argument, got 0"
+        msg = (r"get\(\) takes at least 1 argument \(0 given\)|"
+               r"get expected at least 1 argument, got 0")
         self.assertRaisesRegex(TypeError, msg, {}.get)
 
         msg = r"expected 1 argument, got 0"
@@ -76,11 +77,13 @@ class CFunctionCallsErrorMessages(unittest.TestCase):
         self.assertRaisesRegex(TypeError, msg, getattr)
 
     def test_varargs1max(self):
-        msg = r"input expected at most 1 argument, got 2"
+        msg = (r"input\(\) takes at most 1 argument \(2 given\)|"
+               r"input expected at most 1 argument, got 2")
         self.assertRaisesRegex(TypeError, msg, input, 1, 2)
 
     def test_varargs2max(self):
-        msg = r"get expected at most 2 arguments, got 3"
+        msg = (r"get\(\) takes at most 2 arguments \(3 given\)|"
+               r"get expected at most 2 arguments, got 3")
         self.assertRaisesRegex(TypeError, msg, {}.get, 1, 2, 3)
 
     def test_varargs1_kw(self):
@@ -96,7 +99,7 @@ class CFunctionCallsErrorMessages(unittest.TestCase):
         self.assertRaisesRegex(TypeError, msg, bool, x=2)
 
     def test_varargs4_kw(self):
-        msg = r"^list[.]index\(\) takes no keyword arguments$"
+        msg = r"^(list[.])?index\(\) takes no keyword arguments$"
         self.assertRaisesRegex(TypeError, msg, [].index, x=2)
 
     def test_varargs5_kw(self):

Lib/test/test_enum.py

@@ -2799,11 +2799,13 @@ class TestSpecial(unittest.TestCase):
         class ThirdFailedStrEnum(CustomStrEnum):
             one = '1'
             two = 2  # this will become '2'
-        with self.assertRaisesRegex(TypeError, '.encoding. must be str, not '):
+        with self.assertRaisesRegex(TypeError,
+                r"argument (2|'encoding') must be str, not "):
             class ThirdFailedStrEnum(CustomStrEnum):
                 one = '1'
                 two = b'2', sys.getdefaultencoding
-        with self.assertRaisesRegex(TypeError, '.errors. must be str, not '):
+        with self.assertRaisesRegex(TypeError,
+                r"argument (3|'errors') must be str, not "):
             class ThirdFailedStrEnum(CustomStrEnum):
                 one = '1'
                 two = b'2', 'ascii', 9

Lib/test/test_pyexpat.py

@@ -281,12 +281,10 @@ class NamespaceSeparatorTest(unittest.TestCase):
         expat.ParserCreate(namespace_separator=' ')
 
     def test_illegal(self):
-        try:
+        with self.assertRaisesRegex(TypeError,
+                r"ParserCreate\(\) argument (2|'namespace_separator') "
+                r"must be str or None, not int"):
             expat.ParserCreate(namespace_separator=42)
-            self.fail()
-        except TypeError as e:
-            self.assertEqual(str(e),
-                "ParserCreate() argument 'namespace_separator' must be str or None, not int")
 
         try:
             expat.ParserCreate(namespace_separator='too long')

Modules/_localemodule.c

@@ -11,6 +11,7 @@ This software comes with no warranty. Use at your own risk.
 
 #include "Python.h"
 #include "pycore_fileutils.h"
+#include "pycore_pymem.h"  // _PyMem_Strdup
 
 #include <stdio.h>
 #include <locale.h>

Modules/_sqlite/module.c

@@ -244,7 +244,7 @@ load_functools_lru_cache(PyObject *module)
 static PyMethodDef module_methods[] = {
     PYSQLITE_ADAPT_METHODDEF
     PYSQLITE_COMPLETE_STATEMENT_METHODDEF
-    PYSQLITE_CONNECT_METHODDEF
+    {"connect", _PyCFunction_CAST(pysqlite_connect), METH_FASTCALL|METH_KEYWORDS, pysqlite_connect__doc__},
     PYSQLITE_ENABLE_CALLBACK_TRACE_METHODDEF
     PYSQLITE_REGISTER_ADAPTER_METHODDEF
     PYSQLITE_REGISTER_CONVERTER_METHODDEF

Modules/_struct.c

@@ -108,6 +108,7 @@ class cache_struct_converter(CConverter):
     type = 'PyStructObject *'
     converter = 'cache_struct_converter'
     c_default = "NULL"
+    broken_limited_capi = True
 
     def parse_arg(self, argname, displayname):
         return """
@@ -120,7 +121,7 @@ class cache_struct_converter(CConverter):
     def cleanup(self):
         return "Py_XDECREF(%s);\n" % self.name
 [python start generated code]*/
-/*[python end generated code: output=da39a3ee5e6b4b0d input=d6746621c2fb1a7d]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=14e83804f599ed8f]*/
 
 static int cache_struct_converter(PyObject *, PyObject *, PyStructObject **);
 

Modules/_tracemalloc.c

@@ -1,4 +1,5 @@
 #include "Python.h"
+#include "pycore_tracemalloc.h"  // _PyTraceMalloc_IsTracing
 
 #include "clinic/_tracemalloc.c.h"
 

Modules/clinic/_testclinic_limited.c.h

@@ -37,8 +37,7 @@ my_int_func(PyObject *module, PyObject *arg_)
     int arg;
     int _return_value;
 
-    arg = PyLong_AsInt(arg_);
+    if (!PyArg_Parse(arg_, "i:my_int_func", &arg)) {
-    if (arg == -1 && PyErr_Occurred()) {
         goto exit;
     }
     _return_value = my_int_func_impl(module, arg);
@@ -82,4 +81,4 @@ my_int_sum(PyObject *module, PyObject *args)
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=f9f7209255bb969e input=a9049054013a1b77]*/
+/*[clinic end generated code: output=dcd5203d0d29df3a input=a9049054013a1b77]*/

PC/winreg.c

@@ -220,6 +220,7 @@ class DWORD_converter(unsigned_long_converter):
 class HKEY_converter(CConverter):
     type = 'HKEY'
     converter = 'clinic_HKEY_converter'
+    broken_limited_capi = True
 
     def parse_arg(self, argname, displayname):
         return """
@@ -249,7 +250,7 @@ class self_return_converter(CReturnConverter):
         data.return_conversion.append(
             'return_value = (PyObject *)_return_value;\n')
 [python start generated code]*/
-/*[python end generated code: output=da39a3ee5e6b4b0d input=17e645060c7b8ae1]*/
+/*[python end generated code: output=da39a3ee5e6b4b0d input=f8cb7034338aeaba]*/
 
 #include "clinic/winreg.c.h"
 

Tools/clinic/clinic.py

@@ -975,6 +975,8 @@ class CLanguage(Language):
             func: Function,
             params: dict[int, Parameter],
             argname_fmt: str | None,
+            *,
+            limited_capi: bool,
     ) -> str:
         assert len(params) > 0
         last_param = next(reversed(params.values()))
@@ -986,7 +988,7 @@ class CLanguage(Language):
             if p.is_optional():
                 if argname_fmt:
                     conditions.append(f"nargs < {i+1} && {argname_fmt % i}")
-                elif func.kind.new_or_init:
+                elif func.kind.new_or_init or limited_capi:
                     conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))")
                     containscheck = "PyDict_Contains"
                 else:
@@ -998,7 +1000,9 @@ class CLanguage(Language):
         if len(conditions) > 1:
             condition = f"(({condition}))"
         if last_param.is_optional():
-            if func.kind.new_or_init:
+            if limited_capi:
+                condition = f"kwargs && PyDict_Size(kwargs) && {condition}"
+            elif func.kind.new_or_init:
                 condition = f"kwargs && PyDict_GET_SIZE(kwargs) && {condition}"
             else:
                 condition = f"kwnames && PyTuple_GET_SIZE(kwnames) && {condition}"
@@ -1170,6 +1174,14 @@ class CLanguage(Language):
                 add(field)
             return linear_format(output(), parser_declarations=declarations)
 
+        limited_capi = clinic.limited_capi
+        if limited_capi and (requires_defining_class or pseudo_args or
+                (any(p.is_optional() for p in parameters) and
+                 any(p.is_keyword_only() and not p.is_optional() for p in parameters)) or
+                any(c.broken_limited_capi for c in converters)):
+            warn(f"Function {f.full_name} cannot use limited C API")
+            limited_capi = False
+
         parsearg: str | None
         if not parameters:
             parser_code: list[str] | None
@@ -1230,8 +1242,11 @@ class CLanguage(Language):
                     {c_basename}({self_type}{self_name}, PyObject *%s)
                     """ % argname)
 
-                displayname = parameters[0].get_displayname(0)
+                if limited_capi:
-                parsearg = converters[0].parse_arg(argname, displayname)
+                    parsearg = None
+                else:
+                    displayname = parameters[0].get_displayname(0)
+                    parsearg = converters[0].parse_arg(argname, displayname)
                 if parsearg is None:
                     parsearg = """
                         if (!PyArg_Parse(%s, "{format_units}:{name}", {parse_arguments})) {{
@@ -1250,21 +1265,24 @@ class CLanguage(Language):
             parser_prototype = self.PARSER_PROTOTYPE_VARARGS
             parser_definition = parser_body(parser_prototype, '    {option_group_parsing}')
 
-        elif not requires_defining_class and pos_only == len(parameters) - pseudo_args and clinic.limited_capi:
+        elif (not requires_defining_class and pos_only == len(parameters) and
+              not pseudo_args and limited_capi):
             # positional-only for the limited C API
             flags = "METH_VARARGS"
-
             parser_prototype = self.PARSER_PROTOTYPE_VARARGS
-            parser_code = [normalize_snippet("""
+            if all(isinstance(c, object_converter) and c.format_unit == 'O' for c in converters):
-                if (!PyArg_ParseTuple(args, "{format_units}:{name}",
+                parser_code = [normalize_snippet("""
-                    {parse_arguments}))
+                    if (!PyArg_UnpackTuple(args, "{name}", %d, %d,
-                    goto exit;
+                        {parse_arguments}))
-            """, indent=4)]
+                        goto exit;
-            argname_fmt = 'args[%d]'
+                """ % (min_pos, max_pos), indent=4)]
-            declarations = ""
+            else:
-
+                parser_code = [normalize_snippet("""
-            parser_definition = parser_body(parser_prototype, *parser_code,
+                    if (!PyArg_ParseTuple(args, "{format_units}:{name}",
-                                            declarations=declarations)
+                        {parse_arguments}))
+                        goto exit;
+                """, indent=4)]
+            parser_definition = parser_body(parser_prototype, *parser_code)
 
         elif not requires_defining_class and pos_only == len(parameters) - pseudo_args:
             if not new_or_init:
@@ -1284,7 +1302,6 @@ class CLanguage(Language):
                 nargs = 'PyTuple_GET_SIZE(args)'
                 argname_fmt = 'PyTuple_GET_ITEM(args, %d)'
 
-
             left_args = f"{nargs} - {max_pos}"
             max_args = NO_VARARG if (vararg != NO_VARARG) else max_pos
             parser_code = [normalize_snippet("""
@@ -1384,7 +1401,7 @@ class CLanguage(Language):
                 )
                 nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0"
 
-            if clinic.limited_capi:
+            if limited_capi:
                 # positional-or-keyword arguments
                 flags = "METH_VARARGS|METH_KEYWORDS"
 
@@ -1394,8 +1411,13 @@ class CLanguage(Language):
                         {parse_arguments}))
                         goto exit;
                 """, indent=4)]
-                argname_fmt = 'args[%d]'
+                declarations = "static char *_keywords[] = {{{keywords_c} NULL}};"
-                declarations = ""
+                if deprecated_positionals or deprecated_keywords:
+                    declarations += "\nPy_ssize_t nargs = PyTuple_Size(args);"
+                if deprecated_keywords:
+                    code = self.deprecate_keyword_use(f, deprecated_keywords, None,
+                                                      limited_capi=limited_capi)
+                    parser_code.append(code)
 
             elif not new_or_init:
                 flags = "METH_FASTCALL|METH_KEYWORDS"
@@ -1433,92 +1455,95 @@ class CLanguage(Language):
                 flags = 'METH_METHOD|' + flags
                 parser_prototype = self.PARSER_PROTOTYPE_DEF_CLASS
 
-            if deprecated_keywords:
+            if not limited_capi:
-                code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt)
+                if deprecated_keywords:
-                parser_code.append(code)
+                    code = self.deprecate_keyword_use(f, deprecated_keywords, argname_fmt,
+                                                      limited_capi=limited_capi)
+                    parser_code.append(code)
 
-            add_label: str | None = None
+                add_label: str | None = None
-            for i, p in enumerate(parameters):
+                for i, p in enumerate(parameters):
-                if isinstance(p.converter, defining_class_converter):
+                    if isinstance(p.converter, defining_class_converter):
-                    raise ValueError("defining_class should be the first "
+                        raise ValueError("defining_class should be the first "
-                                     "parameter (after self)")
+                                        "parameter (after self)")
-                displayname = p.get_displayname(i+1)
+                    displayname = p.get_displayname(i+1)
-                parsearg = p.converter.parse_arg(argname_fmt % i, displayname)
+                    parsearg = p.converter.parse_arg(argname_fmt % i, displayname)
-                if parsearg is None:
+                    if parsearg is None:
-                    parser_code = None
+                        parser_code = None
-                    break
+                        break
-                if add_label and (i == pos_only or i == max_pos):
+                    if add_label and (i == pos_only or i == max_pos):
-                    parser_code.append("%s:" % add_label)
+                        parser_code.append("%s:" % add_label)
-                    add_label = None
+                        add_label = None
-                if not p.is_optional():
+                    if not p.is_optional():
-                    parser_code.append(normalize_snippet(parsearg, indent=4))
+                        parser_code.append(normalize_snippet(parsearg, indent=4))
-                elif i < pos_only:
+                    elif i < pos_only:
-                    add_label = 'skip_optional_posonly'
+                        add_label = 'skip_optional_posonly'
-                    parser_code.append(normalize_snippet("""
-                        if (nargs < %d) {{
-                            goto %s;
-                        }}
-                        """ % (i + 1, add_label), indent=4))
-                    if has_optional_kw:
                         parser_code.append(normalize_snippet("""
-                            noptargs--;
+                            if (nargs < %d) {{
-                            """, indent=4))
-                    parser_code.append(normalize_snippet(parsearg, indent=4))
-                else:
-                    if i < max_pos:
-                        label = 'skip_optional_pos'
-                        first_opt = max(min_pos, pos_only)
-                    else:
-                        label = 'skip_optional_kwonly'
-                        first_opt = max_pos + min_kw_only
-                        if vararg != NO_VARARG:
-                            first_opt += 1
-                    if i == first_opt:
-                        add_label = label
-                        parser_code.append(normalize_snippet("""
-                            if (!noptargs) {{
                                 goto %s;
                             }}
-                            """ % add_label, indent=4))
+                            """ % (i + 1, add_label), indent=4))
-                    if i + 1 == len(parameters):
+                        if has_optional_kw:
+                            parser_code.append(normalize_snippet("""
+                                noptargs--;
+                                """, indent=4))
                         parser_code.append(normalize_snippet(parsearg, indent=4))
                     else:
-                        add_label = label
+                        if i < max_pos:
-                        parser_code.append(normalize_snippet("""
+                            label = 'skip_optional_pos'
-                            if (%s) {{
+                            first_opt = max(min_pos, pos_only)
-                            """ % (argname_fmt % i), indent=4))
+                        else:
-                        parser_code.append(normalize_snippet(parsearg, indent=8))
+                            label = 'skip_optional_kwonly'
-                        parser_code.append(normalize_snippet("""
+                            first_opt = max_pos + min_kw_only
-                                if (!--noptargs) {{
+                            if vararg != NO_VARARG:
+                                first_opt += 1
+                        if i == first_opt:
+                            add_label = label
+                            parser_code.append(normalize_snippet("""
+                                if (!noptargs) {{
                                     goto %s;
                                 }}
-                            }}
+                                """ % add_label, indent=4))
-                            """ % add_label, indent=4))
+                        if i + 1 == len(parameters):
+                            parser_code.append(normalize_snippet(parsearg, indent=4))
+                        else:
+                            add_label = label
+                            parser_code.append(normalize_snippet("""
+                                if (%s) {{
+                                """ % (argname_fmt % i), indent=4))
+                            parser_code.append(normalize_snippet(parsearg, indent=8))
+                            parser_code.append(normalize_snippet("""
+                                    if (!--noptargs) {{
+                                        goto %s;
+                                    }}
+                                }}
+                                """ % add_label, indent=4))
 
-            if parser_code is not None:
+                if parser_code is not None:
-                if add_label:
+                    if add_label:
-                    parser_code.append("%s:" % add_label)
+                        parser_code.append("%s:" % add_label)
-            else:
-                declarations = declare_parser(f, hasformat=True)
-                if not new_or_init:
-                    parser_code = [normalize_snippet("""
-                        if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma}
-                            {parse_arguments})) {{
-                            goto exit;
-                        }}
-                        """, indent=4)]
                 else:
-                    parser_code = [normalize_snippet("""
+                    declarations = declare_parser(f, hasformat=True)
-                        if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
+                    if not new_or_init:
-                            {parse_arguments})) {{
+                        parser_code = [normalize_snippet("""
-                            goto exit;
+                            if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser{parse_arguments_comma}
-                        }}
+                                {parse_arguments})) {{
-                        """, indent=4)]
+                                goto exit;
-                    if deprecated_positionals or deprecated_keywords:
+                            }}
-                        declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
+                            """, indent=4)]
-                if deprecated_keywords:
+                    else:
-                    code = self.deprecate_keyword_use(f, deprecated_keywords, None)
+                        parser_code = [normalize_snippet("""
-                    parser_code.append(code)
+                            if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser,
+                                {parse_arguments})) {{
+                                goto exit;
+                            }}
+                            """, indent=4)]
+                        if deprecated_positionals or deprecated_keywords:
+                            declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);"
+                    if deprecated_keywords:
+                        code = self.deprecate_keyword_use(f, deprecated_keywords, None,
+                                                          limited_capi=limited_capi)
+                        parser_code.append(code)
 
             if deprecated_positionals:
                 code = self.deprecate_positional_use(f, deprecated_positionals)
@@ -3098,6 +3123,8 @@ class CConverter(metaclass=CConverterAutoRegister):
     # "#include "name"     // reason"
     include: tuple[str, str] | None = None
 
+    broken_limited_capi: bool = False
+
     # keep in sync with self_converter.__init__!
     def __init__(self,
              # Positional args:
@@ -3262,7 +3289,7 @@ class CConverter(metaclass=CConverterAutoRegister):
         elif self.subclass_of:
             args.append(self.subclass_of)
 
-        s = ("&" if self.parse_by_reference else "") + self.name
+        s = ("&" if self.parse_by_reference else "") + self.parser_name
         args.append(s)
 
         if self.length: