bpo-42685: Improve placing of simple query windows. (GH-23856)

* If parent is specified and mapped, the query widget is
  centered at the center of parent. Its position and size
  can be corrected so that it fits in the virtual root window.
* Otherwise it is centered at the center of the screen.

Lib/enum.py

@@ -178,7 +178,7 @@ class EnumMeta(type):
     Metaclass for Enum
     """
     @classmethod
-    def __prepare__(metacls, cls, bases):
+    def __prepare__(metacls, cls, bases, **kwds):
         # check that previous enum members do not exist
         metacls._check_for_existing_members(cls, bases)
         # create the namespace dict

Lib/test/test_enum.py

@@ -2119,7 +2119,7 @@ class TestEnum(unittest.TestCase):
                 one = '1'
                 two = b'2', 'ascii', 9
                 
-    def test_init_subclass(self):
+    def test_init_subclass_calling(self):
         class MyEnum(Enum):
             def __init_subclass__(cls, **kwds):
                 super(MyEnum, cls).__init_subclass__(**kwds)
@@ -2155,6 +2155,16 @@ class TestEnum(unittest.TestCase):
         self.assertFalse(NeverEnum.__dict__.get('_test1', False))
         self.assertFalse(NeverEnum.__dict__.get('_test2', False))
 
+    def test_init_subclass_parameter(self):
+        class multiEnum(Enum):
+            def __init_subclass__(cls, multi):
+                for member in cls:
+                    member._as_parameter_ = multi * member.value
+        class E(multiEnum, multi=3):
+            A = 1
+            B = 2
+        self.assertEqual(E.A._as_parameter_, 3)
+        self.assertEqual(E.B._as_parameter_, 6)
 
     @unittest.skipUnless(
             sys.version_info[:2] == (3, 9),

Lib/tkinter/simpledialog.py

@@ -55,36 +55,8 @@ class SimpleDialog:
                 b.config(relief=RIDGE, borderwidth=8)
             b.pack(side=LEFT, fill=BOTH, expand=1)
         self.root.protocol('WM_DELETE_WINDOW', self.wm_delete_window)
-        self._set_transient(master)
-
-    def _set_transient(self, master, relx=0.5, rely=0.3):
-        widget = self.root
-        widget.withdraw() # Remain invisible while we figure out the geometry
-        widget.transient(master)
-        widget.update_idletasks() # Actualize geometry information
-        if master.winfo_ismapped():
-            m_width = master.winfo_width()
-            m_height = master.winfo_height()
-            m_x = master.winfo_rootx()
-            m_y = master.winfo_rooty()
-        else:
-            m_width = master.winfo_screenwidth()
-            m_height = master.winfo_screenheight()
-            m_x = m_y = 0
-        w_width = widget.winfo_reqwidth()
-        w_height = widget.winfo_reqheight()
-        x = m_x + (m_width - w_width) * relx
-        y = m_y + (m_height - w_height) * rely
-        if x+w_width > master.winfo_screenwidth():
-            x = master.winfo_screenwidth() - w_width
-        elif x < 0:
-            x = 0
-        if y+w_height > master.winfo_screenheight():
-            y = master.winfo_screenheight() - w_height
-        elif y < 0:
-            y = 0
-        widget.geometry("+%d+%d" % (x, y))
-        widget.deiconify() # Become visible at the desired location
+        self.root.transient(master)
+        _place_window(self.root, master)
 
     def go(self):
         self.root.wait_visibility()
@@ -157,11 +129,7 @@ class Dialog(Toplevel):
 
         self.protocol("WM_DELETE_WINDOW", self.cancel)
 
-        if parent is not None:
-            self.geometry("+%d+%d" % (parent.winfo_rootx()+50,
-                                      parent.winfo_rooty()+50))
-
-        self.deiconify() # become visible now
+        _place_window(self, parent)
 
         self.initial_focus.focus_set()
 
@@ -251,6 +219,37 @@ class Dialog(Toplevel):
         pass # override
 
 
+# Place a toplevel window at the center of parent or screen
+# It is a Python implementation of ::tk::PlaceWindow.
+def _place_window(w, parent=None):
+    w.wm_withdraw() # Remain invisible while we figure out the geometry
+    w.update_idletasks() # Actualize geometry information
+
+    minwidth = w.winfo_reqwidth()
+    minheight = w.winfo_reqheight()
+    maxwidth = w.winfo_vrootwidth()
+    maxheight = w.winfo_vrootheight()
+    if parent is not None and parent.winfo_ismapped():
+        x = parent.winfo_rootx() + (parent.winfo_width() - minwidth) // 2
+        y = parent.winfo_rooty() + (parent.winfo_height() - minheight) // 2
+        vrootx = w.winfo_vrootx()
+        vrooty = w.winfo_vrooty()
+        x = min(x, vrootx + maxwidth - minwidth)
+        x = max(x, vrootx)
+        y = min(y, vrooty + maxheight - minheight)
+        y = max(y, vrooty)
+        if w._windowingsystem == 'aqua':
+            # Avoid the native menu bar which sits on top of everything.
+            y = max(y, 22)
+    else:
+        x = (w.winfo_screenwidth() - minwidth) // 2
+        y = (w.winfo_screenheight() - minheight) // 2
+
+    w.wm_maxsize(maxwidth, maxheight)
+    w.wm_geometry('+%d+%d' % (x, y))
+    w.wm_deiconify() # Become visible at the desired location
+
+
 # --------------------------------------------------------------------
 # convenience dialogues
 

Misc/NEWS.d/next/Library/2020-12-19-17-32-43.bpo-42685.kwZlwp.rst

@@ -0,0 +1,4 @@
+Improved placing of simple query windows in Tkinter (such as
+:func:`tkinter.simpledialog.askinteger`). They are now centered at the
+center of the parent window if it is specified and shown, otherwise at the
+center of the screen.

Misc/NEWS.d/next/Library/2020-12-23-19-43-06.bpo-42727.WH3ODh.rst

@@ -0,0 +1,2 @@
+`EnumMeta.__prepare__` now accepts `**kwds` to properly support
+`__init_subclass__`

Misc/NEWS.d/next/Tools-Demos/2020-12-23-19-42-11.bpo-42726.a5EkTv.rst

@@ -0,0 +1,2 @@
+Fixed Python 3 compatibility issue with gdb/libpython.py handling of attribute
+dictionaries.

Tools/c-analyzer/c_analyzer/__main__.py

@@ -263,7 +263,7 @@ FORMATS = {
 def add_output_cli(parser, *, default='summary'):
     parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS))
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         pass
     return process_args
 
@@ -280,7 +280,7 @@ def _cli_check(parser, checks=None, **kwargs):
         process_checks = add_checks_cli(parser)
     elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]):
         check = checks[0][1:-1]
-        def process_checks(args):
+        def process_checks(args, *, argv=None):
             args.checks = [check]
     else:
         process_checks = add_checks_cli(parser, checks=checks)
@@ -428,9 +428,9 @@ def _cli_data(parser, filenames=None, known=None):
     if known is None:
         sub.add_argument('--known', required=True)
 
-    def process_args(args):
+    def process_args(args, *, argv):
         if args.datacmd == 'dump':
-            process_progress(args)
+            process_progress(args, argv)
     return process_args
 
 
@@ -515,6 +515,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None):
 
     verbosity, traceback_cm = process_args_by_key(
         args,
+        argv,
         processors[cmd],
         ['verbosity', 'traceback_cm'],
     )

Tools/c-analyzer/c_common/scriptutil.py

@@ -192,7 +192,7 @@ def add_verbosity_cli(parser):
     parser.add_argument('-q', '--quiet', action='count', default=0)
     parser.add_argument('-v', '--verbose', action='count', default=0)
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         ns = vars(args)
         key = 'verbosity'
         if key in ns:
@@ -208,7 +208,7 @@ def add_traceback_cli(parser):
     parser.add_argument('--no-traceback', '--no-tb', dest='traceback',
                         action='store_const', const=False)
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         ns = vars(args)
         key = 'traceback_cm'
         if key in ns:
@@ -262,7 +262,7 @@ def add_sepval_cli(parser, opt, dest, choices, *, sep=',', **kwargs):
         #kwargs.setdefault('metavar', opt.upper())
         parser.add_argument(opt, dest=dest, action='append', **kwargs)
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         ns = vars(args)
 
         # XXX Use normalize_selection()?
@@ -293,7 +293,7 @@ def add_file_filtering_cli(parser, *, excluded=None):
 
     excluded = tuple(excluded or ())
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         ns = vars(args)
         key = 'iter_filenames'
         if key in ns:
@@ -323,7 +323,7 @@ def add_progress_cli(parser, *, threshold=VERBOSITY, **kwargs):
     parser.add_argument('--no-progress', dest='track_progress', action='store_false')
     parser.set_defaults(track_progress=True)
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         if args.track_progress:
             ns = vars(args)
             verbosity = ns.get('verbosity', VERBOSITY)
@@ -339,7 +339,7 @@ def add_failure_filtering_cli(parser, pool, *, default=False):
                         metavar=f'"{{all|{"|".join(sorted(pool))}}},..."')
     parser.add_argument('--no-fail', dest='fail', action='store_const', const=())
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         ns = vars(args)
 
         fail = ns.pop('fail')
@@ -371,7 +371,7 @@ def add_failure_filtering_cli(parser, pool, *, default=False):
 def add_kind_filtering_cli(parser, *, default=None):
     parser.add_argument('--kinds', action='append')
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         ns = vars(args)
 
         kinds = []
@@ -486,18 +486,18 @@ def _flatten_processors(processors):
             yield from _flatten_processors(proc)
 
 
-def process_args(args, processors, *, keys=None):
+def process_args(args, argv, processors, *, keys=None):
     processors = _flatten_processors(processors)
     ns = vars(args)
     extracted = {}
     if keys is None:
         for process_args in processors:
-            for key in process_args(args):
+            for key in process_args(args, argv=argv):
                 extracted[key] = ns.pop(key)
     else:
         remainder = set(keys)
         for process_args in processors:
-            hanging = process_args(args)
+            hanging = process_args(args, argv=argv)
             if isinstance(hanging, str):
                 hanging = [hanging]
             for key in hanging or ():
@@ -510,8 +510,8 @@ def process_args(args, processors, *, keys=None):
     return extracted
 
 
-def process_args_by_key(args, processors, keys):
-    extracted = process_args(args, processors, keys=keys)
+def process_args_by_key(args, argv, processors, keys):
+    extracted = process_args(args, argv, processors, keys=keys)
     return [extracted[key] for key in keys]
 
 

Tools/c-analyzer/c_common/tables.py

@@ -1,4 +1,6 @@
 import csv
+import re
+import textwrap
 
 from . import NOT_SET, strutil, fsutil
 
@@ -212,3 +214,177 @@ def _normalize_table_file_props(header, sep):
         else:
             sep = None
     return header, sep
+
+
+##################################
+# stdout tables
+
+WIDTH = 20
+
+
+def resolve_columns(specs):
+    if isinstance(specs, str):
+        specs = specs.replace(',', ' ').strip().split()
+    return _resolve_colspecs(specs)
+
+
+def build_table(specs, *, sep=' ', defaultwidth=None):
+    columns = resolve_columns(specs)
+    return _build_table(columns, sep=sep, defaultwidth=defaultwidth)
+
+
+_COLSPEC_RE = re.compile(textwrap.dedent(r'''
+    ^
+    (?:
+        [[]
+        (
+            (?: [^\s\]] [^\]]* )?
+            [^\s\]]
+        )  # <label>
+        []]
+    )?
+    ( \w+ )  # <field>
+    (?:
+        (?:
+            :
+            ( [<^>] )  # <align>
+            ( \d+ )  # <width1>
+        )
+        |
+        (?:
+            (?:
+                :
+                ( \d+ )  # <width2>
+            )?
+            (?:
+                :
+                ( .*? )  # <fmt>
+            )?
+        )
+    )?
+    $
+'''), re.VERBOSE)
+
+
+def _parse_fmt(fmt):
+    if fmt.startswith(tuple('<^>')):
+        align = fmt[0]
+        width = fmt[1:]
+        if width.isdigit():
+            return int(width), align
+    return None, None
+
+
+def _parse_colspec(raw):
+    m = _COLSPEC_RE.match(raw)
+    if not m:
+        return None
+    label, field, align, width1, width2, fmt = m.groups()
+    if not label:
+        label = field
+    if width1:
+        width = None
+        fmt = f'{align}{width1}'
+    elif width2:
+        width = int(width2)
+        if fmt:
+            _width, _ = _parse_fmt(fmt)
+            if _width == width:
+                width = None
+    else:
+        width = None
+    return field, label, width, fmt
+
+
+def _normalize_colspec(spec):
+    if len(spec) == 1:
+        raw, = spec
+        return _resolve_column(raw)
+
+    if len(spec) == 4:
+        label, field, width, fmt = spec
+        if width:
+            fmt = f'{width}:{fmt}' if fmt else width
+    elif len(raw) == 3:
+        label, field, fmt = spec
+        if not field:
+            label, field = None, label
+        elif not isinstance(field, str) or not field.isidentifier():
+            fmt = f'{field}:{fmt}' if fmt else field
+            label, field = None, label
+    elif len(raw) == 2:
+        label = None
+        field, fmt = raw
+        if not field:
+            field, fmt = fmt, None
+        elif not field.isidentifier() or fmt.isidentifier():
+            label, field = field, fmt
+    else:
+        raise NotImplementedError
+
+    fmt = f':{fmt}' if fmt else ''
+    if label:
+        return _parse_colspec(f'[{label}]{field}{fmt}')
+    else:
+        return _parse_colspec(f'{field}{fmt}')
+
+
+def _resolve_colspec(raw):
+    if isinstance(raw, str):
+        spec = _parse_colspec(raw)
+    else:
+        spec = _normalize_colspec(raw)
+    if spec is None:
+        raise ValueError(f'unsupported column spec {raw!r}')
+    return spec
+
+
+def _resolve_colspecs(columns):
+    parsed = []
+    for raw in columns:
+        column = _resolve_colspec(raw)
+        parsed.append(column)
+    return parsed
+
+
+def _resolve_width(spec, defaultwidth):
+    _, label, width, fmt = spec
+    if width:
+        if not isinstance(width, int):
+            raise NotImplementedError
+        return width
+    elif width and fmt:
+        width, _ = _parse_fmt(fmt)
+        if width:
+            return width
+
+    if not defaultwidth:
+        return WIDTH
+    elif not hasattr(defaultwidth, 'get'):
+        return defaultwidth or WIDTH
+
+    defaultwidths = defaultwidth
+    defaultwidth = defaultwidths.get(None) or WIDTH
+    return defaultwidths.get(label) or defaultwidth
+
+
+def _build_table(columns, *, sep=' ', defaultwidth=None):
+    header = []
+    div = []
+    rowfmt = []
+    for spec in columns:
+        label, field, _, colfmt = spec
+        width = _resolve_width(spec, defaultwidth)
+        if colfmt:
+            colfmt = f':{colfmt}'
+        else:
+            colfmt = f':{width}'
+
+        header.append(f' {{:^{width}}} '.format(label))
+        div.append('-' * (width + 2))
+        rowfmt.append(f' {{{field}{colfmt}}} ')
+    return (
+        sep.join(header),
+        sep.join(div),
+        sep.join(rowfmt),
+    )

Tools/c-analyzer/c_parser/__main__.py

@@ -149,7 +149,7 @@ def add_output_cli(parser):
     parser.add_argument('--showfwd', action='store_true', default=None)
     parser.add_argument('--no-showfwd', dest='showfwd', action='store_false', default=None)
 
-    def process_args(args):
+    def process_args(args, *, argv=None):
         pass
     return process_args
 
@@ -243,6 +243,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset='parse'):
 
     verbosity, traceback_cm = process_args_by_key(
         args,
+        argv,
         processors[cmd],
         ['verbosity', 'traceback_cm'],
     )

Tools/c-analyzer/c_parser/preprocessor/__main__.py

@@ -40,10 +40,10 @@ def add_common_cli(parser, *, get_preprocessor=_get_preprocessor):
     parser.add_argument('--same', action='append')
     process_fail_arg = add_failure_filtering_cli(parser, FAIL)
 
-    def process_args(args):
+    def process_args(args, *, argv):
         ns = vars(args)
 
-        process_fail_arg(args)
+        process_fail_arg(args, argv)
         ignore_exc = ns.pop('ignore_exc')
         # We later pass ignore_exc to _get_preprocessor().
 
@@ -174,6 +174,7 @@ def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *,
 
     verbosity, traceback_cm = process_args_by_key(
         args,
+        argv,
         processors[cmd],
         ['verbosity', 'traceback_cm'],
     )

Tools/c-analyzer/check-c-globals.py

@@ -22,6 +22,7 @@ def parse_args():
     cmd = 'check'
     verbosity, traceback_cm = process_args_by_key(
         args,
+        argv,
         processors,
         ['verbosity', 'traceback_cm'],
     )

Tools/c-analyzer/cpython/__main__.py

@@ -3,11 +3,14 @@ import sys
 
 from c_common.fsutil import expand_filenames, iter_files_by_suffix
 from c_common.scriptutil import (
+    VERBOSITY,
     add_verbosity_cli,
     add_traceback_cli,
     add_commands_cli,
     add_kind_filtering_cli,
     add_files_cli,
+    add_progress_cli,
+    main_for_filenames,
     process_args_by_key,
     configure_logger,
     get_prog,
@@ -17,7 +20,7 @@ import c_parser.__main__ as c_parser
 import c_analyzer.__main__ as c_analyzer
 import c_analyzer as _c_analyzer
 from c_analyzer.info import UNKNOWN
-from . import _analyzer, _parser, REPO_ROOT
+from . import _analyzer, _capi, _files, _parser, REPO_ROOT
 
 
 logger = logging.getLogger(__name__)
@@ -25,9 +28,9 @@ logger = logging.getLogger(__name__)
 
 def _resolve_filenames(filenames):
     if filenames:
-        resolved = (_parser.resolve_filename(f) for f in filenames)
+        resolved = (_files.resolve_filename(f) for f in filenames)
     else:
-        resolved = _parser.iter_filenames()
+        resolved = _files.iter_filenames()
     return resolved
 
 
@@ -204,6 +207,95 @@ def cmd_data(datacmd, **kwargs):
     )
 
 
+def _cli_capi(parser):
+    parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
+    parser.add_argument(f'--public', dest='levels',
+                        action='append_const', const='public')
+    parser.add_argument(f'--no-public', dest='levels',
+                        action='append_const', const='no-public')
+    for level in _capi.LEVELS:
+        parser.add_argument(f'--{level}', dest='levels',
+                            action='append_const', const=level)
+    def process_levels(args, *, argv=None):
+        levels = []
+        for raw in args.levels or ():
+            for level in raw.replace(',', ' ').strip().split():
+                if level == 'public':
+                    levels.append('stable')
+                    levels.append('cpython')
+                elif level == 'no-public':
+                    levels.append('private')
+                    levels.append('internal')
+                elif level in _capi.LEVELS:
+                    levels.append(level)
+                else:
+                    parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
+        args.levels = set(levels)
+
+    parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
+    for kind in _capi.KINDS:
+        parser.add_argument(f'--{kind}', dest='kinds',
+                            action='append_const', const=kind)
+    def process_kinds(args, *, argv=None):
+        kinds = []
+        for raw in args.kinds or ():
+            for kind in raw.replace(',', ' ').strip().split():
+                if kind in _capi.KINDS:
+                    kind.append(kind)
+                else:
+                    parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
+        args.kinds = set(kinds)
+
+    parser.add_argument('--group-by', dest='groupby',
+                        choices=['level', 'kind'])
+
+    parser.add_argument('--format', default='brief')
+    parser.add_argument('--summary', dest='format',
+                        action='store_const', const='summary')
+    def process_format(args, *, argv=None):
+        orig = args.format
+        args.format = _capi.resolve_format(args.format)
+        if isinstance(args.format, str):
+            if args.format not in _capi._FORMATS:
+                parser.error(f'unsupported format {orig!r}')
+
+    parser.add_argument('filenames', nargs='*', metavar='FILENAME')
+    process_progress = add_progress_cli(parser)
+
+    return [
+        process_levels,
+        process_format,
+        process_progress,
+    ]
+
+
+def cmd_capi(filenames=None, *,
+             levels=None,
+             kinds=None,
+             groupby='kind',
+             format='brief',
+             track_progress=None,
+             verbosity=VERBOSITY,
+             **kwargs
+             ):
+    render = _capi.get_renderer(format)
+
+    filenames = _files.iter_header_files(filenames, levels=levels)
+    #filenames = (file for file, _ in main_for_filenames(filenames))
+    if track_progress is not None:
+        filenames = track_progress(filenames)
+    items = _capi.iter_capi(filenames)
+    if levels:
+        items = (item for item in items if item.level in levels)
+    if kinds:
+        items = (item for item in items if item.kind in kinds)
+
+    lines = render(items, groupby=groupby, verbose=verbosity > VERBOSITY)
+    print()
+    for line in lines:
+        print(line)
+
+
 # We do not define any other cmd_*() handlers here,
 # favoring those defined elsewhere.
 
@@ -228,6 +320,11 @@ COMMANDS = {
         [_cli_data],
         cmd_data,
     ),
+    'capi': (
+        'inspect the C-API',
+        [_cli_capi],
+        cmd_capi,
+    ),
 }
 
 
@@ -263,6 +360,7 @@ def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
 
     verbosity, traceback_cm = process_args_by_key(
         args,
+        argv,
         processors[cmd],
         ['verbosity', 'traceback_cm'],
     )

Tools/c-analyzer/cpython/_capi.py

@@ -0,0 +1,479 @@
+from collections import namedtuple
+import os
+import os.path
+import re
+import textwrap
+
+from c_common.tables import build_table, resolve_columns
+from c_parser.parser._regexes import _ind
+from ._files import iter_header_files, resolve_filename
+from . import REPO_ROOT
+
+
+INCLUDE_ROOT = os.path.join(REPO_ROOT, 'Include')
+INCLUDE_CPYTHON = os.path.join(INCLUDE_ROOT, 'cpython')
+INCLUDE_INTERNAL = os.path.join(INCLUDE_ROOT, 'internal')
+
+_MAYBE_NESTED_PARENS = textwrap.dedent(r'''
+    (?:
+        (?: [^(]* [(] [^()]* [)] )* [^(]*
+    )
+''')
+
+CAPI_FUNC = textwrap.dedent(rf'''
+    (?:
+        ^
+        \s*
+        PyAPI_FUNC \s*
+        [(]
+        {_ind(_MAYBE_NESTED_PARENS, 2)}
+        [)] \s*
+        (\w+)  # <func>
+        \s* [(]
+    )
+''')
+CAPI_DATA = textwrap.dedent(rf'''
+    (?:
+        ^
+        \s*
+        PyAPI_DATA \s*
+        [(]
+        {_ind(_MAYBE_NESTED_PARENS, 2)}
+        [)] \s*
+        (\w+)  # <data>
+        \b [^(]
+    )
+''')
+CAPI_INLINE = textwrap.dedent(r'''
+    (?:
+        ^
+        \s*
+        static \s+ inline \s+
+        .*?
+        \s+
+        ( \w+ )  # <inline>
+        \s* [(]
+    )
+''')
+CAPI_MACRO = textwrap.dedent(r'''
+    (?:
+        (\w+)  # <macro>
+        [(]
+    )
+''')
+CAPI_CONSTANT = textwrap.dedent(r'''
+    (?:
+        (\w+)  # <constant>
+        \s+ [^(]
+    )
+''')
+CAPI_DEFINE = textwrap.dedent(rf'''
+    (?:
+        ^
+        \s* [#] \s* define \s+
+        (?:
+            {_ind(CAPI_MACRO, 3)}
+            |
+            {_ind(CAPI_CONSTANT, 3)}
+            |
+            (?:
+                # ignored
+                \w+   # <defined_name>
+                \s*
+                $
+            )
+        )
+    )
+''')
+CAPI_RE = re.compile(textwrap.dedent(rf'''
+    (?:
+        {_ind(CAPI_FUNC, 2)}
+        |
+        {_ind(CAPI_DATA, 2)}
+        |
+        {_ind(CAPI_INLINE, 2)}
+        |
+        {_ind(CAPI_DEFINE, 2)}
+    )
+'''), re.VERBOSE)
+
+KINDS = [
+    'func',
+    'data',
+    'inline',
+    'macro',
+    'constant',
+]
+
+
+def _parse_line(line, prev=None):
+    last = line
+    if prev:
+        if not prev.endswith(os.linesep):
+            prev += os.linesep
+        line = prev + line
+    m = CAPI_RE.match(line)
+    if not m:
+        if not prev and line.startswith('static inline '):
+            return line  # the new "prev"
+        #if 'PyAPI_' in line or '#define ' in line or ' define ' in line:
+        #    print(line)
+        return None
+    results = zip(KINDS, m.groups())
+    for kind, name in results:
+        if name:
+            clean = last.split('//')[0].strip()
+            if clean.endswith('*/'):
+                clean = clean.split('/*')[0].rstrip()
+            if kind == 'macro' or kind == 'constant':
+                if clean.endswith('\\'):
+                    return line  # the new "prev"
+            elif kind == 'inline':
+                if not prev:
+                    if not clean.endswith('}'):
+                        return line  # the new "prev"
+                elif clean != '}':
+                    return line  # the new "prev"
+            elif not clean.endswith(';'):
+                return line  # the new "prev"
+            return name, kind
+    # It was a plain #define.
+    return None
+
+
+LEVELS = {
+    'stable',
+    'cpython',
+    'private',
+    'internal',
+}
+
+def _get_level(filename, name, *,
+               _cpython=INCLUDE_CPYTHON + os.path.sep,
+               _internal=INCLUDE_INTERNAL + os.path.sep,
+               ):
+    if filename.startswith(_internal):
+        return 'internal'
+    elif name.startswith('_'):
+        return 'private'
+    elif os.path.dirname(filename) == INCLUDE_ROOT:
+        return 'stable'
+    elif filename.startswith(_cpython):
+        return 'cpython'
+    else:
+        raise NotImplementedError
+    #return '???'
+
+
+class CAPIItem(namedtuple('CAPIItem', 'file lno name kind level')):
+
+    @classmethod
+    def from_line(cls, line, filename, lno, prev=None):
+        parsed = _parse_line(line, prev)
+        if not parsed:
+            return None, None
+        if isinstance(parsed, str):
+            # incomplete
+            return None, parsed
+        name, kind = parsed
+        level = _get_level(filename, name)
+        self = cls(filename, lno, name, kind, level)
+        if prev:
+            self._text = (prev + line).rstrip().splitlines()
+        else:
+            self._text = [line.rstrip()]
+        return self, None
+
+    @property
+    def relfile(self):
+        return self.file[len(REPO_ROOT) + 1:]
+
+    @property
+    def text(self):
+        try:
+            return self._text
+        except AttributeError:
+            # XXX Actually ready the text from disk?.
+            self._text = []
+            if self.kind == 'data':
+                self._text = [
+                    f'PyAPI_DATA(...) {self.name}',
+                ]
+            elif self.kind == 'func':
+                self._text = [
+                    f'PyAPI_FUNC(...) {self.name}(...);',
+                ]
+            elif self.kind == 'inline':
+                self._text = [
+                    f'static inline {self.name}(...);',
+                ]
+            elif self.kind == 'macro':
+                self._text = [
+                    f'#define {self.name}(...) \\',
+                    f'    ...',
+                ]
+            elif self.kind == 'constant':
+                self._text = [
+                    f'#define {self.name} ...',
+                ]
+            else:
+                raise NotImplementedError
+
+            return self._text
+
+
+def _parse_groupby(raw):
+    if not raw:
+        raw = 'kind'
+
+    if isinstance(raw, str):
+        groupby = raw.replace(',', ' ').strip().split()
+    else:
+        raise NotImplementedError
+
+    if not all(v in ('kind', 'level') for v in groupby):
+        raise ValueError(f'invalid groupby value {raw!r}')
+    return groupby
+
+
+def summarize(items, *, groupby='kind'):
+    summary = {}
+
+    groupby = _parse_groupby(groupby)[0]
+    if groupby == 'kind':
+        outers = KINDS
+        inners = LEVELS
+        def increment(item):
+            summary[item.kind][item.level] += 1
+    elif groupby == 'level':
+        outers = LEVELS
+        inners = KINDS
+        def increment(item):
+            summary[item.level][item.kind] += 1
+    else:
+        raise NotImplementedError
+
+    for outer in outers:
+        summary[outer] = _outer = {}
+        for inner in inners:
+            _outer[inner] = 0
+    for item in items:
+        increment(item)
+
+    return summary
+
+
+def _parse_capi(lines, filename):
+    if isinstance(lines, str):
+        lines = lines.splitlines()
+    prev = None
+    for lno, line in enumerate(lines, 1):
+        parsed, prev = CAPIItem.from_line(line, filename, lno, prev)
+        if parsed:
+            yield parsed
+    if prev:
+        parsed, prev = CAPIItem.from_line('', filename, lno, prev)
+        if parsed:
+            yield parsed
+        if prev:
+            print('incomplete match:')
+            print(filename)
+            print(prev)
+            raise Exception
+
+
+def iter_capi(filenames=None):
+    for filename in iter_header_files(filenames):
+        with open(filename) as infile:
+            for item in _parse_capi(infile, filename):
+                yield item
+
+
+def _collate(items, groupby):
+    groupby = _parse_groupby(groupby)[0]
+    maxfilename = maxname = maxkind = maxlevel = 0
+    collated = {}
+    for item in items:
+        key = getattr(item, groupby)
+        if key in collated:
+            collated[key].append(item)
+        else:
+            collated[key] = [item]
+        maxfilename = max(len(item.relfile), maxfilename)
+        maxname = max(len(item.name), maxname)
+        maxkind = max(len(item.kind), maxkind)
+        maxlevel = max(len(item.level), maxlevel)
+    maxextra = {
+        'kind': maxkind,
+        'level': maxlevel,
+    }
+    return collated, groupby, maxfilename, maxname, maxextra
+
+
+##################################
+# CLI rendering
+
+_LEVEL_MARKERS = {
+    'S': 'stable',
+    'C': 'cpython',
+    'P': 'private',
+    'I': 'internal',
+}
+_KIND_MARKERS = {
+    'F': 'func',
+    'D': 'data',
+    'I': 'inline',
+    'M': 'macro',
+    'C': 'constant',
+}
+
+
+def resolve_format(format):
+    if not format:
+        return 'brief'
+    elif isinstance(format, str) and format in _FORMATS:
+        return format
+    else:
+        return resolve_columns(format)
+
+
+def get_renderer(format):
+    format = resolve_format(format)
+    if isinstance(format, str):
+        try:
+            return _FORMATS[format]
+        except KeyError:
+            raise ValueError(f'unsupported format {format!r}')
+    else:
+        def render(items, **kwargs):
+            return render_table(items, columns=format, **kwargs)
+        return render
+
+
+def render_table(items, *, columns=None, groupby='kind', verbose=False):
+    if groupby:
+        collated, groupby, maxfilename, maxname, maxextra = _collate(items, groupby)
+        if groupby == 'kind':
+            groups = KINDS
+            extras = ['level']
+            markers = {'level': _LEVEL_MARKERS}
+        elif groupby == 'level':
+            groups = LEVELS
+            extras = ['kind']
+            markers = {'kind': _KIND_MARKERS}
+        else:
+            raise NotImplementedError
+    else:
+        # XXX Support no grouping?
+        raise NotImplementedError
+
+    if columns:
+        def get_extra(item):
+            return {extra: getattr(item, extra)
+                    for extra in ('kind', 'level')}
+    else:
+        if verbose:
+            maxextra['kind'] = max(len(kind) for kind in KINDS)
+            maxextra['level'] = max(len(level) for level in LEVELS)
+            extracols = [f'{extra}:{maxextra[extra]}'
+                         for extra in extras]
+            def get_extra(item):
+                return {extra: getattr(item, extra)
+                        for extra in extras}
+        elif len(extras) == 1:
+            extra, = extras
+            extracols = [f'{m}:1' for m in markers[extra]]
+            def get_extra(item):
+                return {m: m if getattr(item, extra) == markers[extra][m] else ''
+                        for m in markers[extra]}
+        else:
+            raise NotImplementedError
+            #extracols = [[f'{m}:1' for m in markers[extra]]
+            #             for extra in extras]
+            #def get_extra(item):
+            #    values = {}
+            #    for extra in extras:
+            #        cur = markers[extra]
+            #        for m in cur:
+            #            values[m] = m if getattr(item, m) == cur[m] else ''
+            #    return values
+        columns = [
+            f'filename:{maxfilename}',
+            f'name:{maxname}',
+            *extracols,
+        ]
+    header, div, fmt = build_table(columns)
+
+    total = 0
+    for group in groups:
+        if group not in collated:
+            continue
+        yield ''
+        yield f' === {group} ==='
+        yield ''
+        yield header
+        yield div
+        for item in collated[group]:
+            yield fmt.format(
+                filename=item.relfile,
+                name=item.name,
+                **get_extra(item),
+            )
+        yield div
+        subtotal = len(collated[group])
+        yield f'  sub-total: {subtotal}'
+        total += subtotal
+    yield ''
+    yield f'total: {total}'
+
+
+def render_full(items, *, groupby=None, verbose=False):
+    if groupby:
+        collated, groupby, _, _, _ = _collate(items, groupby)
+        for group, grouped in collated.items():
+            yield '#' * 25
+            yield f'# {group} ({len(grouped)})'
+            yield '#' * 25
+            yield ''
+            if not grouped:
+                continue
+            for item in grouped:
+                yield from _render_item_full(item, groupby, verbose)
+                yield ''
+    else:
+        for item in items:
+            yield from _render_item_full(item, None, verbose)
+            yield ''
+
+
+def _render_item_full(item, groupby, verbose):
+    yield item.name
+    yield f'  {"filename:":10} {item.relfile}'
+    for extra in ('kind', 'level'):
+        #if groupby != extra:
+            yield f'  {extra+":":10} {getattr(item, extra)}'
+    if verbose:
+        print('  ---------------------------------------')
+        for lno, line in enumerate(item.text, item.lno):
+            print(f'  | {lno:3} {line}')
+        print('  ---------------------------------------')
+
+
+def render_summary(items, *, groupby='kind', verbose=False):
+    total = 0
+    summary = summarize(items, groupby=groupby)
+    # XXX Stablize the sorting to match KINDS/LEVELS.
+    for outer, counts in summary.items():
+        subtotal = sum(c for _, c in counts.items())
+        yield f'{outer + ":":20} ({subtotal})'
+        for inner, count in counts.items():
+            yield f'   {inner + ":":9} {count}'
+        total += subtotal
+    yield f'{"total:":20} ({total})'
+
+
+_FORMATS = {
+    'brief': render_table,
+    'full': render_full,
+    'summary': render_summary,
+}

Tools/c-analyzer/cpython/_files.py

@@ -0,0 +1,69 @@
+import os.path
+
+from c_common.fsutil import expand_filenames, iter_files_by_suffix
+from . import REPO_ROOT, INCLUDE_DIRS, SOURCE_DIRS
+
+
+GLOBS = [
+    'Include/*.h',
+    'Include/internal/*.h',
+    'Modules/**/*.h',
+    'Modules/**/*.c',
+    'Objects/**/*.h',
+    'Objects/**/*.c',
+    'Python/**/*.h',
+    'Parser/**/*.c',
+    'Python/**/*.h',
+    'Parser/**/*.c',
+]
+LEVEL_GLOBS = {
+    'stable': 'Include/*.h',
+    'cpython': 'Include/cpython/*.h',
+    'internal': 'Include/internal/*.h',
+}
+
+
+def resolve_filename(filename):
+    orig = filename
+    filename = os.path.normcase(os.path.normpath(filename))
+    if os.path.isabs(filename):
+        if os.path.relpath(filename, REPO_ROOT).startswith('.'):
+            raise Exception(f'{orig!r} is outside the repo ({REPO_ROOT})')
+        return filename
+    else:
+        return os.path.join(REPO_ROOT, filename)
+
+
+def iter_filenames(*, search=False):
+    if search:
+        yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',))
+        yield from iter_files_by_suffix(SOURCE_DIRS, ('.c',))
+    else:
+        globs = (os.path.join(REPO_ROOT, file) for file in GLOBS)
+        yield from expand_filenames(globs)
+
+
+def iter_header_files(filenames=None, *, levels=None):
+    if not filenames:
+        if levels:
+            levels = set(levels)
+            if 'private' in levels:
+                levels.add('stable')
+                levels.add('cpython')
+            for level, glob in LEVEL_GLOBS.items():
+                if level in levels:
+                    yield from expand_filenames([glob])
+        else:
+            yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',))
+        return
+
+    for filename in filenames:
+        orig = filename
+        filename = resolve_filename(filename)
+        if filename.endswith(os.path.sep):
+            yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',))
+        elif filename.endswith('.h'):
+            yield filename
+        else:
+            # XXX Log it and continue instead?
+            raise ValueError(f'expected .h file, got {orig!r}')

Tools/c-analyzer/cpython/_parser.py

@@ -1,7 +1,6 @@
 import os.path
 import re
 
-from c_common.fsutil import expand_filenames, iter_files_by_suffix
 from c_parser.preprocessor import (
     get_preprocessor as _get_preprocessor,
 )
@@ -9,7 +8,7 @@ from c_parser import (
     parse_file as _parse_file,
     parse_files as _parse_files,
 )
-from . import REPO_ROOT, INCLUDE_DIRS, SOURCE_DIRS
+from . import REPO_ROOT
 
 
 GLOB_ALL = '**/*'
@@ -43,19 +42,6 @@ def clean_lines(text):
 @end=sh@
 '''
 
-GLOBS = [
-    'Include/*.h',
-    'Include/internal/*.h',
-    'Modules/**/*.h',
-    'Modules/**/*.c',
-    'Objects/**/*.h',
-    'Objects/**/*.c',
-    'Python/**/*.h',
-    'Parser/**/*.c',
-    'Python/**/*.h',
-    'Parser/**/*.c',
-]
-
 EXCLUDED = clean_lines('''
 # @begin=conf@
 
@@ -280,26 +266,6 @@ SAME = [
 ]
 
 
-def resolve_filename(filename):
-    orig = filename
-    filename = os.path.normcase(os.path.normpath(filename))
-    if os.path.isabs(filename):
-        if os.path.relpath(filename, REPO_ROOT).startswith('.'):
-            raise Exception(f'{orig!r} is outside the repo ({REPO_ROOT})')
-        return filename
-    else:
-        return os.path.join(REPO_ROOT, filename)
-
-
-def iter_filenames(*, search=False):
-    if search:
-        yield from iter_files_by_suffix(INCLUDE_DIRS, ('.h',))
-        yield from iter_files_by_suffix(SOURCE_DIRS, ('.c',))
-    else:
-        globs = (os.path.join(REPO_ROOT, file) for file in GLOBS)
-        yield from expand_filenames(globs)
-
-
 def get_preprocessor(*,
                      file_macros=None,
                      file_incldirs=None,

Tools/gdb/libpython.py

@@ -468,7 +468,7 @@ class InstanceProxy(object):
     def __repr__(self):
         if isinstance(self.attrdict, dict):
             kwargs = ', '.join(["%s=%r" % (arg, val)
-                                for arg, val in self.attrdict.iteritems()])
+                                for arg, val in self.attrdict.items()])
             return '<%s(%s) at remote 0x%x>' % (self.cl_name,
                                                 kwargs, self.address)
         else: