mirror of https://github.com/python/cpython
806 lines
19 KiB
Python
806 lines
19 KiB
Python
# Regular expression patterns for C syntax.
|
|
#
|
|
# None of these patterns has any capturing. However, a number of them
|
|
# have capturing markers compatible with utils.set_capture_groups().
|
|
|
|
import textwrap
|
|
|
|
|
|
def _ind(text, level=1, edges='both'):
|
|
indent = ' ' * level
|
|
text = textwrap.indent(text, indent)
|
|
if edges == 'pre' or edges == 'both':
|
|
text = '\n' + indent + text.lstrip()
|
|
if edges == 'post' or edges == 'both':
|
|
text = text.rstrip() + '\n' + ' ' * (level - 1)
|
|
return text
|
|
|
|
|
|
#######################################
|
|
# general
|
|
|
|
HEX = r'(?: [0-9a-zA-Z] )'
|
|
|
|
STRING_LITERAL = textwrap.dedent(rf'''
|
|
(?:
|
|
# character literal
|
|
(?:
|
|
['] [^'] [']
|
|
|
|
|
['] \\ . [']
|
|
|
|
|
['] \\x{HEX}{HEX} [']
|
|
|
|
|
['] \\0\d\d [']
|
|
|
|
|
(?:
|
|
['] \\o[01]\d\d [']
|
|
|
|
|
['] \\o2[0-4]\d [']
|
|
|
|
|
['] \\o25[0-5] [']
|
|
)
|
|
)
|
|
|
|
|
# string literal
|
|
(?:
|
|
["] (?: [^"\\]* \\ . )* [^"\\]* ["]
|
|
)
|
|
# end string literal
|
|
)
|
|
''')
|
|
|
|
_KEYWORD = textwrap.dedent(r'''
|
|
(?:
|
|
\b
|
|
(?:
|
|
auto |
|
|
extern |
|
|
register |
|
|
static |
|
|
_Thread_local |
|
|
typedef |
|
|
|
|
const |
|
|
volatile |
|
|
|
|
signed |
|
|
unsigned |
|
|
char |
|
|
short |
|
|
int |
|
|
long |
|
|
float |
|
|
double |
|
|
void |
|
|
|
|
struct |
|
|
union |
|
|
enum |
|
|
|
|
goto |
|
|
return |
|
|
sizeof |
|
|
break |
|
|
continue |
|
|
if |
|
|
else |
|
|
for |
|
|
do |
|
|
while |
|
|
switch |
|
|
case |
|
|
default |
|
|
entry
|
|
)
|
|
\b
|
|
)
|
|
''')
|
|
KEYWORD = rf'''
|
|
# keyword
|
|
{_KEYWORD}
|
|
# end keyword
|
|
'''
|
|
_KEYWORD = ''.join(_KEYWORD.split())
|
|
|
|
IDENTIFIER = r'(?: [a-zA-Z_][a-zA-Z0-9_]* )'
|
|
# We use a negative lookahead to filter out keywords.
|
|
STRICT_IDENTIFIER = rf'(?: (?! {_KEYWORD} ) \b {IDENTIFIER} \b )'
|
|
ANON_IDENTIFIER = rf'(?: (?! {_KEYWORD} ) \b {IDENTIFIER} (?: - \d+ )? \b )'
|
|
|
|
|
|
#######################################
|
|
# types
|
|
|
|
SIMPLE_TYPE = textwrap.dedent(rf'''
|
|
# simple type
|
|
(?:
|
|
\b
|
|
(?:
|
|
void
|
|
|
|
|
(?: signed | unsigned ) # implies int
|
|
|
|
|
(?:
|
|
(?: (?: signed | unsigned ) \s+ )?
|
|
(?: (?: long | short ) \s+ )?
|
|
(?: char | short | int | long | float | double )
|
|
)
|
|
)
|
|
\b
|
|
)
|
|
# end simple type
|
|
''')
|
|
|
|
COMPOUND_TYPE_KIND = r'(?: \b (?: struct | union | enum ) \b )'
|
|
|
|
|
|
#######################################
|
|
# variable declarations
|
|
|
|
_STORAGE = 'auto register static extern _Thread_local'.split()
|
|
STORAGE_CLASS = rf'(?: \b (?: {" | ".join(_STORAGE)} ) \b )'
|
|
TYPE_QUALIFIER = r'(?: \b (?: const | volatile ) \b )'
|
|
PTR_QUALIFIER = rf'(?: [*] (?: \s* {TYPE_QUALIFIER} )? )'
|
|
|
|
TYPE_SPEC = textwrap.dedent(rf'''
|
|
# type spec
|
|
(?:
|
|
{_ind(SIMPLE_TYPE, 2)}
|
|
|
|
|
(?:
|
|
[_]*typeof[_]*
|
|
\s* [(]
|
|
(?: \s* [*&] )*
|
|
\s* {STRICT_IDENTIFIER}
|
|
\s* [)]
|
|
)
|
|
|
|
|
# reference to a compound type
|
|
(?:
|
|
{COMPOUND_TYPE_KIND}
|
|
(?: \s* {ANON_IDENTIFIER} )?
|
|
)
|
|
|
|
|
# reference to a typedef
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
# end type spec
|
|
''')
|
|
|
|
DECLARATOR = textwrap.dedent(rf'''
|
|
# declarator (possibly abstract)
|
|
(?:
|
|
(?: {PTR_QUALIFIER} \s* )*
|
|
(?:
|
|
(?:
|
|
(?: # <IDENTIFIER>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
# Inside the brackets is actually a "constant expression".
|
|
(?: \s* \[ (?: \s* [^\]]+ \s* )? [\]] )* # arrays
|
|
)
|
|
|
|
|
(?:
|
|
[(] \s*
|
|
(?: # <WRAPPED_IDENTIFIER>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
# Inside the brackets is actually a "constant expression".
|
|
(?: \s* \[ (?: \s* [^\]]+ \s* )? [\]] )* # arrays
|
|
\s* [)]
|
|
)
|
|
|
|
|
# func ptr
|
|
(?:
|
|
[(] (?: \s* {PTR_QUALIFIER} )? \s*
|
|
(?: # <FUNC_IDENTIFIER>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
# Inside the brackets is actually a "constant expression".
|
|
(?: \s* \[ (?: \s* [^\]]+ \s* )? [\]] )* # arrays
|
|
\s* [)]
|
|
# We allow for a single level of paren nesting in parameters.
|
|
\s* [(] (?: [^()]* [(] [^)]* [)] )* [^)]* [)]
|
|
)
|
|
)
|
|
)
|
|
# end declarator
|
|
''')
|
|
|
|
VAR_DECL = textwrap.dedent(rf'''
|
|
# var decl (and typedef and func return type)
|
|
(?:
|
|
(?:
|
|
(?: # <STORAGE>
|
|
{STORAGE_CLASS}
|
|
)
|
|
\s*
|
|
)?
|
|
(?:
|
|
(?: # <TYPE_QUAL>
|
|
{TYPE_QUALIFIER}
|
|
)
|
|
\s*
|
|
)?
|
|
(?:
|
|
(?: # <TYPE_SPEC>
|
|
{_ind(TYPE_SPEC, 4)}
|
|
)
|
|
)
|
|
\s*
|
|
(?:
|
|
(?: # <DECLARATOR>
|
|
{_ind(DECLARATOR, 4)}
|
|
)
|
|
)
|
|
)
|
|
# end var decl
|
|
''')
|
|
|
|
INITIALIZER = textwrap.dedent(rf'''
|
|
# initializer
|
|
(?:
|
|
(?:
|
|
[(]
|
|
# no nested parens (e.g. func ptr)
|
|
[^)]*
|
|
[)]
|
|
\s*
|
|
)?
|
|
(?:
|
|
# a string literal
|
|
(?:
|
|
(?: {_ind(STRING_LITERAL, 4)} \s* )*
|
|
{_ind(STRING_LITERAL, 4)}
|
|
)
|
|
|
|
|
|
|
# a simple initializer
|
|
(?:
|
|
(?:
|
|
[^'",;{{]*
|
|
{_ind(STRING_LITERAL, 4)}
|
|
)*
|
|
[^'",;{{]*
|
|
)
|
|
|
|
|
|
|
# a struct/array literal
|
|
(?:
|
|
# We only expect compound initializers with
|
|
# single-variable declarations.
|
|
{{
|
|
(?:
|
|
[^'";]*?
|
|
{_ind(STRING_LITERAL, 5)}
|
|
)*
|
|
[^'";]*?
|
|
}}
|
|
(?= \s* ; ) # Note this lookahead.
|
|
)
|
|
)
|
|
)
|
|
# end initializer
|
|
''')
|
|
|
|
|
|
#######################################
|
|
# compound type declarations
|
|
|
|
STRUCT_MEMBER_DECL = textwrap.dedent(rf'''
|
|
(?:
|
|
# inline compound type decl
|
|
(?:
|
|
(?: # <COMPOUND_TYPE_KIND>
|
|
{COMPOUND_TYPE_KIND}
|
|
)
|
|
(?:
|
|
\s+
|
|
(?: # <COMPOUND_TYPE_NAME>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
)?
|
|
\s* {{
|
|
)
|
|
|
|
|
(?:
|
|
# typed member
|
|
(?:
|
|
# Technically it doesn't have to have a type...
|
|
(?: # <SPECIFIER_QUALIFIER>
|
|
(?: {TYPE_QUALIFIER} \s* )?
|
|
{_ind(TYPE_SPEC, 5)}
|
|
)
|
|
(?:
|
|
# If it doesn't have a declarator then it will have
|
|
# a size and vice versa.
|
|
\s*
|
|
(?: # <DECLARATOR>
|
|
{_ind(DECLARATOR, 6)}
|
|
)
|
|
)?
|
|
)
|
|
|
|
# sized member
|
|
(?:
|
|
\s* [:] \s*
|
|
(?: # <SIZE>
|
|
# This is actually a "constant expression".
|
|
\d+
|
|
|
|
|
[^'",}}]+
|
|
)
|
|
)?
|
|
\s*
|
|
(?: # <ENDING>
|
|
[,;]
|
|
)
|
|
)
|
|
|
|
|
(?:
|
|
\s*
|
|
(?: # <CLOSE>
|
|
}}
|
|
)
|
|
)
|
|
)
|
|
''')
|
|
|
|
ENUM_MEMBER_DECL = textwrap.dedent(rf'''
|
|
(?:
|
|
(?:
|
|
\s*
|
|
(?: # <CLOSE>
|
|
}}
|
|
)
|
|
)
|
|
|
|
|
(?:
|
|
\s*
|
|
(?: # <NAME>
|
|
{IDENTIFIER}
|
|
)
|
|
(?:
|
|
\s* = \s*
|
|
(?: # <INIT>
|
|
# This is actually a "constant expression".
|
|
{_ind(STRING_LITERAL, 4)}
|
|
|
|
|
[^'",}}]+
|
|
)
|
|
)?
|
|
\s*
|
|
(?: # <ENDING>
|
|
, | }}
|
|
)
|
|
)
|
|
)
|
|
''')
|
|
|
|
|
|
#######################################
|
|
# statements
|
|
|
|
SIMPLE_STMT_BODY = textwrap.dedent(rf'''
|
|
# simple statement body
|
|
(?:
|
|
(?:
|
|
[^'"{{}};]*
|
|
{_ind(STRING_LITERAL, 3)}
|
|
)*
|
|
[^'"{{}};]*
|
|
#(?= [;{{] ) # Note this lookahead.
|
|
)
|
|
# end simple statement body
|
|
''')
|
|
SIMPLE_STMT = textwrap.dedent(rf'''
|
|
# simple statement
|
|
(?:
|
|
(?: # <SIMPLE_STMT>
|
|
# stmt-inline "initializer"
|
|
(?:
|
|
return \b
|
|
(?:
|
|
\s*
|
|
{_ind(INITIALIZER, 5)}
|
|
)?
|
|
)
|
|
|
|
|
# variable assignment
|
|
(?:
|
|
(?: [*] \s* )?
|
|
(?:
|
|
{STRICT_IDENTIFIER} \s*
|
|
(?: . | -> ) \s*
|
|
)*
|
|
{STRICT_IDENTIFIER}
|
|
(?: \s* \[ \s* \d+ \s* \] )?
|
|
\s* = \s*
|
|
{_ind(INITIALIZER, 4)}
|
|
)
|
|
|
|
|
# catchall return statement
|
|
(?:
|
|
return \b
|
|
(?:
|
|
(?:
|
|
[^'";]*
|
|
{_ind(STRING_LITERAL, 6)}
|
|
)*
|
|
\s* [^'";]*
|
|
)?
|
|
)
|
|
|
|
|
# simple statement
|
|
(?:
|
|
{_ind(SIMPLE_STMT_BODY, 4)}
|
|
)
|
|
)
|
|
\s*
|
|
(?: # <SIMPLE_ENDING>
|
|
;
|
|
)
|
|
)
|
|
# end simple statement
|
|
''')
|
|
COMPOUND_STMT = textwrap.dedent(rf'''
|
|
# compound statement
|
|
(?:
|
|
\b
|
|
(?:
|
|
(?:
|
|
(?: # <COMPOUND_BARE>
|
|
else | do
|
|
)
|
|
\b
|
|
)
|
|
|
|
|
(?:
|
|
(?: # <COMPOUND_LABELED>
|
|
(?:
|
|
case \b
|
|
(?:
|
|
[^'":]*
|
|
{_ind(STRING_LITERAL, 7)}
|
|
)*
|
|
\s* [^'":]*
|
|
)
|
|
|
|
|
default
|
|
|
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
\s* [:]
|
|
)
|
|
|
|
|
(?:
|
|
(?: # <COMPOUND_PAREN>
|
|
for | while | if | switch
|
|
)
|
|
\s* (?= [(] ) # Note this lookahead.
|
|
)
|
|
)
|
|
\s*
|
|
)
|
|
# end compound statement
|
|
''')
|
|
|
|
|
|
#######################################
|
|
# function bodies
|
|
|
|
LOCAL = textwrap.dedent(rf'''
|
|
(?:
|
|
# an empty statement
|
|
(?: # <EMPTY>
|
|
;
|
|
)
|
|
|
|
|
# inline type decl
|
|
(?:
|
|
(?:
|
|
(?: # <INLINE_LEADING>
|
|
[^;{{}}]+?
|
|
)
|
|
\s*
|
|
)?
|
|
(?: # <INLINE_PRE>
|
|
(?: {STORAGE_CLASS} \s* )?
|
|
(?: {TYPE_QUALIFIER} \s* )?
|
|
)? # </INLINE_PRE>
|
|
(?: # <INLINE_KIND>
|
|
{COMPOUND_TYPE_KIND}
|
|
)
|
|
(?:
|
|
\s+
|
|
(?: # <INLINE_NAME>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
)?
|
|
\s* {{
|
|
)
|
|
|
|
|
# var decl
|
|
(?:
|
|
(?: # <STORAGE>
|
|
{STORAGE_CLASS}
|
|
)? # </STORAGE>
|
|
(?:
|
|
\s*
|
|
(?: # <VAR_DECL>
|
|
{_ind(VAR_DECL, 5)}
|
|
)
|
|
)
|
|
(?:
|
|
(?:
|
|
# initializer
|
|
# We expect only basic initializers.
|
|
\s* = \s*
|
|
(?: # <VAR_INIT>
|
|
{_ind(INITIALIZER, 6)}
|
|
)
|
|
)?
|
|
(?:
|
|
\s*
|
|
(?: # <VAR_ENDING>
|
|
[,;]
|
|
)
|
|
)
|
|
)
|
|
)
|
|
|
|
|
{_ind(COMPOUND_STMT, 2)}
|
|
|
|
|
# start-of-block
|
|
(?:
|
|
(?: # <BLOCK_LEADING>
|
|
(?:
|
|
[^'"{{}};]*
|
|
{_ind(STRING_LITERAL, 5)}
|
|
)*
|
|
[^'"{{}};]*
|
|
# Presumably we will not see "== {{".
|
|
[^\s='"{{}});]
|
|
\s*
|
|
)? # </BLOCK_LEADING>
|
|
(?: # <BLOCK_OPEN>
|
|
{{
|
|
)
|
|
)
|
|
|
|
|
{_ind(SIMPLE_STMT, 2)}
|
|
|
|
|
# end-of-block
|
|
(?: # <BLOCK_CLOSE>
|
|
}}
|
|
)
|
|
)
|
|
''')
|
|
|
|
LOCAL_STATICS = textwrap.dedent(rf'''
|
|
(?:
|
|
# inline type decl
|
|
(?:
|
|
(?:
|
|
(?: # <INLINE_LEADING>
|
|
[^;{{}}]+?
|
|
)
|
|
\s*
|
|
)?
|
|
(?: # <INLINE_PRE>
|
|
(?: {STORAGE_CLASS} \s* )?
|
|
(?: {TYPE_QUALIFIER} \s* )?
|
|
)?
|
|
(?: # <INLINE_KIND>
|
|
{COMPOUND_TYPE_KIND}
|
|
)
|
|
(?:
|
|
\s+
|
|
(?: # <INLINE_NAME>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
)?
|
|
\s* {{
|
|
)
|
|
|
|
|
# var decl
|
|
(?:
|
|
# We only look for static variables.
|
|
(?: # <STATIC_DECL>
|
|
static \b
|
|
(?: \s* {TYPE_QUALIFIER} )?
|
|
\s* {_ind(TYPE_SPEC, 4)}
|
|
\s* {_ind(DECLARATOR, 4)}
|
|
)
|
|
\s*
|
|
(?:
|
|
(?: # <STATIC_INIT>
|
|
= \s*
|
|
{_ind(INITIALIZER, 4)}
|
|
\s*
|
|
[,;{{]
|
|
)
|
|
|
|
|
(?: # <STATIC_ENDING>
|
|
[,;]
|
|
)
|
|
)
|
|
)
|
|
|
|
|
# everything else
|
|
(?:
|
|
(?: # <DELIM_LEADING>
|
|
(?:
|
|
[^'"{{}};]*
|
|
{_ind(STRING_LITERAL, 4)}
|
|
)*
|
|
\s* [^'"{{}};]*
|
|
)
|
|
(?:
|
|
(?: # <BLOCK_OPEN>
|
|
{{
|
|
)
|
|
|
|
|
(?: # <BLOCK_CLOSE>
|
|
}}
|
|
)
|
|
|
|
|
(?: # <STMT_END>
|
|
;
|
|
)
|
|
)
|
|
)
|
|
)
|
|
''')
|
|
|
|
|
|
#######################################
|
|
# global declarations
|
|
|
|
GLOBAL = textwrap.dedent(rf'''
|
|
(?:
|
|
# an empty statement
|
|
(?: # <EMPTY>
|
|
;
|
|
)
|
|
|
|
|
|
|
# compound type decl (maybe inline)
|
|
(?:
|
|
(?:
|
|
(?: # <COMPOUND_LEADING>
|
|
[^;{{}}]+?
|
|
)
|
|
\s*
|
|
)?
|
|
(?: # <COMPOUND_KIND>
|
|
{COMPOUND_TYPE_KIND}
|
|
)
|
|
(?:
|
|
\s+
|
|
(?: # <COMPOUND_NAME>
|
|
{STRICT_IDENTIFIER}
|
|
)
|
|
)?
|
|
\s* {{
|
|
)
|
|
|
|
|
# bogus inline decl artifact
|
|
# This simplifies resolving the relative syntactic ambiguity of
|
|
# inline structs.
|
|
(?:
|
|
(?: # <FORWARD_KIND>
|
|
{COMPOUND_TYPE_KIND}
|
|
)
|
|
\s*
|
|
(?: # <FORWARD_NAME>
|
|
{ANON_IDENTIFIER}
|
|
)
|
|
(?: # <MAYBE_INLINE_ACTUAL>
|
|
[^=,;({{[*\]]*
|
|
[=,;({{]
|
|
)
|
|
)
|
|
|
|
|
|
|
# typedef
|
|
(?:
|
|
\b typedef \b \s*
|
|
(?: # <TYPEDEF_DECL>
|
|
{_ind(VAR_DECL, 4)}
|
|
)
|
|
(?:
|
|
# We expect no inline type definitions in the parameters.
|
|
\s* [(] \s*
|
|
(?: # <TYPEDEF_FUNC_PARAMS>
|
|
[^{{;]*
|
|
)
|
|
\s* [)]
|
|
)?
|
|
\s* ;
|
|
)
|
|
|
|
|
|
|
# func decl/definition & var decls
|
|
# XXX dedicated pattern for funcs (more restricted)?
|
|
(?:
|
|
(?:
|
|
(?: # <VAR_STORAGE>
|
|
{STORAGE_CLASS}
|
|
)
|
|
\s*
|
|
)?
|
|
(?:
|
|
(?: # <FUNC_INLINE>
|
|
\b inline \b
|
|
)
|
|
\s*
|
|
)?
|
|
(?: # <VAR_DECL>
|
|
{_ind(VAR_DECL, 4)}
|
|
)
|
|
(?:
|
|
# func decl / definition
|
|
(?:
|
|
(?:
|
|
# We expect no inline type definitions in the parameters.
|
|
\s* [(] \s*
|
|
(?: # <FUNC_PARAMS>
|
|
[^{{;]*
|
|
)
|
|
\s* [)] \s*
|
|
(?: # <FUNC_DELIM>
|
|
[{{;]
|
|
)
|
|
)
|
|
|
|
|
(?:
|
|
# This is some old-school syntax!
|
|
\s* [(] \s*
|
|
# We throw away the bare names:
|
|
{STRICT_IDENTIFIER}
|
|
(?: \s* , \s* {STRICT_IDENTIFIER} )*
|
|
\s* [)] \s*
|
|
|
|
# We keep the trailing param declarations:
|
|
(?: # <FUNC_LEGACY_PARAMS>
|
|
# There's at least one!
|
|
(?: {TYPE_QUALIFIER} \s* )?
|
|
{_ind(TYPE_SPEC, 7)}
|
|
\s*
|
|
{_ind(DECLARATOR, 7)}
|
|
\s* ;
|
|
(?:
|
|
\s*
|
|
(?: {TYPE_QUALIFIER} \s* )?
|
|
{_ind(TYPE_SPEC, 8)}
|
|
\s*
|
|
{_ind(DECLARATOR, 8)}
|
|
\s* ;
|
|
)*
|
|
)
|
|
\s* {{
|
|
)
|
|
)
|
|
|
|
|
# var / typedef
|
|
(?:
|
|
(?:
|
|
# initializer
|
|
# We expect only basic initializers.
|
|
\s* = \s*
|
|
(?: # <VAR_INIT>
|
|
{_ind(INITIALIZER, 6)}
|
|
)
|
|
)?
|
|
\s*
|
|
(?: # <VAR_ENDING>
|
|
[,;]
|
|
)
|
|
)
|
|
)
|
|
)
|
|
)
|
|
''')
|