from collections import namedtuple import shlex import os import re from ..common import util, info CONTINUATION = '\\' + os.linesep IDENTIFIER = r'(?:\w*[a-zA-Z]\w*)' IDENTIFIER_RE = re.compile('^' + IDENTIFIER + '$') def _coerce_str(value): if not value: return '' return str(value).strip() ############################# # directives DIRECTIVE_START = r''' (?: ^ \s* [#] \s* )''' DIRECTIVE_TEXT = r''' (?: (?: \s+ ( .*\S ) )? \s* $ )''' DIRECTIVE = rf''' (?: {DIRECTIVE_START} ( include | error | warning | pragma | define | undef | if | ifdef | ifndef | elseif | else | endif | __FILE__ | __LINE__ | __DATE __ | __TIME__ | __TIMESTAMP__ ) {DIRECTIVE_TEXT} )''' # (?: # [^\\\n] | # \\ [^\n] | # \\ \n # )+ # ) \n # )''' DIRECTIVE_RE = re.compile(DIRECTIVE, re.VERBOSE) DEFINE = rf''' (?: {DIRECTIVE_START} define \s+ (?: ( \w*[a-zA-Z]\w* ) (?: \s* [(] ([^)]*) [)] )? ) {DIRECTIVE_TEXT} )''' DEFINE_RE = re.compile(DEFINE, re.VERBOSE) def parse_directive(line): """Return the appropriate directive for the given line.""" line = line.strip() if line.startswith('#'): line = line[1:].lstrip() line = '#' + line directive = line #directive = '#' + line while ' ' in directive: directive = directive.replace(' ', ' ') return _parse_directive(directive) def _parse_directive(line): m = DEFINE_RE.match(line) if m: name, args, text = m.groups() if args: args = [a.strip() for a in args.split(',')] return Macro(name, args, text) else: return Constant(name, text) m = DIRECTIVE_RE.match(line) if not m: raise ValueError(f'unsupported directive {line!r}') kind, text = m.groups() if not text: if kind not in ('else', 'endif'): raise ValueError(f'missing text in directive {line!r}') elif kind in ('else', 'endif', 'define'): raise ValueError(f'unexpected text in directive {line!r}') if kind == 'include': directive = Include(text) elif kind in IfDirective.KINDS: directive = IfDirective(kind, text) else: directive = OtherDirective(kind, text) directive.validate() return directive class PreprocessorDirective(util._NTBase): """The base class for directives.""" __slots__ = () KINDS = frozenset([ 'include', 'pragma', 'error', 'warning', 'define', 'undef', 'if', 'ifdef', 'ifndef', 'elseif', 'else', 'endif', '__FILE__', '__DATE__', '__LINE__', '__TIME__', '__TIMESTAMP__', ]) @property def text(self): return ' '.join(v for v in self[1:] if v and v.strip()) or None def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" super().validate() if not self.kind: raise TypeError('missing kind') elif self.kind not in self.KINDS: raise ValueError # text can be anything, including None. class Constant(PreprocessorDirective, namedtuple('Constant', 'kind name value')): """A single "constant" directive ("define").""" __slots__ = () def __new__(cls, name, value=None): self = super().__new__( cls, 'define', name=_coerce_str(name) or None, value=_coerce_str(value) or None, ) return self def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" super().validate() if not self.name: raise TypeError('missing name') elif not IDENTIFIER_RE.match(self.name): raise ValueError(f'name must be identifier, got {self.name!r}') # value can be anything, including None class Macro(PreprocessorDirective, namedtuple('Macro', 'kind name args body')): """A single "macro" directive ("define").""" __slots__ = () def __new__(cls, name, args, body=None): # "args" must be a string or an iterable of strings (or "empty"). if isinstance(args, str): args = [v.strip() for v in args.split(',')] if args: args = tuple(_coerce_str(a) or None for a in args) self = super().__new__( cls, kind='define', name=_coerce_str(name) or None, args=args if args else (), body=_coerce_str(body) or None, ) return self @property def text(self): if self.body: return f'{self.name}({", ".join(self.args)}) {self.body}' else: return f'{self.name}({", ".join(self.args)})' def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" super().validate() if not self.name: raise TypeError('missing name') elif not IDENTIFIER_RE.match(self.name): raise ValueError(f'name must be identifier, got {self.name!r}') for arg in self.args: if not arg: raise ValueError(f'missing arg in {self.args}') elif not IDENTIFIER_RE.match(arg): raise ValueError(f'arg must be identifier, got {arg!r}') # body can be anything, including None class IfDirective(PreprocessorDirective, namedtuple('IfDirective', 'kind condition')): """A single conditional directive (e.g. "if", "ifdef"). This only includes directives that actually provide conditions. The related directives "else" and "endif" are covered by OtherDirective instead. """ __slots__ = () KINDS = frozenset([ 'if', 'ifdef', 'ifndef', 'elseif', ]) @classmethod def _condition_from_raw(cls, raw, kind): #return Condition.from_raw(raw, _kind=kind) condition = _coerce_str(raw) if not condition: return None if kind == 'ifdef': condition = f'defined({condition})' elif kind == 'ifndef': condition = f'! defined({condition})' return condition def __new__(cls, kind, condition): kind = _coerce_str(kind) self = super().__new__( cls, kind=kind or None, condition=cls._condition_from_raw(condition, kind), ) return self @property def text(self): if self.kind == 'ifdef': return self.condition[8:-1] # strip "defined(" elif self.kind == 'ifndef': return self.condition[10:-1] # strip "! defined(" else: return self.condition #return str(self.condition) def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" super().validate() if not self.condition: raise TypeError('missing condition') #else: # for cond in self.condition: # if not cond: # raise ValueError(f'missing condition in {self.condition}') # cond.validate() # if self.kind in ('ifdef', 'ifndef'): # if len(self.condition) != 1: # raise ValueError('too many condition') # if self.kind == 'ifdef': # if not self.condition[0].startswith('defined '): # raise ValueError('bad condition') # else: # if not self.condition[0].startswith('! defined '): # raise ValueError('bad condition') class Include(PreprocessorDirective, namedtuple('Include', 'kind file')): """A single "include" directive. Supported "file" values are either follow the bracket style () or double quotes ("spam.h"). """ __slots__ = () def __new__(cls, file): self = super().__new__( cls, kind='include', file=_coerce_str(file) or None, ) return self def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" super().validate() if not self.file: raise TypeError('missing file') class OtherDirective(PreprocessorDirective, namedtuple('OtherDirective', 'kind text')): """A single directive not covered by another class. This includes the "else", "endif", and "undef" directives, which are otherwise inherently related to the directives covered by the Constant, Macro, and IfCondition classes. Note that all directives must have a text value, except for "else" and "endif" (which must have no text). """ __slots__ = () KINDS = PreprocessorDirective.KINDS - {'include', 'define'} - IfDirective.KINDS def __new__(cls, kind, text): self = super().__new__( cls, kind=_coerce_str(kind) or None, text=_coerce_str(text) or None, ) return self def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" super().validate() if self.text: if self.kind in ('else', 'endif'): raise ValueError('unexpected text in directive') elif self.kind not in ('else', 'endif'): raise TypeError('missing text') ############################# # iterating lines def _recompute_conditions(directive, ifstack): if directive.kind in ('if', 'ifdef', 'ifndef'): ifstack.append( ([], directive.condition)) elif directive.kind == 'elseif': if ifstack: negated, active = ifstack.pop() if active: negated.append(active) else: negated = [] ifstack.append( (negated, directive.condition)) elif directive.kind == 'else': if ifstack: negated, active = ifstack.pop() if active: negated.append(active) ifstack.append( (negated, None)) elif directive.kind == 'endif': if ifstack: ifstack.pop() conditions = [] for negated, active in ifstack: for condition in negated: conditions.append(f'! ({condition})') if active: conditions.append(active) return tuple(conditions) def _iter_clean_lines(lines): lines = iter(enumerate(lines, 1)) for lno, line in lines: # Handle line continuations. while line.endswith(CONTINUATION): try: lno, _line = next(lines) except StopIteration: break line = line[:-len(CONTINUATION)] + ' ' + _line # Deal with comments. after = line line = '' while True: # Look for a comment. before, begin, remainder = after.partition('/*') if '//' in before: before, _, _ = before.partition('//') line += before + ' ' # per the C99 spec break line += before if not begin: break line += ' ' # per the C99 spec # Go until we find the end of the comment. _, end, after = remainder.partition('*/') while not end: try: lno, remainder = next(lines) except StopIteration: raise Exception('unterminated comment') _, end, after = remainder.partition('*/') yield lno, line def iter_lines(lines, *, _iter_clean_lines=_iter_clean_lines, _parse_directive=_parse_directive, _recompute_conditions=_recompute_conditions, ): """Yield (lno, line, directive, active conditions) for each given line. This is effectively a subset of the operations taking place in translation phases 2-4 from the C99 spec (ISO/IEC 9899:TC2); see section 5.1.1.2. Line continuations are removed and comments replaced with a single space. (In both cases "lno" will be the last line involved.) Otherwise each line is returned as-is. "lno" is the (1-indexed) line number for the line. "directive" will be a PreprocessorDirective or None, depending on whether or not there is a directive on the line. "active conditions" is the set of preprocessor conditions (e.g. "defined()") under which the current line of code will be included in compilation. That set is derived from every conditional directive block (e.g. "if defined()", "ifdef", "else") containing that line. That includes nested directives. Note that the current line does not affect the active conditions for iteself. It only impacts subsequent lines. That applies to directives that close blocks (e.g. "endif") just as much as conditional directvies. Also note that "else" and "elseif" directives update the active conditions (for later lines), rather than adding to them. """ ifstack = [] conditions = () for lno, line in _iter_clean_lines(lines): stripped = line.strip() if not stripped.startswith('#'): yield lno, line, None, conditions continue directive = '#' + stripped[1:].lstrip() while ' ' in directive: directive = directive.replace(' ', ' ') directive = _parse_directive(directive) yield lno, line, directive, conditions if directive.kind in ('else', 'endif'): conditions = _recompute_conditions(directive, ifstack) elif isinstance(directive, IfDirective): conditions = _recompute_conditions(directive, ifstack) ############################# # running (platform-specific?) def _gcc(filename, *, _get_argv=(lambda: _get_gcc_argv()), _run=util.run_cmd, ): argv = _get_argv() argv.extend([ '-E', filename, ]) output = _run(argv) return output def _get_gcc_argv(*, _open=open, _run=util.run_cmd, ): with _open('/tmp/print.mk', 'w') as tmpfile: tmpfile.write('print-%:\n') #tmpfile.write('\t@echo $* = $($*)\n') tmpfile.write('\t@echo $($*)\n') argv = ['/usr/bin/make', '-f', 'Makefile', '-f', '/tmp/print.mk', 'print-CC', 'print-PY_CORE_CFLAGS', ] output = _run(argv) gcc, cflags = output.strip().splitlines() argv = shlex.split(gcc.strip()) cflags = shlex.split(cflags.strip()) return argv + cflags def run(filename, *, _gcc=_gcc, ): """Return the text of the given file after running the preprocessor.""" return _gcc(filename)