from collections import namedtuple import enum import os.path import re from c_common import fsutil from c_common.clsutil import classonly import c_common.misc as _misc import c_common.strutil as _strutil import c_common.tables as _tables from .parser._regexes import SIMPLE_TYPE, _STORAGE FIXED_TYPE = _misc.Labeled('FIXED_TYPE') STORAGE = frozenset(_STORAGE) ############################# # kinds @enum.unique class KIND(enum.Enum): # XXX Use these in the raw parser code. TYPEDEF = 'typedef' STRUCT = 'struct' UNION = 'union' ENUM = 'enum' FUNCTION = 'function' VARIABLE = 'variable' STATEMENT = 'statement' @classonly def _from_raw(cls, raw): if raw is None: return None elif isinstance(raw, cls): return raw elif type(raw) is str: # We could use cls[raw] for the upper-case form, # but there's no need to go to the trouble. return cls(raw.lower()) else: raise NotImplementedError(raw) @classonly def by_priority(cls, group=None): if group is None: return cls._ALL_BY_PRIORITY.copy() elif group == 'type': return cls._TYPE_DECLS_BY_PRIORITY.copy() elif group == 'decl': return cls._ALL_DECLS_BY_PRIORITY.copy() elif isinstance(group, str): raise NotImplementedError(group) else: # XXX Treat group as a set of kinds & return in priority order? raise NotImplementedError(group) @classonly def is_type_decl(cls, kind): if kind in cls.TYPES: return True if not isinstance(kind, cls): raise TypeError(f'expected KIND, got {kind!r}') return False @classonly def is_decl(cls, kind): if kind in cls.DECLS: return True if not isinstance(kind, cls): raise TypeError(f'expected KIND, got {kind!r}') return False @classonly def get_group(cls, kind, *, groups=None): if not isinstance(kind, cls): raise TypeError(f'expected KIND, got {kind!r}') if groups is None: groups = ['type'] elif not groups: groups = () elif isinstance(groups, str): group = groups if group not in cls._GROUPS: raise ValueError(f'unsupported group {group!r}') groups = [group] else: unsupported = [g for g in groups if g not in cls._GROUPS] if unsupported: raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}') for group in groups: if kind in cls._GROUPS[group]: return group else: return kind.value @classonly def resolve_group(cls, group): if isinstance(group, cls): return {group} elif isinstance(group, str): try: return cls._GROUPS[group].copy() except KeyError: raise ValueError(f'unsupported group {group!r}') else: resolved = set() for gr in group: resolve.update(cls.resolve_group(gr)) return resolved #return {*cls.resolve_group(g) for g in group} KIND._TYPE_DECLS_BY_PRIORITY = [ # These are in preferred order. KIND.TYPEDEF, KIND.STRUCT, KIND.UNION, KIND.ENUM, ] KIND._ALL_DECLS_BY_PRIORITY = [ # These are in preferred order. *KIND._TYPE_DECLS_BY_PRIORITY, KIND.FUNCTION, KIND.VARIABLE, ] KIND._ALL_BY_PRIORITY = [ # These are in preferred order. *KIND._ALL_DECLS_BY_PRIORITY, KIND.STATEMENT, ] KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY) KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY) KIND._GROUPS = { 'type': KIND.TYPES, 'decl': KIND.DECLS, } KIND._GROUPS.update((k.value, {k}) for k in KIND) def get_kind_group(item): return KIND.get_group(item.kind) ############################# # low-level def _fix_filename(filename, relroot, *, formatted=True, **kwargs): if formatted: fix = fsutil.format_filename else: fix = fsutil.fix_filename return fix(filename, relroot=relroot, **kwargs) class FileInfo(namedtuple('FileInfo', 'filename lno')): @classmethod def from_raw(cls, raw): if isinstance(raw, cls): return raw elif isinstance(raw, tuple): return cls(*raw) elif not raw: return None elif isinstance(raw, str): return cls(raw, -1) else: raise TypeError(f'unsupported "raw": {raw:!r}') def __str__(self): return self.filename def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): filename = _fix_filename(self.filename, relroot, **kwargs) if filename == self.filename: return self return self._replace(filename=filename) class SourceLine(namedtuple('Line', 'file kind data conditions')): KINDS = ( #'directive', # data is ... 'source', # "data" is the line #'comment', # "data" is the text, including comment markers ) @property def filename(self): return self.file.filename @property def lno(self): return self.file.lno class DeclID(namedtuple('DeclID', 'filename funcname name')): """The globally-unique identifier for a declaration.""" @classmethod def from_row(cls, row, **markers): row = _tables.fix_row(row, **markers) return cls(*row) # We have to provde _make() becaose we implemented __new__(). @classmethod def _make(cls, iterable): try: return cls(*iterable) except Exception: super()._make(iterable) raise # re-raise def __new__(cls, filename, funcname, name): self = super().__new__( cls, filename=str(filename) if filename else None, funcname=str(funcname) if funcname else None, name=str(name) if name else None, ) self._compare = tuple(v or '' for v in self) return self def __hash__(self): return super().__hash__() def __eq__(self, other): try: other = tuple(v or '' for v in other) except TypeError: return NotImplemented return self._compare == other def __gt__(self, other): try: other = tuple(v or '' for v in other) except TypeError: return NotImplemented return self._compare > other def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): filename = _fix_filename(self.filename, relroot, **kwargs) if filename == self.filename: return self return self._replace(filename=filename) class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')): @classmethod def from_raw(cls, raw): if isinstance(raw, cls): return raw elif isinstance(raw, tuple): return cls(*raw) else: raise TypeError(f'unsupported "raw": {raw:!r}') @classmethod def from_row(cls, row, columns=None): if not columns: colnames = 'filename funcname name kind data'.split() else: colnames = list(columns) for i, column in enumerate(colnames): if column == 'file': colnames[i] = 'filename' elif column == 'funcname': colnames[i] = 'parent' if len(row) != len(set(colnames)): raise NotImplementedError(columns, row) kwargs = {} for column, value in zip(colnames, row): if column == 'filename': kwargs['file'] = FileInfo.from_raw(value) elif column == 'kind': kwargs['kind'] = KIND(value) elif column in cls._fields: kwargs[column] = value else: raise NotImplementedError(column) return cls(**kwargs) @property def id(self): try: return self._id except AttributeError: if self.kind is KIND.STATEMENT: self._id = None else: self._id = DeclID(str(self.file), self.funcname, self.name) return self._id @property def filename(self): if not self.file: return None return self.file.filename @property def lno(self): if not self.file: return -1 return self.file.lno @property def funcname(self): if not self.parent: return None if type(self.parent) is str: return self.parent else: return self.parent.name def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): fixed = self.file.fix_filename(relroot, **kwargs) if fixed == self.file: return self return self._replace(file=fixed) def as_row(self, columns=None): if not columns: columns = self._fields row = [] for column in columns: if column == 'file': value = self.filename elif column == 'kind': value = self.kind.value elif column == 'data': value = self._render_data() else: value = getattr(self, column) row.append(value) return row def _render_data(self): if not self.data: return None elif isinstance(self.data, str): return self.data else: # XXX raise NotImplementedError def _get_vartype(data): try: vartype = dict(data['vartype']) except KeyError: vartype = dict(data) storage = data.get('storage') else: storage = data.get('storage') or vartype.get('storage') del vartype['storage'] return storage, vartype def get_parsed_vartype(decl): kind = getattr(decl, 'kind', None) if isinstance(decl, ParsedItem): storage, vartype = _get_vartype(decl.data) typequal = vartype['typequal'] typespec = vartype['typespec'] abstract = vartype['abstract'] elif isinstance(decl, dict): kind = decl.get('kind') storage, vartype = _get_vartype(decl) typequal = vartype['typequal'] typespec = vartype['typespec'] abstract = vartype['abstract'] elif isinstance(decl, VarType): storage = None typequal, typespec, abstract = decl elif isinstance(decl, TypeDef): storage = None typequal, typespec, abstract = decl.vartype elif isinstance(decl, Variable): storage = decl.storage typequal, typespec, abstract = decl.vartype elif isinstance(decl, Signature): storage = None typequal, typespec, abstract = decl.returntype elif isinstance(decl, Function): storage = decl.storage typequal, typespec, abstract = decl.signature.returntype elif isinstance(decl, str): vartype, storage = VarType.from_str(decl) typequal, typespec, abstract = vartype else: raise NotImplementedError(decl) return kind, storage, typequal, typespec, abstract def get_default_storage(decl): if decl.kind not in (KIND.VARIABLE, KIND.FUNCTION): return None return 'extern' if decl.parent is None else 'auto' def get_effective_storage(decl, *, default=None): # Note that "static" limits access to just that C module # and "extern" (the default for module-level) allows access # outside the C module. if default is None: default = get_default_storage(decl) if default is None: return None try: storage = decl.storage except AttributeError: storage, _ = _get_vartype(decl.data) return storage or default ############################# # high-level class HighlevelParsedItem: kind = None FIELDS = ('file', 'parent', 'name', 'data') @classmethod def from_parsed(cls, parsed): if parsed.kind is not cls.kind: raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})') data, extra = cls._resolve_data(parsed.data) self = cls( cls._resolve_file(parsed), parsed.name, data, cls._resolve_parent(parsed) if parsed.parent else None, **extra or {} ) self._parsed = parsed return self @classmethod def _resolve_file(cls, parsed): fileinfo = FileInfo.from_raw(parsed.file) if not fileinfo: raise NotImplementedError(parsed) return fileinfo @classmethod def _resolve_data(cls, data): return data, None @classmethod def _raw_data(cls, data, extra): if isinstance(data, str): return data else: raise NotImplementedError(data) @classmethod def _data_as_row(cls, data, extra, colnames): row = {} for colname in colnames: if colname in row: continue rendered = cls._render_data_row_item(colname, data, extra) if rendered is iter(rendered): rendered, = rendered row[colname] = rendered return row @classmethod def _render_data_row_item(cls, colname, data, extra): if colname == 'data': return str(data) else: return None @classmethod def _render_data_row(cls, fmt, data, extra, colnames): if fmt != 'row': raise NotImplementedError datarow = cls._data_as_row(data, extra, colnames) unresolved = [c for c, v in datarow.items() if v is None] if unresolved: raise NotImplementedError(unresolved) for colname, value in datarow.items(): if type(value) != str: if colname == 'kind': datarow[colname] = value.value else: datarow[colname] = str(value) return datarow @classmethod def _render_data(cls, fmt, data, extra): row = cls._render_data_row(fmt, data, extra, ['data']) yield ' '.join(row.values()) @classmethod def _resolve_parent(cls, parsed, *, _kind=None): fileinfo = FileInfo(parsed.file.filename, -1) if isinstance(parsed.parent, str): if parsed.parent.isidentifier(): name = parsed.parent else: # XXX It could be something like " ". raise NotImplementedError(repr(parsed.parent)) parent = ParsedItem(fileinfo, _kind, None, name, None) elif type(parsed.parent) is tuple: # XXX It could be something like (kind, name). raise NotImplementedError(repr(parsed.parent)) else: return parsed.parent Parent = KIND_CLASSES.get(_kind, Declaration) return Parent.from_parsed(parent) @classmethod def _parse_columns(cls, columns): colnames = {} # {requested -> actual} columns = list(columns or cls.FIELDS) datacolumns = [] for i, colname in enumerate(columns): if colname == 'file': columns[i] = 'filename' colnames['file'] = 'filename' elif colname == 'lno': columns[i] = 'line' colnames['lno'] = 'line' elif colname in ('filename', 'line'): colnames[colname] = colname elif colname == 'data': datacolumns.append(colname) colnames[colname] = None elif colname in cls.FIELDS or colname == 'kind': colnames[colname] = colname else: datacolumns.append(colname) colnames[colname] = None return columns, datacolumns, colnames def __init__(self, file, name, data, parent=None, *, _extra=None, _shortkey=None, _key=None, ): self.file = file self.parent = parent or None self.name = name self.data = data self._extra = _extra or {} self._shortkey = _shortkey self._key = _key def __repr__(self): args = [f'{n}={getattr(self, n)!r}' for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]] return f'{type(self).__name__}({", ".join(args)})' def __str__(self): try: return self._str except AttributeError: self._str = next(self.render()) return self._str def __getattr__(self, name): try: return self._extra[name] except KeyError: raise AttributeError(name) def __hash__(self): return hash(self._key) def __eq__(self, other): if isinstance(other, HighlevelParsedItem): return self._key == other._key elif type(other) is tuple: return self._key == other else: return NotImplemented def __gt__(self, other): if isinstance(other, HighlevelParsedItem): return self._key > other._key elif type(other) is tuple: return self._key > other else: return NotImplemented @property def id(self): return self.parsed.id @property def shortkey(self): return self._shortkey @property def key(self): return self._key @property def filename(self): if not self.file: return None return self.file.filename @property def parsed(self): try: return self._parsed except AttributeError: parent = self.parent if parent is not None and not isinstance(parent, str): parent = parent.name self._parsed = ParsedItem( self.file, self.kind, parent, self.name, self._raw_data(), ) return self._parsed def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): if self.file: self.file = self.file.fix_filename(relroot, **kwargs) return self def as_rowdata(self, columns=None): columns, datacolumns, colnames = self._parse_columns(columns) return self._as_row(colnames, datacolumns, self._data_as_row) def render_rowdata(self, columns=None): columns, datacolumns, colnames = self._parse_columns(columns) def data_as_row(data, ext, cols): return self._render_data_row('row', data, ext, cols) rowdata = self._as_row(colnames, datacolumns, data_as_row) for column, value in rowdata.items(): colname = colnames.get(column) if not colname: continue if column == 'kind': value = value.value else: if column == 'parent': if self.parent: value = f'({self.parent.kind.value} {self.parent.name})' if not value: value = '-' elif type(value) is VarType: value = repr(str(value)) else: value = str(value) rowdata[column] = value return rowdata def _as_row(self, colnames, datacolumns, data_as_row): try: data = data_as_row(self.data, self._extra, datacolumns) except NotImplementedError: data = None row = data or {} for column, colname in colnames.items(): if colname == 'filename': value = self.file.filename if self.file else None elif colname == 'line': value = self.file.lno if self.file else None elif colname is None: value = getattr(self, column, None) else: value = getattr(self, colname, None) row.setdefault(column, value) return row def render(self, fmt='line'): fmt = fmt or 'line' try: render = _FORMATS[fmt] except KeyError: raise TypeError(f'unsupported fmt {fmt!r}') try: data = self._render_data(fmt, self.data, self._extra) except NotImplementedError: data = '-' yield from render(self, data) ### formats ### def _fmt_line(parsed, data=None): parts = [ f'<{parsed.kind.value}>', ] parent = '' if parsed.parent: parent = parsed.parent if not isinstance(parent, str): if parent.kind is KIND.FUNCTION: parent = f'{parent.name}()' else: parent = parent.name name = f'<{parent}>.{parsed.name}' else: name = parsed.name if data is None: data = parsed.data elif data is iter(data): data, = data parts.extend([ name, f'<{data}>' if data else '-', f'({str(parsed.file or "")})', ]) yield '\t'.join(parts) def _fmt_full(parsed, data=None): if parsed.kind is KIND.VARIABLE and parsed.parent: prefix = 'local ' suffix = f' ({parsed.parent.name})' else: # XXX Show other prefixes (e.g. global, public) prefix = suffix = '' yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}' for column, info in parsed.render_rowdata().items(): if column == 'kind': continue if column == 'name': continue if column == 'parent' and parsed.kind is not KIND.VARIABLE: continue if column == 'data': if parsed.kind in (KIND.STRUCT, KIND.UNION): column = 'members' elif parsed.kind is KIND.ENUM: column = 'enumerators' elif parsed.kind is KIND.STATEMENT: column = 'text' data, = data else: column = 'signature' data, = data if not data: # yield f'\t{column}:\t-' continue elif isinstance(data, str): yield f'\t{column}:\t{data!r}' else: yield f'\t{column}:' for line in data: yield f'\t\t- {line}' else: yield f'\t{column}:\t{info}' _FORMATS = { 'raw': (lambda v, _d: [repr(v)]), 'brief': _fmt_line, 'line': _fmt_line, 'full': _fmt_full, } ### declarations ## class Declaration(HighlevelParsedItem): @classmethod def from_row(cls, row, **markers): fixed = tuple(_tables.fix_row(row, **markers)) if cls is Declaration: _, _, _, kind, _ = fixed sub = KIND_CLASSES.get(KIND(kind)) if not sub or not issubclass(sub, Declaration): raise TypeError(f'unsupported kind, got {row!r}') else: sub = cls return sub._from_row(fixed) @classmethod def _from_row(cls, row): filename, funcname, name, kind, data = row kind = KIND._from_raw(kind) if kind is not cls.kind: raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}') fileinfo = FileInfo.from_raw(filename) extra = None if isinstance(data, str): data, extra = cls._parse_data(data, fmt='row') if extra: return cls(fileinfo, name, data, funcname, _extra=extra) else: return cls(fileinfo, name, data, funcname) @classmethod def _resolve_parent(cls, parsed, *, _kind=None): if _kind is None: raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})') return super()._resolve_parent(parsed, _kind=_kind) @classmethod def _render_data(cls, fmt, data, extra): if not data: # XXX There should be some! Forward? yield '???' else: yield from cls._format_data(fmt, data, extra) @classmethod def _render_data_row_item(cls, colname, data, extra): if colname == 'data': return cls._format_data('row', data, extra) else: return None @classmethod def _format_data(cls, fmt, data, extra): raise NotImplementedError(fmt) @classmethod def _parse_data(cls, datastr, fmt=None): """This is the reverse of _render_data.""" if not datastr or datastr is _tables.UNKNOWN or datastr == '???': return None, None elif datastr is _tables.EMPTY or datastr == '-': # All the kinds have *something* even it is unknown. raise TypeError('all declarations have data of some sort, got none') else: return cls._unformat_data(datastr, fmt) @classmethod def _unformat_data(cls, datastr, fmt=None): raise NotImplementedError(fmt) class VarType(namedtuple('VarType', 'typequal typespec abstract')): @classmethod def from_str(cls, text): orig = text storage, sep, text = text.strip().partition(' ') if not sep: text = storage storage = None elif storage not in ('auto', 'register', 'static', 'extern'): text = orig storage = None return cls._from_str(text), storage @classmethod def _from_str(cls, text): orig = text if text.startswith(('const ', 'volatile ')): typequal, _, text = text.partition(' ') else: typequal = None # Extract a series of identifiers/keywords. m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text) if not m: raise ValueError(f'invalid vartype text {orig!r}') typespec, abstract = m.groups() return cls(typequal, typespec, abstract or None) def __str__(self): parts = [] if self.qualifier: parts.append(self.qualifier) parts.append(self.spec + (self.abstract or '')) return ' '.join(parts) @property def qualifier(self): return self.typequal @property def spec(self): return self.typespec class Variable(Declaration): kind = KIND.VARIABLE @classmethod def _resolve_parent(cls, parsed): return super()._resolve_parent(parsed, _kind=KIND.FUNCTION) @classmethod def _resolve_data(cls, data): if not data: return None, None storage, vartype = _get_vartype(data) return VarType(**vartype), {'storage': storage} @classmethod def _raw_data(self, data, extra): vartype = data._asdict() return { 'storage': extra['storage'], 'vartype': vartype, } @classmethod def _format_data(cls, fmt, data, extra): storage = extra.get('storage') text = f'{storage} {data}' if storage else str(data) if fmt in ('line', 'brief'): yield text #elif fmt == 'full': elif fmt == 'row': yield text else: raise NotImplementedError(fmt) @classmethod def _unformat_data(cls, datastr, fmt=None): if fmt in ('line', 'brief'): vartype, storage = VarType.from_str(datastr) return vartype, {'storage': storage} #elif fmt == 'full': elif fmt == 'row': vartype, storage = VarType.from_str(datastr) return vartype, {'storage': storage} else: raise NotImplementedError(fmt) def __init__(self, file, name, data, parent=None, storage=None): super().__init__(file, name, data, parent, _extra={'storage': storage or None}, _shortkey=f'({parent.name}).{name}' if parent else name, _key=(str(file), # Tilde comes after all other ascii characters. f'~{parent or ""}~', name, ), ) if storage: if storage not in STORAGE: # The parser must need an update. raise NotImplementedError(storage) # Otherwise we trust the compiler to have validated it. @property def vartype(self): return self.data class Signature(namedtuple('Signature', 'params returntype inline isforward')): @classmethod def from_str(cls, text): orig = text storage, sep, text = text.strip().partition(' ') if not sep: text = storage storage = None elif storage not in ('auto', 'register', 'static', 'extern'): text = orig storage = None return cls._from_str(text), storage @classmethod def _from_str(cls, text): orig = text inline, sep, text = text.partition('|') if not sep: text = inline inline = None isforward = False if text.endswith(';'): text = text[:-1] isforward = True elif text.endswith('{}'): text = text[:-2] index = text.rindex('(') if index < 0: raise ValueError(f'bad signature text {orig!r}') params = text[index:] while params.count('(') <= params.count(')'): index = text.rindex('(', 0, index) if index < 0: raise ValueError(f'bad signature text {orig!r}') params = text[index:] text = text[:index] returntype = VarType._from_str(text.rstrip()) return cls(params, returntype, inline, isforward) def __str__(self): parts = [] if self.inline: parts.extend([ self.inline, '|', ]) parts.extend([ str(self.returntype), self.params, ';' if self.isforward else '{}', ]) return ' '.join(parts) @property def returns(self): return self.returntype @property def typequal(self): return self.returntype.typequal @property def typespec(self): return self.returntype.typespec @property def abstract(self): return self.returntype.abstract class Function(Declaration): kind = KIND.FUNCTION @classmethod def _resolve_data(cls, data): if not data: return None, None kwargs = dict(data) returntype = dict(data['returntype']) del returntype['storage'] kwargs['returntype'] = VarType(**returntype) storage = kwargs.pop('storage') return Signature(**kwargs), {'storage': storage} @classmethod def _raw_data(self, data): # XXX finish! return data @classmethod def _format_data(cls, fmt, data, extra): storage = extra.get('storage') text = f'{storage} {data}' if storage else str(data) if fmt in ('line', 'brief'): yield text #elif fmt == 'full': elif fmt == 'row': yield text else: raise NotImplementedError(fmt) @classmethod def _unformat_data(cls, datastr, fmt=None): if fmt in ('line', 'brief'): sig, storage = Signature.from_str(sig) return sig, {'storage': storage} #elif fmt == 'full': elif fmt == 'row': sig, storage = Signature.from_str(sig) return sig, {'storage': storage} else: raise NotImplementedError(fmt) def __init__(self, file, name, data, parent=None, storage=None): super().__init__(file, name, data, parent, _extra={'storage': storage}) self._shortkey = f'~{name}~ {self.data}' self._key = ( str(file), self._shortkey, ) @property def signature(self): return self.data class TypeDeclaration(Declaration): def __init__(self, file, name, data, parent=None, *, _shortkey=None): if not _shortkey: _shortkey = f'{self.kind.value} {name}' super().__init__(file, name, data, parent, _shortkey=_shortkey, _key=( str(file), _shortkey, ), ) class POTSType(TypeDeclaration): def __init__(self, name): _file = _data = _parent = None super().__init__(_file, name, _data, _parent, _shortkey=name) class FuncPtr(TypeDeclaration): def __init__(self, vartype): _file = _name = _parent = None data = vartype self.vartype = vartype super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>') class TypeDef(TypeDeclaration): kind = KIND.TYPEDEF @classmethod def _resolve_data(cls, data): if not data: raise NotImplementedError(data) kwargs = dict(data) del kwargs['storage'] if 'returntype' in kwargs: vartype = kwargs['returntype'] del vartype['storage'] kwargs['returntype'] = VarType(**vartype) datacls = Signature else: datacls = VarType return datacls(**kwargs), None @classmethod def _raw_data(self, data): # XXX finish! return data @classmethod def _format_data(cls, fmt, data, extra): text = str(data) if fmt in ('line', 'brief'): yield text elif fmt == 'full': yield text elif fmt == 'row': yield text else: raise NotImplementedError(fmt) @classmethod def _unformat_data(cls, datastr, fmt=None): if fmt in ('line', 'brief'): vartype, _ = VarType.from_str(datastr) return vartype, None #elif fmt == 'full': elif fmt == 'row': vartype, _ = VarType.from_str(datastr) return vartype, None else: raise NotImplementedError(fmt) def __init__(self, file, name, data, parent=None): super().__init__(file, name, data, parent, _shortkey=name) @property def vartype(self): return self.data class Member(namedtuple('Member', 'name vartype size')): @classmethod def from_data(cls, raw, index): name = raw.name if raw.name else index vartype = size = None if type(raw.data) is int: size = raw.data elif isinstance(raw.data, str): size = int(raw.data) elif raw.data: vartype = dict(raw.data) del vartype['storage'] if 'size' in vartype: size = vartype.pop('size') if isinstance(size, str) and size.isdigit(): size = int(size) vartype = VarType(**vartype) return cls(name, vartype, size) @classmethod def from_str(cls, text): name, _, vartype = text.partition(': ') if name.startswith('#'): name = int(name[1:]) if vartype.isdigit(): size = int(vartype) vartype = None else: vartype, _ = VarType.from_str(vartype) size = None return cls(name, vartype, size) def __str__(self): name = self.name if isinstance(self.name, str) else f'#{self.name}' return f'{name}: {self.vartype or self.size}' class _StructUnion(TypeDeclaration): @classmethod def _resolve_data(cls, data): if not data: # XXX There should be some! Forward? return None, None return [Member.from_data(v, i) for i, v in enumerate(data)], None @classmethod def _raw_data(self, data): # XXX finish! return data @classmethod def _format_data(cls, fmt, data, extra): if fmt in ('line', 'brief'): members = ', '.join(f'<{m}>' for m in data) yield f'[{members}]' elif fmt == 'full': for member in data: yield f'{member}' elif fmt == 'row': members = ', '.join(f'<{m}>' for m in data) yield f'[{members}]' else: raise NotImplementedError(fmt) @classmethod def _unformat_data(cls, datastr, fmt=None): if fmt in ('line', 'brief'): members = [Member.from_str(m[1:-1]) for m in datastr[1:-1].split(', ')] return members, None #elif fmt == 'full': elif fmt == 'row': members = [Member.from_str(m.rstrip('>').lstrip('<')) for m in datastr[1:-1].split('>, <')] return members, None else: raise NotImplementedError(fmt) def __init__(self, file, name, data, parent=None): super().__init__(file, name, data, parent) @property def members(self): return self.data class Struct(_StructUnion): kind = KIND.STRUCT class Union(_StructUnion): kind = KIND.UNION class Enum(TypeDeclaration): kind = KIND.ENUM @classmethod def _resolve_data(cls, data): if not data: # XXX There should be some! Forward? return None, None enumerators = [e if isinstance(e, str) else e.name for e in data] return enumerators, None @classmethod def _raw_data(self, data): # XXX finish! return data @classmethod def _format_data(cls, fmt, data, extra): if fmt in ('line', 'brief'): yield repr(data) elif fmt == 'full': for enumerator in data: yield f'{enumerator}' elif fmt == 'row': # XXX This won't work with CSV... yield ','.join(data) else: raise NotImplementedError(fmt) @classmethod def _unformat_data(cls, datastr, fmt=None): if fmt in ('line', 'brief'): return _strutil.unrepr(datastr), None #elif fmt == 'full': elif fmt == 'row': return datastr.split(','), None else: raise NotImplementedError(fmt) def __init__(self, file, name, data, parent=None): super().__init__(file, name, data, parent) @property def enumerators(self): return self.data ### statements ### class Statement(HighlevelParsedItem): kind = KIND.STATEMENT @classmethod def _resolve_data(cls, data): # XXX finish! return data, None @classmethod def _raw_data(self, data): # XXX finish! return data @classmethod def _render_data(cls, fmt, data, extra): # XXX Handle other formats? return repr(data) @classmethod def _parse_data(self, datastr, fmt=None): # XXX Handle other formats? return _strutil.unrepr(datastr), None def __init__(self, file, name, data, parent=None): super().__init__(file, name, data, parent, _shortkey=data or '', _key=( str(file), file.lno, # XXX Only one stmt per line? ), ) @property def text(self): return self.data ### KIND_CLASSES = {cls.kind: cls for cls in [ Variable, Function, TypeDef, Struct, Union, Enum, Statement, ]} def resolve_parsed(parsed): if isinstance(parsed, HighlevelParsedItem): return parsed try: cls = KIND_CLASSES[parsed.kind] except KeyError: raise ValueError(f'unsupported kind in {parsed!r}') return cls.from_parsed(parsed) def set_flag(item, name, value): try: setattr(item, name, value) except AttributeError: object.__setattr__(item, name, value) ############################# # composite class Declarations: @classmethod def from_decls(cls, decls): return cls(decls) @classmethod def from_parsed(cls, items): decls = (resolve_parsed(item) for item in items if item.kind is not KIND.STATEMENT) return cls.from_decls(decls) @classmethod def _resolve_key(cls, raw): if isinstance(raw, str): raw = [raw] elif isinstance(raw, Declaration): raw = ( raw.filename if cls._is_public(raw) else None, # `raw.parent` is always None for types and functions. raw.parent if raw.kind is KIND.VARIABLE else None, raw.name, ) extra = None if len(raw) == 1: name, = raw if name: name = str(name) if name.endswith(('.c', '.h')): # This is only legit as a query. key = (name, None, None) else: key = (None, None, name) else: key = (None, None, None) elif len(raw) == 2: parent, name = raw name = str(name) if isinstance(parent, Declaration): key = (None, parent.name, name) elif not parent: key = (None, None, name) else: parent = str(parent) if parent.endswith(('.c', '.h')): key = (parent, None, name) else: key = (None, parent, name) else: key, extra = raw[:3], raw[3:] filename, funcname, name = key filename = str(filename) if filename else None if isinstance(funcname, Declaration): funcname = funcname.name else: funcname = str(funcname) if funcname else None name = str(name) if name else None key = (filename, funcname, name) return key, extra @classmethod def _is_public(cls, decl): # For .c files don't we need info from .h files to make this decision? # XXX Check for "extern". # For now we treat all decls a "private" (have filename set). return False def __init__(self, decls): # (file, func, name) -> decl # "public": # * (None, None, name) # "private", "global": # * (file, None, name) # "private", "local": # * (file, func, name) if hasattr(decls, 'items'): self._decls = decls else: self._decls = {} self._extend(decls) # XXX always validate? def validate(self): for key, decl in self._decls.items(): if type(key) is not tuple or len(key) != 3: raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})') filename, funcname, name = key if not name: raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})') elif type(name) is not str: raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})') # XXX Check filename type? # XXX Check funcname type? if decl.kind is KIND.STATEMENT: raise ValueError(f'expected a declaration, got {decl!r}') def __repr__(self): return f'{type(self).__name__}({list(self)})' def __len__(self): return len(self._decls) def __iter__(self): yield from self._decls def __getitem__(self, key): # XXX Be more exact for the 3-tuple case? if type(key) not in (str, tuple): raise KeyError(f'unsupported key {key!r}') resolved, extra = self._resolve_key(key) if extra: raise KeyError(f'key must have at most 3 parts, got {key!r}') if not resolved[2]: raise ValueError(f'expected name in key, got {key!r}') try: return self._decls[resolved] except KeyError: if type(key) is tuple and len(key) == 3: filename, funcname, name = key else: filename, funcname, name = resolved if filename and not filename.endswith(('.c', '.h')): raise KeyError(f'invalid filename in key {key!r}') elif funcname and funcname.endswith(('.c', '.h')): raise KeyError(f'invalid funcname in key {key!r}') elif name and name.endswith(('.c', '.h')): raise KeyError(f'invalid name in key {key!r}') else: raise # re-raise @property def types(self): return self._find(kind=KIND.TYPES) @property def functions(self): return self._find(None, None, None, KIND.FUNCTION) @property def variables(self): return self._find(None, None, None, KIND.VARIABLE) def iter_all(self): yield from self._decls.values() def get(self, key, default=None): try: return self[key] except KeyError: return default #def add_decl(self, decl, key=None): # decl = _resolve_parsed(decl) # self._add_decl(decl, key) def find(self, *key, **explicit): if not key: if not explicit: return iter(self) return self._find(**explicit) resolved, extra = self._resolve_key(key) filename, funcname, name = resolved if not extra: kind = None elif len(extra) == 1: kind, = extra else: raise KeyError(f'key must have at most 4 parts, got {key!r}') implicit= {} if filename: implicit['filename'] = filename if funcname: implicit['funcname'] = funcname if name: implicit['name'] = name if kind: implicit['kind'] = kind return self._find(**implicit, **explicit) def _find(self, filename=None, funcname=None, name=None, kind=None): for decl in self._decls.values(): if filename and decl.filename != filename: continue if funcname: if decl.kind is not KIND.VARIABLE: continue if decl.parent.name != funcname: continue if name and decl.name != name: continue if kind: kinds = KIND.resolve_group(kind) if decl.kind not in kinds: continue yield decl def _add_decl(self, decl, key=None): if key: if type(key) not in (str, tuple): raise NotImplementedError((key, decl)) # Any partial key will be turned into a full key, but that # same partial key will still match a key lookup. resolved, _ = self._resolve_key(key) if not resolved[2]: raise ValueError(f'expected name in key, got {key!r}') key = resolved # XXX Also add with the decl-derived key if not the same? else: key, _ = self._resolve_key(decl) self._decls[key] = decl def _extend(self, decls): decls = iter(decls) # Check only the first item. for decl in decls: if isinstance(decl, Declaration): self._add_decl(decl) # Add the rest without checking. for decl in decls: self._add_decl(decl) elif isinstance(decl, HighlevelParsedItem): raise NotImplementedError(decl) else: try: key, decl = decl except ValueError: raise NotImplementedError(decl) if not isinstance(decl, Declaration): raise NotImplementedError(decl) self._add_decl(decl, key) # Add the rest without checking. for key, decl in decls: self._add_decl(decl, key) # The iterator will be exhausted at this point.