541 lines
15 KiB
Python
541 lines
15 KiB
Python
import io
|
|
import logging
|
|
import os
|
|
import os.path
|
|
import re
|
|
import sys
|
|
|
|
from c_common import fsutil
|
|
from c_common.logging import VERBOSITY, Printer
|
|
from c_common.scriptutil import (
|
|
add_verbosity_cli,
|
|
add_traceback_cli,
|
|
add_sepval_cli,
|
|
add_progress_cli,
|
|
add_files_cli,
|
|
add_commands_cli,
|
|
process_args_by_key,
|
|
configure_logger,
|
|
get_prog,
|
|
filter_filenames,
|
|
iter_marks,
|
|
)
|
|
from c_parser.info import KIND
|
|
from c_parser.match import is_type_decl
|
|
from .match import filter_forward
|
|
from . import (
|
|
analyze as _analyze,
|
|
datafiles as _datafiles,
|
|
check_all as _check_all,
|
|
)
|
|
|
|
|
|
KINDS = [
|
|
KIND.TYPEDEF,
|
|
KIND.STRUCT,
|
|
KIND.UNION,
|
|
KIND.ENUM,
|
|
KIND.FUNCTION,
|
|
KIND.VARIABLE,
|
|
KIND.STATEMENT,
|
|
]
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
#######################################
|
|
# table helpers
|
|
|
|
TABLE_SECTIONS = {
|
|
'types': (
|
|
['kind', 'name', 'data', 'file'],
|
|
KIND.is_type_decl,
|
|
(lambda v: (v.kind.value, v.filename or '', v.name)),
|
|
),
|
|
'typedefs': 'types',
|
|
'structs': 'types',
|
|
'unions': 'types',
|
|
'enums': 'types',
|
|
'functions': (
|
|
['name', 'data', 'file'],
|
|
(lambda kind: kind is KIND.FUNCTION),
|
|
(lambda v: (v.filename or '', v.name)),
|
|
),
|
|
'variables': (
|
|
['name', 'parent', 'data', 'file'],
|
|
(lambda kind: kind is KIND.VARIABLE),
|
|
(lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
|
|
),
|
|
'statements': (
|
|
['file', 'parent', 'data'],
|
|
(lambda kind: kind is KIND.STATEMENT),
|
|
(lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
|
|
),
|
|
KIND.TYPEDEF: 'typedefs',
|
|
KIND.STRUCT: 'structs',
|
|
KIND.UNION: 'unions',
|
|
KIND.ENUM: 'enums',
|
|
KIND.FUNCTION: 'functions',
|
|
KIND.VARIABLE: 'variables',
|
|
KIND.STATEMENT: 'statements',
|
|
}
|
|
|
|
|
|
def _render_table(items, columns, relroot=None):
|
|
# XXX improve this
|
|
header = '\t'.join(columns)
|
|
div = '--------------------'
|
|
yield header
|
|
yield div
|
|
total = 0
|
|
for item in items:
|
|
rowdata = item.render_rowdata(columns)
|
|
row = [rowdata[c] for c in columns]
|
|
if relroot and 'file' in columns:
|
|
index = columns.index('file')
|
|
row[index] = os.path.relpath(row[index], relroot)
|
|
yield '\t'.join(row)
|
|
total += 1
|
|
yield div
|
|
yield f'total: {total}'
|
|
|
|
|
|
def build_section(name, groupitems, *, relroot=None):
|
|
info = TABLE_SECTIONS[name]
|
|
while type(info) is not tuple:
|
|
if name in KINDS:
|
|
name = info
|
|
info = TABLE_SECTIONS[info]
|
|
|
|
columns, match_kind, sortkey = info
|
|
items = (v for v in groupitems if match_kind(v.kind))
|
|
items = sorted(items, key=sortkey)
|
|
def render():
|
|
yield ''
|
|
yield f'{name}:'
|
|
yield ''
|
|
for line in _render_table(items, columns, relroot):
|
|
yield line
|
|
return items, render
|
|
|
|
|
|
#######################################
|
|
# the checks
|
|
|
|
CHECKS = {
|
|
#'globals': _check_globals,
|
|
}
|
|
|
|
|
|
def add_checks_cli(parser, checks=None, *, add_flags=None):
|
|
default = False
|
|
if not checks:
|
|
checks = list(CHECKS)
|
|
default = True
|
|
elif isinstance(checks, str):
|
|
checks = [checks]
|
|
if (add_flags is None and len(checks) > 1) or default:
|
|
add_flags = True
|
|
|
|
process_checks = add_sepval_cli(parser, '--check', 'checks', checks)
|
|
if add_flags:
|
|
for check in checks:
|
|
parser.add_argument(f'--{check}', dest='checks',
|
|
action='append_const', const=check)
|
|
return [
|
|
process_checks,
|
|
]
|
|
|
|
|
|
def _get_check_handlers(fmt, printer, verbosity=VERBOSITY):
|
|
div = None
|
|
def handle_after():
|
|
pass
|
|
if not fmt:
|
|
div = ''
|
|
def handle_failure(failure, data):
|
|
data = repr(data)
|
|
if verbosity >= 3:
|
|
logger.info(f'failure: {failure}')
|
|
logger.info(f'data: {data}')
|
|
else:
|
|
logger.warn(f'failure: {failure} (data: {data})')
|
|
elif fmt == 'raw':
|
|
def handle_failure(failure, data):
|
|
print(f'{failure!r} {data!r}')
|
|
elif fmt == 'brief':
|
|
def handle_failure(failure, data):
|
|
parent = data.parent or ''
|
|
funcname = parent if isinstance(parent, str) else parent.name
|
|
name = f'({funcname}).{data.name}' if funcname else data.name
|
|
failure = failure.split('\t')[0]
|
|
print(f'{data.filename}:{name} - {failure}')
|
|
elif fmt == 'summary':
|
|
def handle_failure(failure, data):
|
|
print(_fmt_one_summary(data, failure))
|
|
elif fmt == 'full':
|
|
div = ''
|
|
def handle_failure(failure, data):
|
|
name = data.shortkey if data.kind is KIND.VARIABLE else data.name
|
|
parent = data.parent or ''
|
|
funcname = parent if isinstance(parent, str) else parent.name
|
|
known = 'yes' if data.is_known else '*** NO ***'
|
|
print(f'{data.kind.value} {name!r} failed ({failure})')
|
|
print(f' file: {data.filename}')
|
|
print(f' func: {funcname or "-"}')
|
|
print(f' name: {data.name}')
|
|
print(f' data: ...')
|
|
print(f' type unknown: {known}')
|
|
else:
|
|
if fmt in FORMATS:
|
|
raise NotImplementedError(fmt)
|
|
raise ValueError(f'unsupported fmt {fmt!r}')
|
|
return handle_failure, handle_after, div
|
|
|
|
|
|
#######################################
|
|
# the formats
|
|
|
|
def fmt_raw(analysis):
|
|
for item in analysis:
|
|
yield from item.render('raw')
|
|
|
|
|
|
def fmt_brief(analysis):
|
|
# XXX Support sorting.
|
|
items = sorted(analysis)
|
|
for kind in KINDS:
|
|
if kind is KIND.STATEMENT:
|
|
continue
|
|
for item in items:
|
|
if item.kind is not kind:
|
|
continue
|
|
yield from item.render('brief')
|
|
yield f' total: {len(items)}'
|
|
|
|
|
|
def fmt_summary(analysis):
|
|
# XXX Support sorting and grouping.
|
|
items = list(analysis)
|
|
total = len(items)
|
|
|
|
def section(name):
|
|
_, render = build_section(name, items)
|
|
yield from render()
|
|
|
|
yield from section('types')
|
|
yield from section('functions')
|
|
yield from section('variables')
|
|
yield from section('statements')
|
|
|
|
yield ''
|
|
# yield f'grand total: {len(supported) + len(unsupported)}'
|
|
yield f'grand total: {total}'
|
|
|
|
|
|
def _fmt_one_summary(item, extra=None):
|
|
parent = item.parent or ''
|
|
funcname = parent if isinstance(parent, str) else parent.name
|
|
if extra:
|
|
return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}'
|
|
else:
|
|
return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}'
|
|
|
|
|
|
def fmt_full(analysis):
|
|
# XXX Support sorting.
|
|
items = sorted(analysis, key=lambda v: v.key)
|
|
yield ''
|
|
for item in items:
|
|
yield from item.render('full')
|
|
yield ''
|
|
yield f'total: {len(items)}'
|
|
|
|
|
|
FORMATS = {
|
|
'raw': fmt_raw,
|
|
'brief': fmt_brief,
|
|
'summary': fmt_summary,
|
|
'full': fmt_full,
|
|
}
|
|
|
|
|
|
def add_output_cli(parser, *, default='summary'):
|
|
parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS))
|
|
|
|
def process_args(args, *, argv=None):
|
|
pass
|
|
return process_args
|
|
|
|
|
|
#######################################
|
|
# the commands
|
|
|
|
def _cli_check(parser, checks=None, **kwargs):
|
|
if isinstance(checks, str):
|
|
checks = [checks]
|
|
if checks is False:
|
|
process_checks = None
|
|
elif checks is None:
|
|
process_checks = add_checks_cli(parser)
|
|
elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]):
|
|
check = checks[0][1:-1]
|
|
def process_checks(args, *, argv=None):
|
|
args.checks = [check]
|
|
else:
|
|
process_checks = add_checks_cli(parser, checks=checks)
|
|
process_progress = add_progress_cli(parser)
|
|
process_output = add_output_cli(parser, default=None)
|
|
process_files = add_files_cli(parser, **kwargs)
|
|
return [
|
|
process_checks,
|
|
process_progress,
|
|
process_output,
|
|
process_files,
|
|
]
|
|
|
|
|
|
def cmd_check(filenames, *,
|
|
checks=None,
|
|
ignored=None,
|
|
fmt=None,
|
|
failfast=False,
|
|
iter_filenames=None,
|
|
relroot=fsutil.USE_CWD,
|
|
track_progress=None,
|
|
verbosity=VERBOSITY,
|
|
_analyze=_analyze,
|
|
_CHECKS=CHECKS,
|
|
**kwargs
|
|
):
|
|
if not checks:
|
|
checks = _CHECKS
|
|
elif isinstance(checks, str):
|
|
checks = [checks]
|
|
checks = [_CHECKS[c] if isinstance(c, str) else c
|
|
for c in checks]
|
|
printer = Printer(verbosity)
|
|
(handle_failure, handle_after, div
|
|
) = _get_check_handlers(fmt, printer, verbosity)
|
|
|
|
filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
|
|
filenames = filter_filenames(filenames, iter_filenames, relroot)
|
|
if track_progress:
|
|
filenames = track_progress(filenames)
|
|
|
|
logger.info('analyzing files...')
|
|
analyzed = _analyze(filenames, **kwargs)
|
|
analyzed.fix_filenames(relroot, normalize=False)
|
|
decls = filter_forward(analyzed, markpublic=True)
|
|
|
|
logger.info('checking analysis results...')
|
|
failed = []
|
|
for data, failure in _check_all(decls, checks, failfast=failfast):
|
|
if data is None:
|
|
printer.info('stopping after one failure')
|
|
break
|
|
if div is not None and len(failed) > 0:
|
|
printer.info(div)
|
|
failed.append(data)
|
|
handle_failure(failure, data)
|
|
handle_after()
|
|
|
|
printer.info('-------------------------')
|
|
logger.info(f'total failures: {len(failed)}')
|
|
logger.info('done checking')
|
|
|
|
if fmt == 'summary':
|
|
print('Categorized by storage:')
|
|
print()
|
|
from .match import group_by_storage
|
|
grouped = group_by_storage(failed, ignore_non_match=False)
|
|
for group, decls in grouped.items():
|
|
print()
|
|
print(group)
|
|
for decl in decls:
|
|
print(' ', _fmt_one_summary(decl))
|
|
print(f'subtotal: {len(decls)}')
|
|
|
|
if len(failed) > 0:
|
|
sys.exit(len(failed))
|
|
|
|
|
|
def _cli_analyze(parser, **kwargs):
|
|
process_progress = add_progress_cli(parser)
|
|
process_output = add_output_cli(parser)
|
|
process_files = add_files_cli(parser, **kwargs)
|
|
return [
|
|
process_progress,
|
|
process_output,
|
|
process_files,
|
|
]
|
|
|
|
|
|
# XXX Support filtering by kind.
|
|
def cmd_analyze(filenames, *,
|
|
fmt=None,
|
|
iter_filenames=None,
|
|
relroot=fsutil.USE_CWD,
|
|
track_progress=None,
|
|
verbosity=None,
|
|
_analyze=_analyze,
|
|
formats=FORMATS,
|
|
**kwargs
|
|
):
|
|
verbosity = verbosity if verbosity is not None else 3
|
|
|
|
try:
|
|
do_fmt = formats[fmt]
|
|
except KeyError:
|
|
raise ValueError(f'unsupported fmt {fmt!r}')
|
|
|
|
filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
|
|
filenames = filter_filenames(filenames, iter_filenames, relroot)
|
|
if track_progress:
|
|
filenames = track_progress(filenames)
|
|
|
|
logger.info('analyzing files...')
|
|
analyzed = _analyze(filenames, **kwargs)
|
|
analyzed.fix_filenames(relroot, normalize=False)
|
|
decls = filter_forward(analyzed, markpublic=True)
|
|
|
|
for line in do_fmt(decls):
|
|
print(line)
|
|
|
|
|
|
def _cli_data(parser, filenames=None, known=None):
|
|
ArgumentParser = type(parser)
|
|
common = ArgumentParser(add_help=False)
|
|
# These flags will get processed by the top-level parse_args().
|
|
add_verbosity_cli(common)
|
|
add_traceback_cli(common)
|
|
|
|
subs = parser.add_subparsers(dest='datacmd')
|
|
|
|
sub = subs.add_parser('show', parents=[common])
|
|
if known is None:
|
|
sub.add_argument('--known', required=True)
|
|
if filenames is None:
|
|
sub.add_argument('filenames', metavar='FILE', nargs='+')
|
|
|
|
sub = subs.add_parser('dump', parents=[common])
|
|
if known is None:
|
|
sub.add_argument('--known')
|
|
sub.add_argument('--show', action='store_true')
|
|
process_progress = add_progress_cli(sub)
|
|
|
|
sub = subs.add_parser('check', parents=[common])
|
|
if known is None:
|
|
sub.add_argument('--known', required=True)
|
|
|
|
def process_args(args, *, argv):
|
|
if args.datacmd == 'dump':
|
|
process_progress(args, argv)
|
|
return process_args
|
|
|
|
|
|
def cmd_data(datacmd, filenames, known=None, *,
|
|
_analyze=_analyze,
|
|
formats=FORMATS,
|
|
extracolumns=None,
|
|
relroot=fsutil.USE_CWD,
|
|
track_progress=None,
|
|
**kwargs
|
|
):
|
|
kwargs.pop('verbosity', None)
|
|
usestdout = kwargs.pop('show', None)
|
|
if datacmd == 'show':
|
|
do_fmt = formats['summary']
|
|
if isinstance(known, str):
|
|
known, _ = _datafiles.get_known(known, extracolumns, relroot)
|
|
for line in do_fmt(known):
|
|
print(line)
|
|
elif datacmd == 'dump':
|
|
filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
|
|
if track_progress:
|
|
filenames = track_progress(filenames)
|
|
analyzed = _analyze(filenames, **kwargs)
|
|
analyzed.fix_filenames(relroot, normalize=False)
|
|
if known is None or usestdout:
|
|
outfile = io.StringIO()
|
|
_datafiles.write_known(analyzed, outfile, extracolumns,
|
|
relroot=relroot)
|
|
print(outfile.getvalue())
|
|
else:
|
|
_datafiles.write_known(analyzed, known, extracolumns,
|
|
relroot=relroot)
|
|
elif datacmd == 'check':
|
|
raise NotImplementedError(datacmd)
|
|
else:
|
|
raise ValueError(f'unsupported data command {datacmd!r}')
|
|
|
|
|
|
COMMANDS = {
|
|
'check': (
|
|
'analyze and fail if the given C source/header files have any problems',
|
|
[_cli_check],
|
|
cmd_check,
|
|
),
|
|
'analyze': (
|
|
'report on the state of the given C source/header files',
|
|
[_cli_analyze],
|
|
cmd_analyze,
|
|
),
|
|
'data': (
|
|
'check/manage local data (e.g. knwon types, ignored vars, caches)',
|
|
[_cli_data],
|
|
cmd_data,
|
|
),
|
|
}
|
|
|
|
|
|
#######################################
|
|
# the script
|
|
|
|
def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None):
|
|
import argparse
|
|
parser = argparse.ArgumentParser(
|
|
prog=prog or get_prog(),
|
|
)
|
|
|
|
processors = add_commands_cli(
|
|
parser,
|
|
commands={k: v[1] for k, v in COMMANDS.items()},
|
|
commonspecs=[
|
|
add_verbosity_cli,
|
|
add_traceback_cli,
|
|
],
|
|
subset=subset,
|
|
)
|
|
|
|
args = parser.parse_args(argv)
|
|
ns = vars(args)
|
|
|
|
cmd = ns.pop('cmd')
|
|
|
|
verbosity, traceback_cm = process_args_by_key(
|
|
args,
|
|
argv,
|
|
processors[cmd],
|
|
['verbosity', 'traceback_cm'],
|
|
)
|
|
# "verbosity" is sent to the commands, so we put it back.
|
|
args.verbosity = verbosity
|
|
|
|
return cmd, ns, verbosity, traceback_cm
|
|
|
|
|
|
def main(cmd, cmd_kwargs):
|
|
try:
|
|
run_cmd = COMMANDS[cmd][0]
|
|
except KeyError:
|
|
raise ValueError(f'unsupported cmd {cmd!r}')
|
|
run_cmd(**cmd_kwargs)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
|
|
configure_logger(verbosity)
|
|
with traceback_cm:
|
|
main(cmd, cmd_kwargs)
|