diff --git a/Makefile b/Makefile index 425f6717d5..3078b79190 100644 --- a/Makefile +++ b/Makefile @@ -303,7 +303,6 @@ check_%: @echo uorb_graphs: - @./Tools/uorb_graph/create_from_startupscript.sh @./Tools/uorb_graph/create.py --src-path src --exclude-path src/examples --file Tools/uorb_graph/graph_full @$(MAKE) --no-print-directory px4_fmu-v2_default uorb_graph @$(MAKE) --no-print-directory px4_fmu-v4_default uorb_graph diff --git a/Tools/uorb_graph/.gitignore b/Tools/uorb_graph/.gitignore index d69d1a4252..92d70396b6 100644 --- a/Tools/uorb_graph/.gitignore +++ b/Tools/uorb_graph/.gitignore @@ -1,2 +1,4 @@ *.json +*.fv +*.fv.pdf diff --git a/Tools/uorb_graph/create.py b/Tools/uorb_graph/create.py index 6df985c59d..54175a965a 100755 --- a/Tools/uorb_graph/create.py +++ b/Tools/uorb_graph/create.py @@ -3,12 +3,14 @@ from __future__ import print_function import argparse -import os import codecs -import re import colorsys import json +import logging +import os +import re import sys +from typing import Optional, Set, Tuple parser = argparse.ArgumentParser( @@ -20,13 +22,18 @@ parser.add_argument('-s', '--src-path', action='append', parser.add_argument('-e', '--exclude-path', action='append', help='Excluded path(s), can be specified multiple times', default=[]) +parser.add_argument('--merge-depends', action='store_true', + help='Merge library topics inte the modules that depend on them.') +parser.add_argument('-v','--verbosity', action='count', + help='increase output verbosity; primarily for debugging; repeat for more detail', + default=0) parser.add_argument('-f', '--file', metavar='file', action='store', help='output file name prefix', default='graph') parser.add_argument('-o', '--output', metavar='output', action='store', help='output format (json or graphviz)', default='json') -parser.add_argument('--use-topic-union', action='store_true', +parser.add_argument('-u','--use-topic-union', action='store_true', help=''' Use the union of all publication and subscription topics (useful for complete graphs or only few/single module(s)). The default is to use the intersection @@ -36,14 +43,9 @@ parser.add_argument('-m', '--modules', action='store', 'MAIN, e.g. from a startup script)', default='') -args = parser.parse_args() - - -g_debug = False -def dbg_print(string): - if g_debug: - print(string) +logging.basicConfig(level=logging.WARNING) +log = logging.getLogger() def get_N_colors(N, s=0.8, v=0.9): """ get N distinct colors as a list of hex strings """ @@ -59,7 +61,7 @@ class PubSub(object): """ Collects either publication or subscription information for nodes (modules and topics) & edges """ - def __init__(self, is_publication, topic_blacklist, orb_pub_sub_regexes, special_cases): + def __init__(self, name, topic_blacklist, regexes): """ :param is_publication: if True, publications, False for subscriptions @@ -68,189 +70,216 @@ class PubSub(object): (e.g. orb_subscribe). They need to have 2 captures, the second one is the one capturing ORB_ID( """ - self._module_pubsubs = {} # key = module name, value = set of topic names - self._special_cases = special_cases - self._special_cases_matched = None + self._name = name self._topic_blacklist = topic_blacklist - self._orb_pub_sub_regexes = orb_pub_sub_regexes - if is_publication: - self._method = 'Publication' - else: - self._method = 'Subscription' + self._regexes = set([ re.compile(regex) for regex in regexes]) - def reset(self): - self._special_cases_matched = [False]*len(self._special_cases) - - def filter_modules(self, module_whitelist): - remove = [k for k in self._module_pubsubs if k not in module_whitelist] - for k in remove: del self._module_pubsubs[k] - - - def check_if_match_found(self, modules): - """ check if all special cases got a match (if not, it means the source - code got changed) - """ - for i, (module_match, file_match_re, src_match_re, _) in enumerate(self._special_cases): - if module_match in modules and src_match_re is not None: - if not self._special_cases_matched[i]: - raise Exception('Module '+module_match+ - ': no match for '+self._method+' special case'+ - src_match_re.pattern+'. The case needs to be updated') - - - def extract(self, file_name, src_str, module, orb_id_vehicle_attitude_controls_topic): + def match(self, source_line: str) -> Set[str]: """ Extract subscribed/published topics from a source string :param src_str: string of C/C++ code with comments and whitespace removed + :return: if any topic was found, it is returned as a str. Otherwise, None """ - orb_pubsub_matches = [] - for regex in self._orb_pub_sub_regexes: - orb_pubsub_matches += re.findall(regex, src_str) + matches = set() + for regex in self._regexes: + # just the matches for this particular pattern: + match = regex.search(source_line) - orb_id = 'ORB_ID(' + if match is None: + continue - for _, match in orb_pubsub_matches: - if match == 'ORB_ID_VEHICLE_ATTITUDE_CONTROLS': # special case - match = orb_id+orb_id_vehicle_attitude_controls_topic + # # all regexes should contain 3 capture groups (or else this code block crashes) + # total_match = match.group(0) + route_group, topic_group = match.groups() + + log.debug(f" ####:{self._name}: {route_group}, {topic_group}") + + # # TODO: handle this case... but not sure where, yet + # if match == 'ORB_ID_VEHICLE_ATTITUDE_CONTROLS': # special case + # match = orb_id+orb_id_vehicle_attitude_controls_topic # match has the form: '[ORB_ID(]' - if match.startswith(orb_id): - topic_name = match[len(orb_id):] - self._add_topic(topic_name, file_name, module) + if route_group: + if route_group == 'ORB_ID': + log.debug(" >>> Found ORB_ID topic: " + topic_group + " w/regex: " + str(regex.pattern)) + self._add_topic(matches, topic_group) + break + elif route_group == '<' and topic_group.endswith('_s'): + topic_group = topic_group[:-2] + log.debug(" >>> Found C++ template-declaration: " + topic_group + " w/regex: " + str(regex.pattern)) + self._add_topic(matches, topic_group) + # continue processing + elif route_group in ['{','('] and topic_group.endswith('_s'): + topic_group = topic_group[:-2] + log.debug(" >>> Found standard declaration: " + topic_group + " w/regex: " + str(regex.pattern)) + self._add_topic(matches, topic_group) + break + elif route_group == '[': + if topic_group.endswith('_s'): + topic_group = topic_group[:-2] + log.debug(" >>> Found array declaration: " + topic_group + " w/regex: " + str(regex.pattern)) + self._add_topic(matches, topic_group) + break + else: + # no topic found -- ambiguity -- return an empty set + return set() + elif 'Multi' in route_group and topic_group.endswith('_s'): + topic_group = topic_group[:-2] + log.debug(" >>> Found 'multi' declaration: " + topic_group + " w/regex: " + str(regex.pattern)) + self._add_topic(matches, topic_group) + break + else: + raise SyntaxError('!!! Encountered regex case: `route_group` contains unrecognized value!: '+ route_group+' (::'+str(regex.pattern)+')\n' + + " ("+ route_group+', '+topic_group +")\n" + + " " + source_line) + else: - ignore_found = False - for module_match, file_match_re, _, ignore_re in self._special_cases: - if module == module_match: - if file_match_re.search(file_name): - if ignore_re.search(match): - ignore_found = True - if not ignore_found: - # If we land here, we need to add another special case - raise Exception(self._method+' w/o ORB_ID(): '+match+' in ' - +file_name+' ('+module+'). You need to add another special case.') + raise SyntaxError(" !!! unhandled case: unknown-variant: "+route_group+", " + topic_group + " ....from regex: " + str(regex.pattern)) - # handle special cases - for i, (module_match, file_match_re, src_match_re, _) in enumerate(self._special_cases): - if src_match_re is None: - continue - if module == module_match: - if file_match_re.search(file_name): - matches = src_match_re.findall(src_str) - for match in matches: - # match has the form: '[ORB_ID(]' - if match.startswith(orb_id): - topic_name = match[len(orb_id):] - dbg_print('Found '+self._method+' for special case in ' - +module+': '+topic_name) - self._add_topic(topic_name, file_name, module) - self._special_cases_matched[i] = True - else: - # this is not fatal, as it could be a method delaration/definition - dbg_print('Special case '+self._method+' w/o ORB_ID(): ' - +match+' in '+file_name+' ('+module+')') + return matches - def _add_topic(self, topic_name, file_name, module): - """ add a subscription/publication for a module - """ + def _add_topic(self, topic_set: Set[str], topic_name: str): + """ add topic to set, unless the topic is ignored """ if topic_name in self._topic_blacklist: - dbg_print('ignoring blacklisted topic '+topic_name) + log.debug(" XX Ignoring blacklisted topic " + topic_name) return + else: + return topic_set.add(topic_name) - if module is None: - if not file_name.endswith('hott/messages.cpp'): # hott has a special module structure. just ignore it - print('Warning: found '+self._method+' without associated module: ' - +topic_name+' in '+file_name) - return +class Publications(PubSub): + """ Collects topic publication information for scopes """ - if not module in self._module_pubsubs: - self._module_pubsubs[module] = set() - self._module_pubsubs[module].add(topic_name) + def __init__(self, topic_blacklist, regexes): + super().__init__('PUB', topic_blacklist, regexes) - def get_topics(self, modules): - """ get the set of topics - :param modules: list of modules to take into account - """ - topics = set() - for module in modules: - if module in self._module_pubsubs: - topics |= self._module_pubsubs[module] - return topics +class Subscriptions(PubSub): + """ Collects topic subscription information for scopes """ + + def __init__(self, topic_blacklist, regexes): + super().__init__('SUB', topic_blacklist, regexes) + + +class Ambiguities(PubSub): + """ Collects topic information that cannot be classified """ + + def __init__(self, topic_blacklist, regexes): + super().__init__('AMB', topic_blacklist, regexes) + + +class Scope(object): + """ Defines a scope to add dependencies or topics to """ + def __init__(self, typename, name): + self.publications = set() + self.subscriptions = set() + self.dependencies = set() + self.ambiguities = set() + self._name = name + self._typename = typename + + def add_dependency(self, dependency_name: str): + if isinstance( dependency_name, str): + self.dependencies.add(dependency_name) + + def is_empty(self): + return (0 == len(self.publications)) and (0==len(self.subscriptions)) @property - def pubsubs(self): - """ get dict of all publication/subscriptions (key=modules, value=set of - topic names""" - return self._module_pubsubs + def name(self): + return self._name + def reduce_ambiguities(self) -> Set[str]: + self.ambiguities = self.ambiguities - self.subscriptions - self.ambiguities + return self.dependencies + + @property + def typename(self): + return self._typename + + # define these so we can hash these classes in dicts and sets + def __hash__(self): + return self._name.__hash__() + + def __eq__(self, other): + if isinstance(other, str): + return self._name == other + else: + return self._name == other._name + +class LibraryScope(Scope): + def __init__(self, name): + super().__init__('Library',name) + +class ModuleScope(Scope): + def __init__(self, name): + super().__init__('Module',name) class Graph(object): """ Collects Node and Edge information by parsing the source tree """ - def __init__(self, module_whitelist=[], topic_blacklist=[]): - self._current_module = [] # stack with current module (they can be nested) - self._all_modules = set() # set of all found modules - self._comment_remove_pattern = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', - re.DOTALL | re.MULTILINE) + def __init__(self, **kwargs): + """ + :kwargs: + - scope_whitelist + - scope_blacklist + - topic_blacklist + """ + + self._comment_remove_pattern = re.compile( r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) self._whitespace_pattern = re.compile(r'\s+') - self._module_whitelist = module_whitelist - self._excluded_paths = [] + self._scope_blacklist = set(kwargs.get('scope_blacklist',set())) + self._scope_whitelist = set(kwargs.get('scope_whitelist',set())) + + self._path_blacklist = [] + + self._topic_blacklist = set(kwargs.get('topic_blacklist',set())) self._orb_id_vehicle_attitude_controls_topic = 'actuator_controls_0' - self._orb_id_vehicle_attitude_controls_re = \ - re.compile(r'\#define\s+ORB_ID_VEHICLE_ATTITUDE_CONTROLS\s+([^,)]+)') + self._orb_id_vehicle_attitude_controls_re = re.compile(r'\#define\s+ORB_ID_VEHICLE_ATTITUDE_CONTROLS\s+([^,)]+)') - self._module_subscriptions = {} # key = module name, value = set of topic names - self._module_publications = {} # key = module name, value = set of topic names + self._warnings = [] # list off all ambiguous scan sites - self._modules = set() # all modules - self._topics = set() # all topics + self._current_scope = [] # stack with current module (they can be nested) + + self._found_modules = {} # dict of all found modules + self._found_libraries = {} # dict of all found modules + + self._print_nodes = set() # combination of libraries + modules + self._print_topics = set() # all topics self._topic_colors = {} # key = topic, value = color (html string) - # handle special cases - # format: list of tuples with 4 entries: - # - module name to match (module MAIN) - # - regex for file name(s) to match within the module (matched against the full path) - # - regex to extract the topic name: the match must be ORB_ID( - # Note: whitespace is removed from source code, so it does not need to be - # accounted for in the regex. - # If this is None, it will just be ignored - # - regex to ignore matches in the form orb_[subscribe|advertise]( - # (the expectation is that the previous matching ORB_ID() will be passed - # to this, so that we can ignore it) - special_cases_sub = [ - ('listener', r'.*', None, r'^(id)$'), - ('logger', r'.*', None, r'^(topic|sub\.metadata|_polling_topic_meta)$'), + # note: the source-file-string is pre-processed to remove whitespace -- regexes should ignore whitespace + # note: the regexes should have at least 3 capture groups '()'; otherwise they break downstream code + capture_cases_sub = [r"\borb_subscribe(?:_multi|)\b\s*\(\s*(ORB_ID)\s*\(\s*(\w+)", + r"(?:uORB::)Subscription(?:Interval|)\s+\w+\s*[\{\(]\s*(ORB_ID)\s*\(\s*(\w+)", + r"(?:uORB::)Subscription(?:Data|MultiArray|Blocking|)\s*(<)\s*(\w+)", + r"(?:uORB::)SubscriptionCallbackWorkItem\s+\w+\s*\{\s*this,\s*(ORB_ID)\((\w+)", + ] + self._subscriptions = Subscriptions( self._topic_blacklist, capture_cases_sub) - ] - special_cases_sub = [(a, re.compile(b), re.compile(c) if c is not None else None, re.compile(d)) - for a,b,c,d in special_cases_sub] + # note: the source-file-string is pre-processed to remove whitespace -- regexes should ignore whitespace + # note: the regexes should have at least 3 capture groups '()'; otherwise they break downstream code + capture_cases_pub = [r"(?:uORB::)Publication(?:Data|Multi|)\s*(<)(\w+)>", + r"orb_advertise(?:_multi|_queue|_multi_queue|)\s*\(\s*(ORB_ID)\s*\(\s*(\w+)", + r"orb_publish(?:_auto|)\s*\(\s*(ORB_ID)\s*\(\s*(\w+)", + r"(?:uORB::)Publication(?:Data|Multi|)\s*<\w+>\s+\w+\s*[\(\{](ORB_ID)\((\w+)" + ] + self._publications = Publications( self._topic_blacklist, capture_cases_pub) - self._subscriptions = PubSub(False, topic_blacklist, - [r"\borb_subscribe(_multi|)\b\(([^,)]+)"], - special_cases_sub) + # note: the source-file-string is pre-processed to remove whitespace -- regexes should ignore whitespace + # note: the regexes should have at least 3 capture groups '()'; otherwise they break downstream code + capture_cases_ambiguous = [ r"orb_copy\s*\(\s*(ORB_ID)\s*\(\s*(\w+)", + r"(?:uORB::)Subscription\s+\w+\s*(\[)\s*\w+\s*\]()", + r"(ORB_ID)\s*\(\s*(\w+)", + ] + self._ambiguities = Ambiguities( self._topic_blacklist, capture_cases_ambiguous) - - special_cases_pub = [ - ('replay', r'Replay\.cpp$', None, r'^sub\.orb_meta$'), - - ('uavcan', r'sensors/.*\.cpp$', None, r'^_orb_topic$'), - ] - special_cases_pub = [(a, re.compile(b), re.compile(c) if c is not None else None, re.compile(d)) - for a,b,c,d in special_cases_pub] - self._publications = PubSub(True, topic_blacklist, - [r"\borb_advertise(_multi|_queue|_multi_queue|)\b\(([^,)]+)", - r"\borb_publish_auto()\b\(([^,)]+)"], - special_cases_pub) - - - def _get_current_module(self): - if len(self._current_module) == 0: + def _get_current_scope(self): + if len(self._current_scope) == 0: return None - return self._current_module[-1] + return self._current_scope[-1] - def build(self, src_path_list, excluded_paths=[], use_topic_pubsub_union=True): + def build(self, src_path_list, path_blacklist=[], **kwargs): """ parse the source tree & extract pub/sub information. :param use_topic_pubsub_union: if true, use all topics that have a publisher or subscriber. If false, use only topics with at least one @@ -259,50 +288,114 @@ class Graph(object): fill in self._module_subsciptions & self._module_publications """ - self._subscriptions.reset() - self._publications.reset() + self._path_blacklist = [os.path.normpath(p) for p in path_blacklist] - self._excluded_paths = [os.path.normpath(p) for p in excluded_paths] for path in src_path_list: - self._build_recursive(path) + log.info("## Add src path: " + path ) + self._build_recursive(path, **kwargs) - # filter by whitelist - if len(self._module_whitelist) > 0: - self._subscriptions.filter_modules(self._module_whitelist) - self._publications.filter_modules(self._module_whitelist) + # Summarize the found counts: (all topics are defined in 'dependency' library) + log.info('### Summary: Total Scanned:') + log.info(' Libraries Count: '+str(len(self._found_libraries))) + log.info(' Modules Count: '+str(len(self._found_modules))) + log.info(' Warnings Count: '+str(len(self._warnings))) - # modules & topics sets - self._modules = set(list(self._publications.pubsubs.keys()) + - list(self._subscriptions.pubsubs.keys())) - print('number of modules: '+str(len(self._modules))) - self._topics = self._get_topics(use_topic_pubsub_union=use_topic_pubsub_union) - print('number of topics: '+str(len(self._topics))) + if kwargs['merge_depends']: + graph.merge_depends() + + # filter all scopes, topics into only the scopes + topics to output + self._generate_print_lists(use_topic_pubsub_union=kwargs['use_topic_pubsub_union'], merge_depends=kwargs['merge_depends']) + + # Summarize the found counts: + print(' ### Summary (in-scope):') + print(' Ambiguous Count: '+str(len(self._print_ambiguities))) + print(' Scope Count: '+str(len(self._print_scopes))) + print(' Topics Count: '+str(len(self._print_topics))) + print(' Warnings Count: '+str(len(self._warnings))) + + if 0 < len(self._warnings): + # print out the list of warning-sites: + log.info('## Warning Sites:') + for w in self._warnings: + # warnings tuple contains: (current_scope, file_name, line_number, line) + log.info(" -['{}']:{:<64s}:{} = {}".format(w[0].name, w[1].lstrip('/.'), w[2], w[3] )) # initialize colors - color_list = get_N_colors(len(self._topics), 0.7, 0.85) + color_list = get_N_colors(len(self._print_topics), 0.7, 0.85) self._topic_colors = {} - for i, topic in enumerate(self._topics): + for i, topic in enumerate(self._print_topics): self._topic_colors[topic] = color_list[i] - # validate that all special rules got used - self._subscriptions.check_if_match_found(self._all_modules) - self._publications.check_if_match_found(self._all_modules) + def _generate_print_lists(self, use_topic_pubsub_union, merge_depends): + """ generate the set of scopes (modules + libraries) and topics to print to output """ + subscribed_topics = set() + published_topics = set() + ambiguous_topics = set() - def _get_topics(self, use_topic_pubsub_union=True): - """ get the set of topics - """ - subscribed_topics = self._subscriptions.get_topics(self._modules) - published_topics = self._publications.get_topics(self._modules) + # gather all possible modules... + # all_scopes = self._found_libraries | self._found_modules # Python 3.9 or greater + all_scopes = { **self._found_libraries, **self._found_modules } # Python 3.5 or greater + + if 0 == len(self._scope_whitelist): + select_scopes = self._found_modules + else: + select_scopes = {} + for scope_name in self._scope_whitelist: + if scope_name in all_scopes: + select_scopes[scope_name] = all_scopes[scope_name] + if not isinstance(select_scopes, dict) or 0 == len(select_scopes): + raise TypeError("'select_scopes' should be a set!! aborting.") + + log.debug(f' >> Condensing found topics: scope -> total') + for name,scope in select_scopes.items(): + log.debug(f' @@ Scope: {name}') + + log.debug(f' ## Subs: {name}') + for topic in scope.subscriptions: + log.debug(f' - {topic}') + subscribed_topics.add(topic) + + log.debug(f' ## Pubs: {name}') + for topic in scope.publications: + log.debug(f' - {topic}') + published_topics.add(topic) + + scope.reduce_ambiguities() + + log.debug(f' ## Ambiguities: {name}') + for topic in scope.ambiguities: + log.debug(f' - {topic}') + ambiguous_topics.add(topic) + + # filter modules iff they have at least a subscription or a publication + scopes_with_topic = {} + for name,scope in select_scopes.items(): + if not scope.is_empty(): + scopes_with_topic[name] = scope + + self._print_ambiguities = ambiguous_topics if use_topic_pubsub_union: - return subscribed_topics | published_topics - return subscribed_topics & published_topics + self._print_topics = subscribed_topics | published_topics + self._print_scopes = scopes_with_topic + else: + self._print_topics = subscribed_topics & published_topics - def _build_recursive(self, path): + # cull scopes to only those that pub or sub to a topic that has both + intersect_scopes = {} + for name,scope in scopes_with_topic.items(): + all_scope_topics = scope.publications | scope.subscriptions + for topic in all_scope_topics: + if topic in self._print_topics: + intersect_scopes[scope.name] = scope + break + self._print_scopes = intersect_scopes - if os.path.normpath(path) in self._excluded_paths: - dbg_print('ignoring excluded path '+path) + + def _build_recursive(self, path, **kwargs): + if os.path.normpath(path) in self._path_blacklist: + log.debug('ignoring excluded path '+path) return entries = os.listdir(path) @@ -311,53 +404,89 @@ class Graph(object): cmake_file = 'CMakeLists.txt' new_module = False if cmake_file in entries: - new_module = self._extract_module_name(os.path.join(path, cmake_file)) + new_module = self._extract_build_information(os.path.join(path, cmake_file), **kwargs) # iterate directories recursively for entry in entries: file_name = os.path.join(path, entry) if os.path.isdir(file_name): - self._build_recursive(file_name) + self._build_recursive(file_name, **kwargs) # iterate source files - # Note: we could skip the entries if we're not in a module, but we don't - # so that we get appropriate error messages to know where we miss subs - # or pubs - for entry in entries: - file_name = os.path.join(path, entry) - if os.path.isfile(file_name): - _, ext = os.path.splitext(file_name) - if ext in ['.cpp', '.c', '.h', '.hpp']: - self._process_source_file(file_name) + # Note: Skip all entries if we're not in a module -- both finding known pubs/subs and emitting warnings + if (0 == len(self._scope_whitelist)) or (0 < len(self._current_scope)) and (self._current_scope[-1].name in self._scope_whitelist): + for entry in entries: + file_name = os.path.join(path, entry) + if os.path.isfile(file_name): + _, ext = os.path.splitext(file_name) + if ext in ['.cpp', '.c', '.h', '.hpp']: + self._process_source_file(file_name) if new_module: - self._current_module.pop() + self._current_scope.pop() - def _extract_module_name(self, file_name): + def _extract_build_information(self, file_name, **kwargs): """ extract the module name from a CMakeLists.txt file and store - in self._current_module if there is any """ + in self._current_scope if there is any """ + datafile = open(file_name) found_module_def = False + found_module_depends = False + found_library_def = False for line in datafile: if 'px4_add_module' in line: # must contain 'px4_add_module' found_module_def = True + elif 'px4_add_library' in line: # must contain 'px4_add_module' + tokens = line.split('(') + if 1 < len(tokens): + found_library_def = True + library_name = tokens[1].split()[0].strip().rstrip(')') + library_scope = LibraryScope(library_name) + self._current_scope.append(library_scope) + self._found_libraries[library_name] = library_scope + if self._in_scope(): + log.debug(' >> found library: ' + library_name) + + # we can return early because we have no further information to collect from libraries + return True + elif found_module_def and 'DEPENDS' in line.upper(): + found_module_depends = True + elif found_module_depends: + # two tabs is a *sketchy* heuristic -- spacing isn't guaranteed by cmake; + # ... but the hard-tabs *is* specified by PX4 coding standards, so it's likely to be consistent + if line.startswith('\t\t'): + depends = [dep.strip() for dep in line.split()] + for name in depends: + self._current_scope[-1].add_dependency(name) + if kwargs['merge_depends']: + if (0 < len(self._scope_whitelist)) and self._current_scope[-1].name in self._scope_whitelist: + # if we whitelist a module with dependencies, whitelist the dependencies, too + self._scope_whitelist.add(name) + + else: + found_module_depends = False ## done with the 'DEPENDS' section. words = line.split() # get the definition of MAIN if found_module_def and 'MAIN' in words and len(words) >= 2: - self._current_module.append(words[1]) - self._all_modules.add(words[1]) - dbg_print('Found module name: '+words[1]) - return True - return False + module_name = words[1] + module_scope = ModuleScope(module_name) + self._current_scope.append(module_scope) + self._found_modules[module_name] = module_scope + if self._in_scope(): + log.debug(' >> Found module name: ' + module_scope.name) + + return (found_library_def or found_module_def) def _process_source_file(self, file_name): """ extract information from a single source file """ + log.debug( " >> extracting topics from file: " + file_name ) + with codecs.open(file_name, 'r', 'utf-8') as f: try: content = f.read() @@ -366,10 +495,15 @@ class Graph(object): return - current_module = self._get_current_module() - if current_module == 'uorb_tests': # skip this + current_scope = self._get_current_scope() + + if current_scope is None: + return # ignore declarations outside of a declared module + elif current_scope.name in self._scope_blacklist: return - if current_module == 'uorb': + elif current_scope.name == 'uorb_tests': # skip this + return + elif current_scope.name == 'uorb': # search and validate the ORB_ID_VEHICLE_ATTITUDE_CONTROLS define matches = self._orb_id_vehicle_attitude_controls_re.findall(content) @@ -383,66 +517,71 @@ class Graph(object): return # skip uorb module for the rest + line_number = 0 + for line in content.splitlines(): + line_number += 1 + pub_topics = self._publications.match(line) + for each_topic in pub_topics: + current_scope.publications.add(each_topic) + if pub_topics: + continue - if content.lower().find('orb_') != -1: # approximative filter to quickly - # discard files we're not interested in - # (speedup the parsing) - src = self._comment_remover(content) - src = re.sub(self._whitespace_pattern, '', src) # remove all whitespace + sub_topics = self._subscriptions.match(line) + for each_topic in sub_topics: + current_scope.subscriptions.add(each_topic) + if sub_topics: + continue + ambi_topics = self._ambiguities.match(line) + for each_topic in ambi_topics: + current_scope.ambiguities.add(each_topic) + self._warnings.append((current_scope, file_name, line_number, line)) - # subscriptions - self._subscriptions.extract(file_name, src, current_module, - self._orb_id_vehicle_attitude_controls_topic) + def _in_scope(self, scope_name = None): + if 0 == len(self._scope_whitelist): + return True + elif 0 < len(self._current_scope): + if None is scope_name: + scope_name = self._current_scope[-1].name + if scope_name in self._scope_whitelist: + return True - # publications - self._publications.extract(file_name, src, current_module, - self._orb_id_vehicle_attitude_controls_topic) + return False - # TODO: handle Publication & Subscription template classes + def merge_depends(self): + for modname,module in self._found_modules.items(): + if self._in_scope(modname): + for depname in module.dependencies: + if depname in self._found_libraries: + dep = self._found_libraries[depname] + # copy topics from library to depending library + for topic in dep.publications: + module.publications.add(topic) + for topic in dep.subscriptions: + module.subscriptions.add(topic) + for topic in dep.ambiguities: + module.ambiguities - - - def _comment_remover(self, text): - """ remove C++ & C style comments. - Source: https://stackoverflow.com/a/241506 """ - def replacer(match): - s = match.group(0) - if s.startswith('/'): - return " " # note: a space and not an empty string - else: - return s - return re.sub(self._comment_remove_pattern, replacer, text) + # omit all libraries -- they've already been merged into their respective dependees + self._scope_whitelist = set([ str(s) for s in self._scope_whitelist if s not in self._found_libraries]) @property - def modules(self): + def output_scopes(self): """ get the set of all modules """ - return self._modules + return self._print_scopes @property - def topics(self): + def output_topics(self): """ get set set of all topics """ - return self._topics + return self._print_topics @property def topic_colors(self): """ get a dict of all topic colors with key=topic, value=color """ return self._topic_colors - @property - def module_subscriptions(self): - """ get a dict of all subscriptions with key=module name, value=set(topic names) """ - return self._subscriptions.pubsubs - - @property - def module_publications(self): - """ get a dict of all publications with key=module name, value=set(topic names) """ - return self._publications.pubsubs - - - class OutputGraphviz(object): """ write graph using Graphviz """ @@ -463,44 +602,40 @@ class OutputGraphviz(object): ratio = 1 # aspect ratio - modules = self._graph.modules - topics = self._graph.topics + output_topics = self._graph.output_topics + output_scopes = self._graph.output_scopes topic_colors = self._graph.topic_colors - module_publications = self._graph.module_publications - module_subscriptions = self._graph.module_subscriptions graph_attr={'splines': 'true', 'ratio': str(ratio), 'overlap': 'false'} graph_attr['sep'] = '"+15,15"' # increase spacing between nodes graph = Digraph(comment='autogenerated graph with graphviz using uorb_graph.py', engine=engine, graph_attr=graph_attr) - - # nodes - for module in modules: - graph.node('m_'+module, module, shape='box', fontcolor='#ffffff', + # scopes: modules + log.info(' > Writing scopes') + for name,_ in output_scopes.items(): + graph.node('m_'+name, name, shape='box', fontcolor='#ffffff', style='filled', color='#666666', fontsize='16') - for topic in topics: + log.info(' > Writing topics') + for topic in output_topics: graph.node('t_'+topic, topic, shape='ellipse', fontcolor='#ffffff', style='filled', color=topic_colors[topic]) - # edges + log.info(' > Writing publish edges') if show_publications: - for module in modules: - if module in module_publications: - for topic in module_publications[module]: - if topic in topics: - graph.edge('m_'+module, 't_'+topic, - color=topic_colors[topic], style='dashed') + for scope_name,scope in output_scopes.items(): + for topic in scope.publications: + if topic in output_topics: + graph.edge('m_'+scope_name, 't_'+topic, color=topic_colors[topic], style='dashed') + log.info(' > Writing subscribe edges') if show_subscriptions: - for module in modules: - if module in module_subscriptions: - for topic in module_subscriptions[module]: - if topic in topics: - graph.edge('t_'+topic, 'm_'+module, - color=topic_colors[topic]) + for scope_name,scope in output_scopes.items(): + for topic in scope.subscriptions: + if topic in output_topics: + graph.edge('t_'+topic, 'm_'+scope_name, color=topic_colors[topic]) graph.render(file_name, view=False) @@ -515,11 +650,9 @@ class OutputJSON(object): print('Writing to '+file_name) - modules = self._graph.modules - topics = self._graph.topics + output_topics = self._graph.output_topics + output_scopes = self._graph.output_scopes topic_colors = self._graph.topic_colors - module_publications = self._graph.module_publications - module_subscriptions = self._graph.module_subscriptions data = {} nodes = [] @@ -528,16 +661,16 @@ class OutputJSON(object): # (sort by length, such that short names are last. The rendering order # will be the same, so that in case of an overlap, the shorter label # will be on top) - for module in sorted(modules, key=len, reverse=True): + for scope_tuple in sorted(output_scopes.items(), key=(lambda st: len(st[0])), reverse=True): node = {} - node['id'] = 'm_'+module - node['name'] = module - node['type'] = 'module' + node['id'] = 'm_'+scope_tuple[0] + node['name'] = scope_tuple[0] + node['type'] = scope_tuple[1].typename node['color'] = '#666666' # TODO: add url to open module documentation? nodes.append(node) - for topic in sorted(topics, key=len, reverse=True): + for topic in sorted(output_topics, key=len, reverse=True): node = {} node['id'] = 't_'+topic node['name'] = topic @@ -552,28 +685,27 @@ class OutputJSON(object): edges = [] - # edges - for module in modules: - if module in module_publications: - for topic in module_publications[module]: - if topic in topics: - edge = {} - edge['source'] = 'm_'+module - edge['target'] = 't_'+topic - edge['color'] = topic_colors[topic] - edge['style'] = 'dashed' - edges.append(edge) - for module in modules: - if module in module_subscriptions: - for topic in module_subscriptions[module]: - if topic in topics: - edge = {} - edge['source'] = 't_'+topic - edge['target'] = 'm_'+module - edge['color'] = topic_colors[topic] - edge['style'] = 'normal' - edges.append(edge) + # edges + for name,scope in output_scopes.items(): + for topic in scope.publications: + if topic in output_topics: + edge = {} + edge['source'] = 'm_'+name + edge['target'] = 't_'+topic + edge['color'] = topic_colors[topic] + edge['style'] = 'dashed' + edges.append(edge) + + for name,scope in output_scopes.items(): + for topic in scope.subscriptions: + if topic in output_topics: + edge = {} + edge['source'] = 't_'+topic + edge['target'] = 'm_'+name + edge['color'] = topic_colors[topic] + edge['style'] = 'normal' + edges.append(edge) data['links'] = edges @@ -583,21 +715,48 @@ class OutputJSON(object): if "__main__" == __name__: + args = parser.parse_args() + + if 0 < args.verbosity: + if 1 == args.verbosity: + log.setLevel(logging.INFO) + print("set log level to INFO") + else: # implicity 1< + log.setLevel(logging.DEBUG) + print("set log level to DEBUG") + # ignore topics that are subscribed/published by many topics, but are not really # useful to show in the graph topic_blacklist = [ 'parameter_update', 'mavlink_log', 'log_message' ] print('Excluded topics: '+str(topic_blacklist)) - if len(args.modules) == 0: - module_whitelist = [] - else: - module_whitelist = [ m.strip() for m in args.modules.split(',')] + # ignore certain modules; for any reason + scope_blacklist = [] + if scope_blacklist: + print('Excluded Modules: '+str(topic_blacklist)) - graph = Graph(module_whitelist=module_whitelist, topic_blacklist=topic_blacklist) + if len(args.modules) == 0: + scope_whitelist = [] + else: + scope_whitelist = [ m.strip() for m in args.modules.split(',')] + scope_whitelist = set(scope_whitelist) + + graph = Graph(scope_whitelist=scope_whitelist, scope_blacklist=scope_blacklist, topic_blacklist=topic_blacklist) if len(args.src_path) == 0: args.src_path = ['src'] - graph.build(args.src_path, args.exclude_path, use_topic_pubsub_union=args.use_topic_union) + if not os.path.exists(args.src_path[0]): + print(f" !?could not find source directory: {args.src_path[0]}") + script_path = os.path.dirname(os.path.realpath(__file__)) + args.src_path[0] = os.path.realpath(os.path.join( script_path, '..', '..', 'src' )) + print(f" >> guessing at path: {args.src_path[0]}") + + for path in args.src_path: + if not os.path.exists(path): + print(f" !?could not find source directory: {path} -- please check path!") + print(f" Exiting.") + + graph.build(args.src_path, args.exclude_path, use_topic_pubsub_union=args.use_topic_union, merge_depends=args.merge_depends) if args.output == 'json': output_json = OutputJSON(graph) @@ -607,7 +766,7 @@ if "__main__" == __name__: try: from graphviz import Digraph except ImportError as e: - print("Failed to import graphviz: " + e) + print("Failed to import graphviz: " + str(e)) print("") print("You may need to install it with:") print(" pip3 install --user graphviz") @@ -618,5 +777,7 @@ if "__main__" == __name__: output_graphviz.write(args.file+'.fv', engine=engine) output_graphviz.write(args.file+'_subs.fv', show_publications=False, engine=engine) output_graphviz.write(args.file+'_pubs.fv', show_subscriptions=False, engine=engine) + elif args.output == 'none': + pass else: print('Error: unknown output format '+args.output) diff --git a/Tools/uorb_graph/create_from_startupscript.sh b/Tools/uorb_graph/create_from_startupscript.sh deleted file mode 100755 index c0bd7e70da..0000000000 --- a/Tools/uorb_graph/create_from_startupscript.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /bin/bash -# create the graph from a posix (e.g. SITL) startup script - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -startup_file="$SCRIPT_DIR"/../../posix-configs/SITL/init/ekf2/typhoon_h480 -[ -n "$1" ] && startup_file=$1 -# get the modules as comma-separated list -modules=$(cat "$startup_file"|cut -f1 -d' '|sort|uniq|tr '\n' ,) - -cd "$SCRIPT_DIR/../.." -"$SCRIPT_DIR"/create.py --src-path src -m "$modules" -f "$SCRIPT_DIR/graph_runtime_sitl" -