diff --git a/Tools/autotest/logger_metadata/emit_html.py b/Tools/autotest/logger_metadata/emit_html.py index 08f16160f0..506dfbc104 100644 --- a/Tools/autotest/logger_metadata/emit_html.py +++ b/Tools/autotest/logger_metadata/emit_html.py @@ -37,14 +37,24 @@ DO NOT EDIT print('

%s

' % docco.description, file=self.fh) print(' ', file=self.fh) - print(" ", + print(" ", file=self.fh) for f in docco.fields_order: if "description" in docco.fields[f]: fdesc = docco.fields[f]["description"] else: fdesc = "" - print(' ' % (f, fdesc), + if "units" in docco.fields[f] and docco.fields[f]["units"]!="": + ftypeunits = docco.fields[f]["units"] + elif "fmt" in docco.fields[f] and "char" in docco.fields[f]["fmt"]: + ftypeunits = docco.fields[f]["fmt"] + elif "bitmaskenum" in docco.fields[f]: + ftypeunits = "bitmask" + elif "valueenum" in docco.fields[f]: + ftypeunits = "enum" + else: + ftypeunits = "" + print(' ' % (f, ftypeunits, fdesc), file=self.fh) print('
FieldNameDescription
FieldNameUnits/TypeDescription
%s%s
%s%s%s
', file=self.fh) diff --git a/Tools/autotest/logger_metadata/emit_md.py b/Tools/autotest/logger_metadata/emit_md.py index d6d1cb7877..f9ae44492b 100644 --- a/Tools/autotest/logger_metadata/emit_md.py +++ b/Tools/autotest/logger_metadata/emit_md.py @@ -67,13 +67,23 @@ DO NOT EDIT if docco.url is not None: desc += f' ([Read more...]({docco.url}))' print(desc, file=self.fh) - print("\n|FieldName|Description|\n|---|---|", file=self.fh) + print("\n|FieldName|Units/Type|Description|\n|---|---|---|", file=self.fh) for f in docco.fields_order: if "description" in docco.fields[f]: fdesc = docco.fields[f]["description"] else: fdesc = "" - print(f'|{f}|{fdesc}|', file=self.fh) + if "units" in docco.fields[f] and docco.fields[f]["units"]!="": + ftypeunits = docco.fields[f]["units"] + elif "fmt" in docco.fields[f] and "char" in docco.fields[f]["fmt"]: + ftypeunits = docco.fields[f]["fmt"] + elif "bitmaskenum" in docco.fields[f]: + ftypeunits = "bitmask" + elif "valueenum" in docco.fields[f]: + ftypeunits = "enum" + else: + ftypeunits = "" + print(f'|{f}|{ftypeunits}|{fdesc}|', file=self.fh) print("", file=self.fh) self.stop() diff --git a/Tools/autotest/logger_metadata/emit_rst.py b/Tools/autotest/logger_metadata/emit_rst.py index 6b48ca5b31..7e513a0cc8 100644 --- a/Tools/autotest/logger_metadata/emit_rst.py +++ b/Tools/autotest/logger_metadata/emit_rst.py @@ -37,17 +37,23 @@ This is a list of log messages which may be present in logs produced and stored rows = [] for f in docco.fields_order: + # Populate the description column if "description" in docco.fields[f]: fdesc = docco.fields[f]["description"] else: fdesc = "" + # Initialise Type/Unit and check for enum/bitfields + ftypeunit = "" fieldnamething = None if "bitmaskenum" in docco.fields[f]: fieldnamething = "bitmaskenum" table_label = "Bitmask values" + ftypeunit = "bitmask" elif "valueenum" in docco.fields[f]: fieldnamething = "valueenum" table_label = "Values" + ftypeunit = "enum" + # If an enum/bitmask is defined, build the table if fieldnamething is not None: enum_name = docco.fields[f][fieldnamething] if enum_name not in enumerations: @@ -62,7 +68,13 @@ This is a list of log messages which may be present in logs produced and stored comment = "" bitmaskrows.append([enumentry.name, str(enumentry.value), comment]) fdesc += "\n%s:\n\n%s" % (table_label, self.tablify(bitmaskrows)) - rows.append([f, fdesc]) + # Populate the Type/Units column + if "units" in docco.fields[f] and docco.fields[f]["units"] != "": + ftypeunit = docco.fields[f]["units"] + elif "fmt" in docco.fields[f] and "char" in docco.fields[f]["fmt"]: + ftypeunit = docco.fields[f]["fmt"] + # Add the new row + rows.append([f, ftypeunit, fdesc]) print(self.tablify(rows), file=self.fh) diff --git a/Tools/autotest/logger_metadata/emit_xml.py b/Tools/autotest/logger_metadata/emit_xml.py index c0fc39b0c0..05073364c9 100644 --- a/Tools/autotest/logger_metadata/emit_xml.py +++ b/Tools/autotest/logger_metadata/emit_xml.py @@ -31,24 +31,37 @@ class XMLEmitter(emitter.Emitter): xml_fields = etree.SubElement(xml_logformat, 'fields') for f in docco.fields_order: - xml_field = etree.SubElement(xml_fields, 'field', name=f) + units = docco.fields[f]['units'] if "units" in docco.fields[f] else "" + fmt = docco.fields[f]['fmt'] if "fmt" in docco.fields[f] else "" + xml_field = etree.SubElement(xml_fields, 'field', name=f, units=units, type=fmt) if "description" in docco.fields[f]: xml_description2 = etree.SubElement(xml_field, 'description') xml_description2.text = docco.fields[f]["description"] + # Check for enum/bitfield + fieldnamething = None if "bitmaskenum" in docco.fields[f]: - enum_name = docco.fields[f]["bitmaskenum"] + fieldnamething = "bitmaskenum" + xmlenumtag = "bitmask" + xmlentrytag = "bit" + elif "valueenum" in docco.fields[f]: + fieldnamething = "valueenum" + xmlenumtag = "enum" + xmlentrytag = "element" + # If an enum/bitmask is defined, include this in the XML + if fieldnamething is not None: + enum_name = docco.fields[f][fieldnamething] if enum_name not in enumerations: raise Exception("Unknown enum (%s) (have %s)" % (enum_name, "\n".join(sorted(enumerations.keys())))) - bit_mask = enumerations[enum_name] - xml_bitmask = etree.SubElement(xml_field, 'bitmask') - for bit in bit_mask.entries: - xml_bitmask_bit = etree.SubElement(xml_bitmask, 'bit', name=bit.name) - xml_bitmask_bit_value = etree.SubElement(xml_bitmask_bit, 'value') - xml_bitmask_bit_value.text = str(bit.value) - if bit.comment is not None: - xml_bitmask_bit_comment = etree.SubElement(xml_bitmask_bit, 'description') - xml_bitmask_bit_comment.text = bit.comment + enum = enumerations[enum_name] + xml_enum = etree.SubElement(xml_field, xmlenumtag, name=enum_name) + for entry in enum.entries: + xml_enum_entry = etree.SubElement(xml_enum, xmlentrytag, name=entry.name) + xml_enum_entry_value = etree.SubElement(xml_enum_entry, 'value') + xml_enum_entry_value.text = str(entry.value) + if entry.comment is not None: + xml_enum_entry_comment = etree.SubElement(xml_enum_entry, 'description') + xml_enum_entry_comment.text = entry.comment if xml_fields.text is None and not len(xml_fields): xml_fields.text = '\n' # add on next line in case of empty element. self.stop() diff --git a/Tools/autotest/logger_metadata/parse.py b/Tools/autotest/logger_metadata/parse.py index ea58198ea2..a3ce3221c6 100755 --- a/Tools/autotest/logger_metadata/parse.py +++ b/Tools/autotest/logger_metadata/parse.py @@ -19,6 +19,7 @@ from enum_parse import EnumDocco topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../') topdir = os.path.realpath(topdir) +# Regular expressions for finding message information in code comments re_loggermessage = re.compile(r"@LoggerMessage\s*:\s*([\w,]+)", re.MULTILINE) re_commentline = re.compile(r"\s*//") re_description = re.compile(r"\s*//\s*@Description\s*:\s*(.*)") @@ -29,9 +30,39 @@ re_fieldbitmaskenum = re.compile(r"\s*//\s*@FieldBitmaskEnum\s*:\s*(\w+):\s*(.*) re_fieldvalueenum = re.compile(r"\s*//\s*@FieldValueEnum\s*:\s*(\w+):\s*(.*)") re_vehicles = re.compile(r"\s*//\s*@Vehicles\s*:\s*(.*)") -# TODO: validate URLS actually return 200 -# TODO: augment with other information from log definitions; type and units... +# Regular expressions for finding message definitions in structure format +re_start_messagedef = re.compile(r"^\s*{?\s*LOG_[A-Z0-9_]+_[MSGTA]+[A-Z0-9_]*\s*,") +re_deffield = r'[\s\\]*"?([\w\-#?%]+)"?\s*' +re_full_messagedef = re.compile(r'\s*LOG_\w+\s*,\s*\w+\([^)]+\)[\s\\]*,' + f'{re_deffield},{re_deffield},' + r'[\s\\]*"?([\w,]+)"?[\s\\]*,' + f'{re_deffield},{re_deffield}' , re.MULTILINE) +re_fmt_define = re.compile(r'#define\s+(\w+_FMT)\s+"([\w\-#?%]+)"') +re_units_define = re.compile(r'#define\s+(\w+_UNITS)\s+"([\w\-#?%]+)"') +re_mults_define = re.compile(r'#define\s+(\w+_MULTS)\s+"([\w\-#?%]+)"') +# Regular expressions for finding message definitions in Write calls +re_start_writecall = re.compile(r"\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(") +re_writefield = r'\s*"([\w\-#?%,]+)"\s*' +re_full_writecall = re.compile(r'\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(' + f'{re_writefield},{re_writefield},{re_writefield},({re_writefield},{re_writefield})?' , re.MULTILINE) + +# Regular expression for extracting unit and multipliers from structure +re_units_mults_struct = re.compile(r"^\s*{\s*'([\w\-#?%!/])',"+r'\s*"?([\w\-#?%./]*)"?\s*}') + +# TODO: validate URLS actually return 200 + +# Lookup tables are populated by reading LogStructure.h +log_fmt_lookup = {} +log_units_lookup = {} +log_mult_lookup = {} + +# Lookup table to convert multiplier to prefix +mult_prefix_lookup = { + 0: "", + 1: "", + 1e-1: "d", # deci- + 1e-2: "c", # centi- + 1e-3: "m", # milli- + 1e-6: "μ", # micro- + 1e-9: "n" # nano- +} class LoggerDocco(object): @@ -53,20 +84,48 @@ class LoggerDocco(object): emit_xml.XMLEmitter(), emit_md.MDEmitter(), ] + self.msg_fmts_list = {} + self.msg_units_list = {} + self.msg_mults_list = {} class Docco(object): def __init__(self, name): self.name = name self.url = None - self.description = None + if isinstance(name,list): + self.description = [None] * len(name) + else: + self.description = None self.fields = {} self.fields_order = [] self.vehicles = None self.bits_enums = [] + def add_name(self, name): + # If self.name/description aren't lists, convert them + if isinstance(self.name,str): + self.name = [self.name] + self.description = [self.description] + # Replace any existing empty descriptions with empty strings + for i in range(0,len(self.description)): + if self.description[i] is None: + self.description[i] = "" + # Extend the name and description lists + if isinstance(name,list): + self.name.extend(name) + self.description.extend([None] * len(name)) + else: + self.name.append(name) + self.description.append(None) + def set_description(self, desc): - self.description = desc + if isinstance(self.description,list): + for i in range(0,len(self.description)): + if self.description[i] is None: + self.description[i] = desc + else: + self.description = desc def set_url(self, url): self.url = url @@ -99,13 +158,107 @@ class LoggerDocco(object): self.ensure_field(field) self.fields[field]["bitmaskenum"] = bits - def set_fieldbitmaskvalue(self, field, bits): + def set_fieldvalueenum(self, field, bits): self.ensure_field(field) self.fields[field]["valueenum"] = bits def set_vehicles(self, vehicles): self.vehicles = vehicles + def set_fmts(self, fmts): + # If no fields are defined, do nothing + if len(self.fields_order)==0: + return + # Make sure lengths match up + if len(fmts) != len(self.fields_order): + print(f"Number of fmts don't match fields: msg={self.name} fmts={fmts} num_fields={len(self.fields_order)}") + return + # Loop through the list + for idx in range(0,len(fmts)): + if fmts[idx] in log_fmt_lookup: + self.fields[self.fields_order[idx]]["fmt"] = log_fmt_lookup[fmts[idx]] + else: + print(f"Unrecognised format character: {fmts[idx]} in message {self.name}") + + def set_units(self, units, mults): + # If no fields are defined, do nothing + if len(self.fields_order)==0: + return + # Make sure lengths match up + if len(units) != len(self.fields_order) or len(units) != len(mults): + print(f"Number of units/mults/fields don't match: msg={self.name} units={units} mults={mults} num_fields={len(self.fields_order)}") + return + # Loop through the list + for idx in range(0,len(units)): + # Get the index into fields from field_order + f = self.fields_order[idx] + # Convert unit char to base unit + if units[idx] in log_units_lookup: + baseunit = log_units_lookup[units[idx]] + else: + print(f"Unrecognised units character: {units[idx]} in message {self.name}") + continue + # Do nothing if this field has no unit defined + if baseunit == "": + continue + # Convert mult char to value + if mults[idx] in log_mult_lookup: + mult = log_mult_lookup[mults[idx]] + mult_num = float(mult) + else: + print(f"Unrecognised multiplier character: {mults[idx]} in message {self.name}") + continue + # Check if the defined format for this field contains its own multiplier + # If so, the presented value will be the base-unit directly + if 'fmt' in self.fields[f] and self.fields[f]['fmt'].endswith("* 100"): + self.fields[f]["units"] = baseunit + elif 'fmt' in self.fields[f] and "latitude/longitude" in self.fields[f]['fmt']: + self.fields[f]["units"] = baseunit + # Check if we have a defined prefix for this multiplier + elif mult_num in mult_prefix_lookup: + self.fields[f]["units"] = f"{mult_prefix_lookup[mult_num]}{baseunit}" + # If all else fails, set the unit as the multipler and base unit together + else: + self.fields[f]["units"] = f"{mult} {baseunit}" + + def populate_lookups(self): + # Initialise the lookup tables + # Read the contents of the LogStructure.h file + structfile = os.path.join(topdir, "libraries", "AP_Logger", "LogStructure.h") + with open(structfile) as f: + lines = f.readlines() + f.close() + # Initialise current section to none + section = "none" + # Loop through the lines in the file + for line in lines: + # Look for the start of fmt/unit/mult info + if line.startswith("Format characters"): + section = "fmt" + elif line.startswith("const struct UnitStructure"): + section = "units" + elif line.startswith("const struct MultiplierStructure"): + section = "mult" + # Read formats from code comment, e.g.: + # b : int8_t + elif section == "fmt": + if "*/" in line: + section = "none" + else: + parts = line.split(":") + log_fmt_lookup[parts[0].strip()] = parts[1].strip() + # Read units or multipliers from C struct definition, e.g.: + # { '2', 1e2 }, or { 'J', "W.s" }, + elif section != "none": + if "};" in line: + section = "none" + else: + u = re_units_mults_struct.search(line) + if u is not None and section == "units": + log_units_lookup[u.group(1)] = u.group(2) + if u is not None and section == "mult": + log_mult_lookup[u.group(1)] = u.group(2) + def search_for_files(self, dirs_to_search): _next = [] for _dir in dirs_to_search: @@ -122,17 +275,86 @@ class LoggerDocco(object): if len(_next): self.search_for_files(_next) + def parse_messagedef(self,messagedef): + # Merge concatinated strings and remove comments + messagedef = re.sub(r'"\s+"', '', messagedef) + messagedef = re.sub(r'//[^\n]*', '', messagedef) + # Extract details from a structure definition + d = re_full_messagedef.search(messagedef) + if d is not None: + self.msg_fmts_list[d.group(1)] = d.group(2) + self.msg_units_list[d.group(1)] = d.group(4) + self.msg_mults_list[d.group(1)] = d.group(5) + return + # Extract details from a WriteStreaming call + d = re_full_writecall.search(messagedef) + if d is not None: + if d.group(1) in self.msg_fmts_list: + return + if d.group(5) is None: + self.msg_fmts_list[d.group(1)] = d.group(3) + else: + self.msg_fmts_list[d.group(1)] = d.group(6) + self.msg_units_list[d.group(1)] = d.group(3) + self.msg_mults_list[d.group(1)] = d.group(5) + return + # Didn't parse + #print(f"Unable to parse: {messagedef}") + + def search_messagedef_start(self,line,prevmessagedef=""): + # Look for the start of a structure definition + d = re_start_messagedef.search(line) + if d is not None: + messagedef = line + if "}" in line: + self.parse_messagedef(messagedef) + return "" + else: + return messagedef + # Look for a new call to WriteStreaming + d = re_start_writecall.search(line) + if d is not None: + messagedef = line + if ";" in line: + self.parse_messagedef(messagedef) + return "" + else: + return messagedef + # If we didn't find a new one, continue with any previous state + return prevmessagedef + def parse_file(self, filepath): with open(filepath) as f: # print("Opened (%s)" % filepath) lines = f.readlines() f.close() - state_outside = "outside" - state_inside = "inside" - state = state_outside - docco = None + state_outside = "outside" + state_inside = "inside" + messagedef = "" + state = state_outside + docco = None for line in lines: + if messagedef: + messagedef = messagedef + line + if "}" in line or ";" in line: + self.parse_messagedef(messagedef) + messagedef = "" if state == state_outside: + # Check for start of a message definition + messagedef = self.search_messagedef_start(line,messagedef) + + # Check for fmt/unit/mult #define + u = re_fmt_define.search(line) + if u is not None: + self.msg_fmts_list[u.group(1)] = u.group(2) + u = re_units_define.search(line) + if u is not None: + self.msg_units_list[u.group(1)] = u.group(2) + u = re_mults_define.search(line) + if u is not None: + self.msg_mults_list[u.group(1)] = u.group(2) + + # Check for the @LoggerMessage tag indicating the start of the docco block m = re_loggermessage.search(line) if m is None: continue @@ -142,11 +364,22 @@ class LoggerDocco(object): state = state_inside docco = LoggerDocco.Docco(name) elif state == state_inside: + # If this line is not a comment, then this is the end of the docco block if not re_commentline.match(line): state = state_outside if docco.vehicles is None or self.vehicle in docco.vehicles: self.finalise_docco(docco) + messagedef = self.search_messagedef_start(line) continue + # Check for an multiple @LoggerMessage lines in this docco block + m = re_loggermessage.search(line) + if m is not None: + name = m.group(1) + if "," in name: + name = name.split(",") + docco.add_name(name) + continue + # Find and extract data from the various docco fields m = re_description.match(line) if m is not None: docco.set_description(m.group(1)) @@ -169,7 +402,7 @@ class LoggerDocco(object): continue m = re_fieldvalueenum.match(line) if m is not None: - docco.set_fieldbitmaskvalue(m.group(1), m.group(2)) + docco.set_fieldvalueenum(m.group(1), m.group(2)) continue m = re_vehicles.match(line) if m is not None: @@ -187,14 +420,48 @@ class LoggerDocco(object): new_doccos = [] for docco in self.doccos: if isinstance(docco.name, list): - for name in docco.name: + for name,desc in zip(docco.name, docco.description): tmpdocco = copy.copy(docco) tmpdocco.name = name + tmpdocco.description = desc new_doccos.append(tmpdocco) else: new_doccos.append(docco) new_doccos = sorted(new_doccos, key=lambda x : x.name) + # Try to attach the formats/units/multipliers + for docco in new_doccos: + # Apply the Formats to the docco + if docco.name in self.msg_fmts_list: + if "FMT" in self.msg_fmts_list[docco.name]: + if self.msg_fmts_list[docco.name] in self.msg_fmts_list: + docco.set_fmts(self.msg_fmts_list[self.msg_fmts_list[docco.name]]) + else: + docco.set_fmts(self.msg_fmts_list[docco.name]) + else: + print(f"No formats found for message {docco.name}") + # Get the Units + units = None + if docco.name in self.msg_units_list: + if "UNITS" in self.msg_units_list[docco.name]: + if self.msg_units_list[docco.name] in self.msg_units_list: + units = self.msg_units_list[self.msg_units_list[docco.name]] + else: + units = self.msg_units_list[docco.name] + # Get the Multipliers + mults = None + if docco.name in self.msg_mults_list: + if "MULTS" in self.msg_mults_list[docco.name]: + if self.msg_mults_list[docco.name] in self.msg_mults_list: + mults = self.msg_mults_list[self.msg_mults_list[docco.name]] + else: + mults = self.msg_mults_list[docco.name] + # Apply the units/mults to the docco + if units is not None and mults is not None: + docco.set_units(units,mults) + elif units is not None or mults is not None: + print(f"Cannot find matching units/mults for message {docco.name}") + enums_by_name = {} for enum in self.enumerations: enums_by_name[enum.name] = enum @@ -202,6 +469,7 @@ class LoggerDocco(object): emitter.emit(new_doccos, enums_by_name) def run(self): + self.populate_lookups() self.enumerations = enum_parse.EnumDocco(self.vehicle).get_enumerations() self.files = [] self.search_for_files([self.vehicle_map[self.vehicle], "libraries"])