autotest: Provide format and unit/multiplier info for log messages

Definitions of each character are extracted from LogStructure.h
Data is extracted by parsing the logging definition struct
Also parse WriteMessage() calls for messages not defined in struct
Add support to separate log descriptions for messages with same field list
Compute derived unit from combination of format, unit and multiplier
For XML output the format and derived unit into new attributes
Add enumerations to the XML output (bitmasks were already done)
For MD,RST,HTML, output either derived unit, 'char[n]', 'bitmask' or 'enum'
Fix support for Blimp by adding it to the parse_enum.py lookup table
This commit is contained in:
Simon Hancock 2023-12-22 10:52:52 +00:00 committed by Andrew Tridgell
parent 01c8717308
commit c0a503d74d
5 changed files with 340 additions and 27 deletions

View File

@ -37,14 +37,24 @@ DO NOT EDIT
print(' <h2>%s</h2>' %
docco.description, file=self.fh)
print(' <table>', file=self.fh)
print(" <tr><th>FieldName</th><th>Description</th><tr>",
print(" <tr><th>FieldName</th><th>Units/Type</th><th>Description</th><tr>",
file=self.fh)
for f in docco.fields_order:
if "description" in docco.fields[f]:
fdesc = docco.fields[f]["description"]
else:
fdesc = ""
print(' <tr><td>%s</td><td>%s</td></tr>' % (f, fdesc),
if "units" in docco.fields[f] and docco.fields[f]["units"]!="":
ftypeunits = docco.fields[f]["units"]
elif "fmt" in docco.fields[f] and "char" in docco.fields[f]["fmt"]:
ftypeunits = docco.fields[f]["fmt"]
elif "bitmaskenum" in docco.fields[f]:
ftypeunits = "bitmask"
elif "valueenum" in docco.fields[f]:
ftypeunits = "enum"
else:
ftypeunits = ""
print(' <tr><td>%s</td><td>%s</td><td>%s</td></tr>' % (f, ftypeunits, fdesc),
file=self.fh)
print(' </table>', file=self.fh)

View File

@ -67,13 +67,23 @@ DO NOT EDIT
if docco.url is not None:
desc += f' ([Read more...]({docco.url}))'
print(desc, file=self.fh)
print("\n|FieldName|Description|\n|---|---|", file=self.fh)
print("\n|FieldName|Units/Type|Description|\n|---|---|---|", file=self.fh)
for f in docco.fields_order:
if "description" in docco.fields[f]:
fdesc = docco.fields[f]["description"]
else:
fdesc = ""
print(f'|{f}|{fdesc}|', file=self.fh)
if "units" in docco.fields[f] and docco.fields[f]["units"]!="":
ftypeunits = docco.fields[f]["units"]
elif "fmt" in docco.fields[f] and "char" in docco.fields[f]["fmt"]:
ftypeunits = docco.fields[f]["fmt"]
elif "bitmaskenum" in docco.fields[f]:
ftypeunits = "bitmask"
elif "valueenum" in docco.fields[f]:
ftypeunits = "enum"
else:
ftypeunits = ""
print(f'|{f}|{ftypeunits}|{fdesc}|', file=self.fh)
print("", file=self.fh)
self.stop()

View File

@ -37,17 +37,23 @@ This is a list of log messages which may be present in logs produced and stored
rows = []
for f in docco.fields_order:
# Populate the description column
if "description" in docco.fields[f]:
fdesc = docco.fields[f]["description"]
else:
fdesc = ""
# Initialise Type/Unit and check for enum/bitfields
ftypeunit = ""
fieldnamething = None
if "bitmaskenum" in docco.fields[f]:
fieldnamething = "bitmaskenum"
table_label = "Bitmask values"
ftypeunit = "bitmask"
elif "valueenum" in docco.fields[f]:
fieldnamething = "valueenum"
table_label = "Values"
ftypeunit = "enum"
# If an enum/bitmask is defined, build the table
if fieldnamething is not None:
enum_name = docco.fields[f][fieldnamething]
if enum_name not in enumerations:
@ -62,7 +68,13 @@ This is a list of log messages which may be present in logs produced and stored
comment = ""
bitmaskrows.append([enumentry.name, str(enumentry.value), comment])
fdesc += "\n%s:\n\n%s" % (table_label, self.tablify(bitmaskrows))
rows.append([f, fdesc])
# Populate the Type/Units column
if "units" in docco.fields[f] and docco.fields[f]["units"] != "":
ftypeunit = docco.fields[f]["units"]
elif "fmt" in docco.fields[f] and "char" in docco.fields[f]["fmt"]:
ftypeunit = docco.fields[f]["fmt"]
# Add the new row
rows.append([f, ftypeunit, fdesc])
print(self.tablify(rows), file=self.fh)

View File

@ -31,24 +31,37 @@ class XMLEmitter(emitter.Emitter):
xml_fields = etree.SubElement(xml_logformat, 'fields')
for f in docco.fields_order:
xml_field = etree.SubElement(xml_fields, 'field', name=f)
units = docco.fields[f]['units'] if "units" in docco.fields[f] else ""
fmt = docco.fields[f]['fmt'] if "fmt" in docco.fields[f] else ""
xml_field = etree.SubElement(xml_fields, 'field', name=f, units=units, type=fmt)
if "description" in docco.fields[f]:
xml_description2 = etree.SubElement(xml_field, 'description')
xml_description2.text = docco.fields[f]["description"]
# Check for enum/bitfield
fieldnamething = None
if "bitmaskenum" in docco.fields[f]:
enum_name = docco.fields[f]["bitmaskenum"]
fieldnamething = "bitmaskenum"
xmlenumtag = "bitmask"
xmlentrytag = "bit"
elif "valueenum" in docco.fields[f]:
fieldnamething = "valueenum"
xmlenumtag = "enum"
xmlentrytag = "element"
# If an enum/bitmask is defined, include this in the XML
if fieldnamething is not None:
enum_name = docco.fields[f][fieldnamething]
if enum_name not in enumerations:
raise Exception("Unknown enum (%s) (have %s)" %
(enum_name, "\n".join(sorted(enumerations.keys()))))
bit_mask = enumerations[enum_name]
xml_bitmask = etree.SubElement(xml_field, 'bitmask')
for bit in bit_mask.entries:
xml_bitmask_bit = etree.SubElement(xml_bitmask, 'bit', name=bit.name)
xml_bitmask_bit_value = etree.SubElement(xml_bitmask_bit, 'value')
xml_bitmask_bit_value.text = str(bit.value)
if bit.comment is not None:
xml_bitmask_bit_comment = etree.SubElement(xml_bitmask_bit, 'description')
xml_bitmask_bit_comment.text = bit.comment
enum = enumerations[enum_name]
xml_enum = etree.SubElement(xml_field, xmlenumtag, name=enum_name)
for entry in enum.entries:
xml_enum_entry = etree.SubElement(xml_enum, xmlentrytag, name=entry.name)
xml_enum_entry_value = etree.SubElement(xml_enum_entry, 'value')
xml_enum_entry_value.text = str(entry.value)
if entry.comment is not None:
xml_enum_entry_comment = etree.SubElement(xml_enum_entry, 'description')
xml_enum_entry_comment.text = entry.comment
if xml_fields.text is None and not len(xml_fields):
xml_fields.text = '\n' # add </param> on next line in case of empty element.
self.stop()

View File

@ -19,6 +19,7 @@ from enum_parse import EnumDocco
topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../')
topdir = os.path.realpath(topdir)
# Regular expressions for finding message information in code comments
re_loggermessage = re.compile(r"@LoggerMessage\s*:\s*([\w,]+)", re.MULTILINE)
re_commentline = re.compile(r"\s*//")
re_description = re.compile(r"\s*//\s*@Description\s*:\s*(.*)")
@ -29,9 +30,39 @@ re_fieldbitmaskenum = re.compile(r"\s*//\s*@FieldBitmaskEnum\s*:\s*(\w+):\s*(.*)
re_fieldvalueenum = re.compile(r"\s*//\s*@FieldValueEnum\s*:\s*(\w+):\s*(.*)")
re_vehicles = re.compile(r"\s*//\s*@Vehicles\s*:\s*(.*)")
# TODO: validate URLS actually return 200
# TODO: augment with other information from log definitions; type and units...
# Regular expressions for finding message definitions in structure format
re_start_messagedef = re.compile(r"^\s*{?\s*LOG_[A-Z0-9_]+_[MSGTA]+[A-Z0-9_]*\s*,")
re_deffield = r'[\s\\]*"?([\w\-#?%]+)"?\s*'
re_full_messagedef = re.compile(r'\s*LOG_\w+\s*,\s*\w+\([^)]+\)[\s\\]*,' + f'{re_deffield},{re_deffield},' + r'[\s\\]*"?([\w,]+)"?[\s\\]*,' + f'{re_deffield},{re_deffield}' , re.MULTILINE)
re_fmt_define = re.compile(r'#define\s+(\w+_FMT)\s+"([\w\-#?%]+)"')
re_units_define = re.compile(r'#define\s+(\w+_UNITS)\s+"([\w\-#?%]+)"')
re_mults_define = re.compile(r'#define\s+(\w+_MULTS)\s+"([\w\-#?%]+)"')
# Regular expressions for finding message definitions in Write calls
re_start_writecall = re.compile(r"\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(")
re_writefield = r'\s*"([\w\-#?%,]+)"\s*'
re_full_writecall = re.compile(r'\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(' + f'{re_writefield},{re_writefield},{re_writefield},({re_writefield},{re_writefield})?' , re.MULTILINE)
# Regular expression for extracting unit and multipliers from structure
re_units_mults_struct = re.compile(r"^\s*{\s*'([\w\-#?%!/])',"+r'\s*"?([\w\-#?%./]*)"?\s*}')
# TODO: validate URLS actually return 200
# Lookup tables are populated by reading LogStructure.h
log_fmt_lookup = {}
log_units_lookup = {}
log_mult_lookup = {}
# Lookup table to convert multiplier to prefix
mult_prefix_lookup = {
0: "",
1: "",
1e-1: "d", # deci-
1e-2: "c", # centi-
1e-3: "m", # milli-
1e-6: "μ", # micro-
1e-9: "n" # nano-
}
class LoggerDocco(object):
@ -53,20 +84,48 @@ class LoggerDocco(object):
emit_xml.XMLEmitter(),
emit_md.MDEmitter(),
]
self.msg_fmts_list = {}
self.msg_units_list = {}
self.msg_mults_list = {}
class Docco(object):
def __init__(self, name):
self.name = name
self.url = None
self.description = None
if isinstance(name,list):
self.description = [None] * len(name)
else:
self.description = None
self.fields = {}
self.fields_order = []
self.vehicles = None
self.bits_enums = []
def add_name(self, name):
# If self.name/description aren't lists, convert them
if isinstance(self.name,str):
self.name = [self.name]
self.description = [self.description]
# Replace any existing empty descriptions with empty strings
for i in range(0,len(self.description)):
if self.description[i] is None:
self.description[i] = ""
# Extend the name and description lists
if isinstance(name,list):
self.name.extend(name)
self.description.extend([None] * len(name))
else:
self.name.append(name)
self.description.append(None)
def set_description(self, desc):
self.description = desc
if isinstance(self.description,list):
for i in range(0,len(self.description)):
if self.description[i] is None:
self.description[i] = desc
else:
self.description = desc
def set_url(self, url):
self.url = url
@ -99,13 +158,107 @@ class LoggerDocco(object):
self.ensure_field(field)
self.fields[field]["bitmaskenum"] = bits
def set_fieldbitmaskvalue(self, field, bits):
def set_fieldvalueenum(self, field, bits):
self.ensure_field(field)
self.fields[field]["valueenum"] = bits
def set_vehicles(self, vehicles):
self.vehicles = vehicles
def set_fmts(self, fmts):
# If no fields are defined, do nothing
if len(self.fields_order)==0:
return
# Make sure lengths match up
if len(fmts) != len(self.fields_order):
print(f"Number of fmts don't match fields: msg={self.name} fmts={fmts} num_fields={len(self.fields_order)}")
return
# Loop through the list
for idx in range(0,len(fmts)):
if fmts[idx] in log_fmt_lookup:
self.fields[self.fields_order[idx]]["fmt"] = log_fmt_lookup[fmts[idx]]
else:
print(f"Unrecognised format character: {fmts[idx]} in message {self.name}")
def set_units(self, units, mults):
# If no fields are defined, do nothing
if len(self.fields_order)==0:
return
# Make sure lengths match up
if len(units) != len(self.fields_order) or len(units) != len(mults):
print(f"Number of units/mults/fields don't match: msg={self.name} units={units} mults={mults} num_fields={len(self.fields_order)}")
return
# Loop through the list
for idx in range(0,len(units)):
# Get the index into fields from field_order
f = self.fields_order[idx]
# Convert unit char to base unit
if units[idx] in log_units_lookup:
baseunit = log_units_lookup[units[idx]]
else:
print(f"Unrecognised units character: {units[idx]} in message {self.name}")
continue
# Do nothing if this field has no unit defined
if baseunit == "":
continue
# Convert mult char to value
if mults[idx] in log_mult_lookup:
mult = log_mult_lookup[mults[idx]]
mult_num = float(mult)
else:
print(f"Unrecognised multiplier character: {mults[idx]} in message {self.name}")
continue
# Check if the defined format for this field contains its own multiplier
# If so, the presented value will be the base-unit directly
if 'fmt' in self.fields[f] and self.fields[f]['fmt'].endswith("* 100"):
self.fields[f]["units"] = baseunit
elif 'fmt' in self.fields[f] and "latitude/longitude" in self.fields[f]['fmt']:
self.fields[f]["units"] = baseunit
# Check if we have a defined prefix for this multiplier
elif mult_num in mult_prefix_lookup:
self.fields[f]["units"] = f"{mult_prefix_lookup[mult_num]}{baseunit}"
# If all else fails, set the unit as the multipler and base unit together
else:
self.fields[f]["units"] = f"{mult} {baseunit}"
def populate_lookups(self):
# Initialise the lookup tables
# Read the contents of the LogStructure.h file
structfile = os.path.join(topdir, "libraries", "AP_Logger", "LogStructure.h")
with open(structfile) as f:
lines = f.readlines()
f.close()
# Initialise current section to none
section = "none"
# Loop through the lines in the file
for line in lines:
# Look for the start of fmt/unit/mult info
if line.startswith("Format characters"):
section = "fmt"
elif line.startswith("const struct UnitStructure"):
section = "units"
elif line.startswith("const struct MultiplierStructure"):
section = "mult"
# Read formats from code comment, e.g.:
# b : int8_t
elif section == "fmt":
if "*/" in line:
section = "none"
else:
parts = line.split(":")
log_fmt_lookup[parts[0].strip()] = parts[1].strip()
# Read units or multipliers from C struct definition, e.g.:
# { '2', 1e2 }, or { 'J', "W.s" },
elif section != "none":
if "};" in line:
section = "none"
else:
u = re_units_mults_struct.search(line)
if u is not None and section == "units":
log_units_lookup[u.group(1)] = u.group(2)
if u is not None and section == "mult":
log_mult_lookup[u.group(1)] = u.group(2)
def search_for_files(self, dirs_to_search):
_next = []
for _dir in dirs_to_search:
@ -122,17 +275,86 @@ class LoggerDocco(object):
if len(_next):
self.search_for_files(_next)
def parse_messagedef(self,messagedef):
# Merge concatinated strings and remove comments
messagedef = re.sub(r'"\s+"', '', messagedef)
messagedef = re.sub(r'//[^\n]*', '', messagedef)
# Extract details from a structure definition
d = re_full_messagedef.search(messagedef)
if d is not None:
self.msg_fmts_list[d.group(1)] = d.group(2)
self.msg_units_list[d.group(1)] = d.group(4)
self.msg_mults_list[d.group(1)] = d.group(5)
return
# Extract details from a WriteStreaming call
d = re_full_writecall.search(messagedef)
if d is not None:
if d.group(1) in self.msg_fmts_list:
return
if d.group(5) is None:
self.msg_fmts_list[d.group(1)] = d.group(3)
else:
self.msg_fmts_list[d.group(1)] = d.group(6)
self.msg_units_list[d.group(1)] = d.group(3)
self.msg_mults_list[d.group(1)] = d.group(5)
return
# Didn't parse
#print(f"Unable to parse: {messagedef}")
def search_messagedef_start(self,line,prevmessagedef=""):
# Look for the start of a structure definition
d = re_start_messagedef.search(line)
if d is not None:
messagedef = line
if "}" in line:
self.parse_messagedef(messagedef)
return ""
else:
return messagedef
# Look for a new call to WriteStreaming
d = re_start_writecall.search(line)
if d is not None:
messagedef = line
if ";" in line:
self.parse_messagedef(messagedef)
return ""
else:
return messagedef
# If we didn't find a new one, continue with any previous state
return prevmessagedef
def parse_file(self, filepath):
with open(filepath) as f:
# print("Opened (%s)" % filepath)
lines = f.readlines()
f.close()
state_outside = "outside"
state_inside = "inside"
state = state_outside
docco = None
state_outside = "outside"
state_inside = "inside"
messagedef = ""
state = state_outside
docco = None
for line in lines:
if messagedef:
messagedef = messagedef + line
if "}" in line or ";" in line:
self.parse_messagedef(messagedef)
messagedef = ""
if state == state_outside:
# Check for start of a message definition
messagedef = self.search_messagedef_start(line,messagedef)
# Check for fmt/unit/mult #define
u = re_fmt_define.search(line)
if u is not None:
self.msg_fmts_list[u.group(1)] = u.group(2)
u = re_units_define.search(line)
if u is not None:
self.msg_units_list[u.group(1)] = u.group(2)
u = re_mults_define.search(line)
if u is not None:
self.msg_mults_list[u.group(1)] = u.group(2)
# Check for the @LoggerMessage tag indicating the start of the docco block
m = re_loggermessage.search(line)
if m is None:
continue
@ -142,11 +364,22 @@ class LoggerDocco(object):
state = state_inside
docco = LoggerDocco.Docco(name)
elif state == state_inside:
# If this line is not a comment, then this is the end of the docco block
if not re_commentline.match(line):
state = state_outside
if docco.vehicles is None or self.vehicle in docco.vehicles:
self.finalise_docco(docco)
messagedef = self.search_messagedef_start(line)
continue
# Check for an multiple @LoggerMessage lines in this docco block
m = re_loggermessage.search(line)
if m is not None:
name = m.group(1)
if "," in name:
name = name.split(",")
docco.add_name(name)
continue
# Find and extract data from the various docco fields
m = re_description.match(line)
if m is not None:
docco.set_description(m.group(1))
@ -169,7 +402,7 @@ class LoggerDocco(object):
continue
m = re_fieldvalueenum.match(line)
if m is not None:
docco.set_fieldbitmaskvalue(m.group(1), m.group(2))
docco.set_fieldvalueenum(m.group(1), m.group(2))
continue
m = re_vehicles.match(line)
if m is not None:
@ -187,14 +420,48 @@ class LoggerDocco(object):
new_doccos = []
for docco in self.doccos:
if isinstance(docco.name, list):
for name in docco.name:
for name,desc in zip(docco.name, docco.description):
tmpdocco = copy.copy(docco)
tmpdocco.name = name
tmpdocco.description = desc
new_doccos.append(tmpdocco)
else:
new_doccos.append(docco)
new_doccos = sorted(new_doccos, key=lambda x : x.name)
# Try to attach the formats/units/multipliers
for docco in new_doccos:
# Apply the Formats to the docco
if docco.name in self.msg_fmts_list:
if "FMT" in self.msg_fmts_list[docco.name]:
if self.msg_fmts_list[docco.name] in self.msg_fmts_list:
docco.set_fmts(self.msg_fmts_list[self.msg_fmts_list[docco.name]])
else:
docco.set_fmts(self.msg_fmts_list[docco.name])
else:
print(f"No formats found for message {docco.name}")
# Get the Units
units = None
if docco.name in self.msg_units_list:
if "UNITS" in self.msg_units_list[docco.name]:
if self.msg_units_list[docco.name] in self.msg_units_list:
units = self.msg_units_list[self.msg_units_list[docco.name]]
else:
units = self.msg_units_list[docco.name]
# Get the Multipliers
mults = None
if docco.name in self.msg_mults_list:
if "MULTS" in self.msg_mults_list[docco.name]:
if self.msg_mults_list[docco.name] in self.msg_mults_list:
mults = self.msg_mults_list[self.msg_mults_list[docco.name]]
else:
mults = self.msg_mults_list[docco.name]
# Apply the units/mults to the docco
if units is not None and mults is not None:
docco.set_units(units,mults)
elif units is not None or mults is not None:
print(f"Cannot find matching units/mults for message {docco.name}")
enums_by_name = {}
for enum in self.enumerations:
enums_by_name[enum.name] = enum
@ -202,6 +469,7 @@ class LoggerDocco(object):
emitter.emit(new_doccos, enums_by_name)
def run(self):
self.populate_lookups()
self.enumerations = enum_parse.EnumDocco(self.vehicle).get_enumerations()
self.files = []
self.search_for_files([self.vehicle_map[self.vehicle], "libraries"])