Mirror
/
ardupilot
mirror of https://github.com/ArduPilot/ardupilot
5
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

autotest: logger_metadata: flake8 cleanliness

This commit is contained in:
Peter Barker 2024-10-20 10:47:13 +11:00 committed by Andrew Tridgell
parent de0f3ddebe
commit 756df1cfc0
2 changed files with 61 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
Tools/autotest/logger_metadata/enum_parse.py
View File

@ -1,5 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
'''
AP_FLAKE8_CLEAN
'''
from __future__ import print_function from __future__ import print_function
import argparse import argparse
@ -10,6 +14,7 @@ import sys
topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../') topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../../')
topdir = os.path.realpath(topdir) topdir = os.path.realpath(topdir)
class EnumDocco(object): class EnumDocco(object):
vehicle_map = { vehicle_map = {
@ -35,35 +40,35 @@ class EnumDocco(object):
# attempts to extract name, value and comment from line. # attempts to extract name, value and comment from line.
# Match: " FRED, // optional comment" # Match: " FRED, // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+)\s*,? *(?://[/>]* *(.*) *)?$", line) m = re.match(r"\s*([A-Z0-9_a-z]+)\s*,? *(?://[/>]* *(.*) *)?$", line)
if m is not None: if m is not None:
return (m.group(1), None, m.group(2)) return (m.group(1), None, m.group(2))
# Match: " FRED, /* optional comment */" # Match: " FRED, /* optional comment */"
m = re.match("\s*([A-Z0-9_a-z]+)\s*,? *(?:/[*] *(.*) *[*]/ *)?$", line) m = re.match(r"\s*([A-Z0-9_a-z]+)\s*,? *(?:/[*] *(.*) *[*]/ *)?$", line)
if m is not None: if m is not None:
return (m.group(1), None, m.group(2)) return (m.group(1), None, m.group(2))
# Match: " FRED = 17, // optional comment" # Match: " FRED = 17, // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+)\s*=\s*([-0-9]+)\s*,?(?:\s*//[/<]*\s*(.*) *)?$", m = re.match(r"\s*([A-Z0-9_a-z]+)\s*=\s*([-0-9]+)\s*,?(?:\s*//[/<]*\s*(.*) *)?$",
line) line)
if m is not None: if m is not None:
return (m.group(1), m.group(2), m.group(3)) return (m.group(1), m.group(2), m.group(3))
# Match: " FRED = 17, // optional comment" # Match: " FRED = 17, // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+) *= *([-0-9]+) *,?(?: */* *(.*) *)? *[*]/ *$", m = re.match(r"\s*([A-Z0-9_a-z]+) *= *([-0-9]+) *,?(?: */* *(.*) *)? *[*]/ *$",
line) line)
if m is not None: if m is not None:
return (m.group(1), m.group(2), m.group(3)) return (m.group(1), m.group(2), m.group(3))
# Match: " FRED = 1U<<0, // optional comment" # Match: " FRED = 1U<<0, // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+) *= *[(]?1U? *[<][<] *(\d+)(?:, *// *(.*) *)?", m = re.match(r"\s*([A-Z0-9_a-z]+) *= *[(]?1U? *[<][<] *(\d+)(?:, *// *(.*) *)?",
line) line)
if m is not None: if m is not None:
return (m.group(1), 1 << int(m.group(2)), m.group(3)) return (m.group(1), 1 << int(m.group(2)), m.group(3))
# Match: " FRED = 0xabc, // optional comment" # Match: " FRED = 0xabc, // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+) *= *(?:0[xX]([0-9A-Fa-f]+))(?:, *// *(.*) *)?", m = re.match(r"\s*([A-Z0-9_a-z]+) *= *(?:0[xX]([0-9A-Fa-f]+))(?:, *// *(.*) *)?",
line) line)
if m is not None: if m is not None:
return (m.group(1), int(m.group(2), 16), m.group(3)) return (m.group(1), int(m.group(2), 16), m.group(3))
@ -71,19 +76,18 @@ class EnumDocco(object):
'''start discarded matches - lines we understand but can't do anything '''start discarded matches - lines we understand but can't do anything
with:''' with:'''
# Match: " FRED = 17, // optional comment" # Match: " FRED = 17, // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+) *= *(\w+) *,?(?: *// *(.*) *)?$", m = re.match(r"\s*([A-Z0-9_a-z]+) *= *(\w+) *,?(?: *// *(.*) *)?$",
line) line)
if m is not None: if m is not None:
return (None, None, None) return (None, None, None)
# Match: " FRED = FOO(17), // optional comment" # Match: " FRED = FOO(17), // optional comment"
m = re.match("\s*([A-Z0-9_a-z]+) *= *(\w+) *\\( *(\w+) *\\) *,?(?: *// *(.*) *)?$", m = re.match(r"\s*([A-Z0-9_a-z]+) *= *(\w+) *\\( *(\w+) *\\) *,?(?: *// *(.*) *)?$",
line) line)
if m is not None: if m is not None:
return (None, None, None) return (None, None, None)
# Match: " FRED = 1U<<0, // optional comment"
# Match: " FRED = 1U<<0, // optional comment" m = re.match(r"\s*([A-Z0-9_a-z]+) *= *[(]?3U? *[<][<] *(\d+)(?:, *// *(.*) *)?",
m = re.match("\s*([A-Z0-9_a-z]+) *= *[(]?3U? *[<][<] *(\d+)(?:, *// *(.*) *)?",
line) line)
if m is not None: if m is not None:
return (m.group(1), 1 << int(m.group(2)), m.group(3)) return (m.group(1), 1 << int(m.group(2)), m.group(3))
@ -112,21 +116,21 @@ class EnumDocco(object):
break break
line = line.rstrip() line = line.rstrip()
# print("state=%s line: %s" % (state, line)) # print("state=%s line: %s" % (state, line))
if re.match("\s*//.*", line): if re.match(r"\s*//.*", line):
continue continue
if state == "outside": if state == "outside":
if re.match("class .*;", line) is not None: if re.match("class .*;", line) is not None:
# forward-declaration of a class # forward-declaration of a class
continue continue
m = re.match("class *([:\w]+)", line) m = re.match(r"class *([:\w]+)", line)
if m is not None: if m is not None:
in_class = m.group(1) in_class = m.group(1)
continue continue
m = re.match("namespace *(\w+)", line) m = re.match(r"namespace *(\w+)", line)
if m is not None: if m is not None:
in_class = m.group(1) in_class = m.group(1)
continue continue
m = re.match(".*enum\s*(class)? *([\w]+)\s*(?::.*_t)? *{(.*)};", line) m = re.match(r".*enum\s*(class)? *([\w]+)\s*(?::.*_t)? *{(.*)};", line)
if m is not None: if m is not None:
# all one one line! Thanks! # all one one line! Thanks!
enum_name = m.group(2) enum_name = m.group(2)
@ -142,7 +146,7 @@ class EnumDocco(object):
enumerations.append(new_enumeration) enumerations.append(new_enumeration)
continue continue
m = re.match(".*enum\s*(class)? *([\w]+)\s*(?::.*_t)? *{", line) m = re.match(r".*enum\s*(class)? *([\w]+)\s*(?::.*_t)? *{", line)
if m is not None: if m is not None:
enum_name = m.group(2) enum_name = m.group(2)
debug("%s: %s" % (source_file, enum_name)) debug("%s: %s" % (source_file, enum_name))
@ -152,15 +156,15 @@ class EnumDocco(object):
skip_enumeration = False skip_enumeration = False
continue continue
if state == "inside": if state == "inside":
if re.match("\s*$", line): if re.match(r"\s*$", line):
continue continue
if re.match("#if", line): if re.match(r"#if", line):
continue continue
if re.match("#endif", line): if re.match(r"#endif", line):
continue continue
if re.match("#else", line): if re.match(r"#else", line):
continue continue
if re.match(".*}\s*\w*(\s*=\s*[\w:]+)?;", line): if re.match(r".*}\s*\w*(\s*=\s*[\w:]+)?;", line):
# potential end of enumeration # potential end of enumeration
if not skip_enumeration: if not skip_enumeration:
if enum_name is None: if enum_name is None:
@ -189,7 +193,7 @@ class EnumDocco(object):
else: else:
value = int(value) value = int(value)
last_value = value last_value = value
# print("entry=%s value=%s comment=%s" % (name, value, comment)) # print("entry=%s value=%s comment=%s" % (name, value, comment))
entries.append(EnumDocco.EnumEntry(name, value, comment)) entries.append(EnumDocco.EnumEntry(name, value, comment))
return enumerations return enumerations
@ -253,4 +257,3 @@ if __name__ == '__main__':
sys.exit(1) sys.exit(1)
s.run() s.run()
# print("Enumerations: %s" % s.enumerations)

58
Tools/autotest/logger_metadata/parse.py
View File

@ -1,5 +1,9 @@
#!/usr/bin/env python #!/usr/bin/env python
'''
AP_FLAKE8_CLEAN
'''
from __future__ import print_function from __future__ import print_function
import argparse import argparse
@ -33,7 +37,11 @@ re_vehicles = re.compile(r"\s*//\s*@Vehicles\s*:\s*(.*)")
# Regular expressions for finding message definitions in structure format # Regular expressions for finding message definitions in structure format
re_start_messagedef = re.compile(r"^\s*{?\s*LOG_[A-Z0-9_]+_[MSGTA]+[A-Z0-9_]*\s*,") re_start_messagedef = re.compile(r"^\s*{?\s*LOG_[A-Z0-9_]+_[MSGTA]+[A-Z0-9_]*\s*,")
re_deffield = r'[\s\\]*"?([\w\-#?%]+)"?\s*' re_deffield = r'[\s\\]*"?([\w\-#?%]+)"?\s*'
re_full_messagedef = re.compile(r'\s*LOG_\w+\s*,\s*\w+\([^)]+\)[\s\\]*,' + f'{re_deffield},{re_deffield},' + r'[\s\\]*"?([\w,]+)"?[\s\\]*,' + f'{re_deffield},{re_deffield}' , re.MULTILINE) re_full_messagedef = re.compile(r'\s*LOG_\w+\s*,\s*\w+\([^)]+\)[\s\\]*,' +
f'{re_deffield},{re_deffield},' +
r'[\s\\]*"?([\w,]+)"?[\s\\]*,' +
f'{re_deffield},{re_deffield}',
re.MULTILINE)
re_fmt_define = re.compile(r'#define\s+(\w+_FMT)\s+"([\w\-#?%]+)"') re_fmt_define = re.compile(r'#define\s+(\w+_FMT)\s+"([\w\-#?%]+)"')
re_units_define = re.compile(r'#define\s+(\w+_UNITS)\s+"([\w\-#?%]+)"') re_units_define = re.compile(r'#define\s+(\w+_UNITS)\s+"([\w\-#?%]+)"')
re_mults_define = re.compile(r'#define\s+(\w+_MULTS)\s+"([\w\-#?%]+)"') re_mults_define = re.compile(r'#define\s+(\w+_MULTS)\s+"([\w\-#?%]+)"')
@ -41,7 +49,9 @@ re_mults_define = re.compile(r'#define\s+(\w+_MULTS)\s+"([\w\-#?%]+)"')
# Regular expressions for finding message definitions in Write calls # Regular expressions for finding message definitions in Write calls
re_start_writecall = re.compile(r"\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(") re_start_writecall = re.compile(r"\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(")
re_writefield = r'\s*"([\w\-#?%,]+)"\s*' re_writefield = r'\s*"([\w\-#?%,]+)"\s*'
re_full_writecall = re.compile(r'\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(' + f'{re_writefield},{re_writefield},{re_writefield},({re_writefield},{re_writefield})?' , re.MULTILINE) re_full_writecall = re.compile(r'\s*[AP:]*logger[\(\)]*.Write[StreamingCrcl]*\(' +
f'{re_writefield},{re_writefield},{re_writefield},({re_writefield},{re_writefield})?',
re.MULTILINE)
# Regular expression for extracting unit and multipliers from structure # Regular expression for extracting unit and multipliers from structure
re_units_mults_struct = re.compile(r"^\s*{\s*'([\w\-#?%!/])',"+r'\s*"?([\w\-#?%./]*)"?\s*}') re_units_mults_struct = re.compile(r"^\s*{\s*'([\w\-#?%!/])',"+r'\s*"?([\w\-#?%./]*)"?\s*}')
@ -64,6 +74,7 @@ mult_prefix_lookup = {
1e-9: "n" # nano- 1e-9: "n" # nano-
} }
class LoggerDocco(object): class LoggerDocco(object):
vehicle_map = { vehicle_map = {
@ -93,7 +104,7 @@ class LoggerDocco(object):
def __init__(self, name): def __init__(self, name):
self.name = name self.name = name
self.url = None self.url = None
if isinstance(name,list): if isinstance(name, list):
self.description = [None] * len(name) self.description = [None] * len(name)
else: else:
self.description = None self.description = None
@ -104,15 +115,15 @@ class LoggerDocco(object):
def add_name(self, name): def add_name(self, name):
# If self.name/description aren't lists, convert them # If self.name/description aren't lists, convert them
if isinstance(self.name,str): if isinstance(self.name, str):
self.name = [self.name] self.name = [self.name]
self.description = [self.description] self.description = [self.description]
# Replace any existing empty descriptions with empty strings # Replace any existing empty descriptions with empty strings
for i in range(0,len(self.description)): for i in range(0, len(self.description)):
if self.description[i] is None: if self.description[i] is None:
self.description[i] = "" self.description[i] = ""
# Extend the name and description lists # Extend the name and description lists
if isinstance(name,list): if isinstance(name, list):
self.name.extend(name) self.name.extend(name)
self.description.extend([None] * len(name)) self.description.extend([None] * len(name))
else: else:
@ -120,8 +131,8 @@ class LoggerDocco(object):
self.description.append(None) self.description.append(None)
def set_description(self, desc): def set_description(self, desc):
if isinstance(self.description,list): if isinstance(self.description, list):
for i in range(0,len(self.description)): for i in range(0, len(self.description)):
if self.description[i] is None: if self.description[i] is None:
self.description[i] = desc self.description[i] = desc
else: else:
@ -147,7 +158,7 @@ class LoggerDocco(object):
count = 0 count = 0
entries = [] entries = []
for bit in bits: for bit in bits:
entries.append(EnumDocco.EnumEntry(bit, 1<<count, None)) entries.append(EnumDocco.EnumEntry(bit, 1 << count, None))
count += 1 count += 1
bitmask_name = self.name + field bitmask_name = self.name + field
self.bits_enums.append(EnumDocco.Enumeration(bitmask_name, entries)) self.bits_enums.append(EnumDocco.Enumeration(bitmask_name, entries))
@ -167,14 +178,14 @@ class LoggerDocco(object):
def set_fmts(self, fmts): def set_fmts(self, fmts):
# If no fields are defined, do nothing # If no fields are defined, do nothing
if len(self.fields_order)==0: if len(self.fields_order) == 0:
return return
# Make sure lengths match up # Make sure lengths match up
if len(fmts) != len(self.fields_order): if len(fmts) != len(self.fields_order):
print(f"Number of fmts don't match fields: msg={self.name} fmts={fmts} num_fields={len(self.fields_order)}") print(f"Number of fmts don't match fields: msg={self.name} fmts={fmts} num_fields={len(self.fields_order)}")
return return
# Loop through the list # Loop through the list
for idx in range(0,len(fmts)): for idx in range(0, len(fmts)):
if fmts[idx] in log_fmt_lookup: if fmts[idx] in log_fmt_lookup:
self.fields[self.fields_order[idx]]["fmt"] = log_fmt_lookup[fmts[idx]] self.fields[self.fields_order[idx]]["fmt"] = log_fmt_lookup[fmts[idx]]
else: else:
@ -182,14 +193,14 @@ class LoggerDocco(object):
def set_units(self, units, mults): def set_units(self, units, mults):
# If no fields are defined, do nothing # If no fields are defined, do nothing
if len(self.fields_order)==0: if len(self.fields_order) == 0:
return return
# Make sure lengths match up # Make sure lengths match up
if len(units) != len(self.fields_order) or len(units) != len(mults): if len(units) != len(self.fields_order) or len(units) != len(mults):
print(f"Number of units/mults/fields don't match: msg={self.name} units={units} mults={mults} num_fields={len(self.fields_order)}") print(f"Number of units/mults/fields don't match: msg={self.name} units={units} mults={mults} num_fields={len(self.fields_order)}") # noqa:E501
return return
# Loop through the list # Loop through the list
for idx in range(0,len(units)): for idx in range(0, len(units)):
# Get the index into fields from field_order # Get the index into fields from field_order
f = self.fields_order[idx] f = self.fields_order[idx]
# Convert unit char to base unit # Convert unit char to base unit
@ -275,7 +286,7 @@ class LoggerDocco(object):
if len(_next): if len(_next):
self.search_for_files(_next) self.search_for_files(_next)
def parse_messagedef(self,messagedef): def parse_messagedef(self, messagedef):
# Merge concatinated strings and remove comments # Merge concatinated strings and remove comments
messagedef = re.sub(r'"\s+"', '', messagedef) messagedef = re.sub(r'"\s+"', '', messagedef)
messagedef = re.sub(r'//[^\n]*', '', messagedef) messagedef = re.sub(r'//[^\n]*', '', messagedef)
@ -299,9 +310,9 @@ class LoggerDocco(object):
self.msg_mults_list[d.group(1)] = d.group(5) self.msg_mults_list[d.group(1)] = d.group(5)
return return
# Didn't parse # Didn't parse
#print(f"Unable to parse: {messagedef}") # print(f"Unable to parse: {messagedef}")
def search_messagedef_start(self,line,prevmessagedef=""): def search_messagedef_start(self, line, prevmessagedef=""):
# Look for the start of a structure definition # Look for the start of a structure definition
d = re_start_messagedef.search(line) d = re_start_messagedef.search(line)
if d is not None: if d is not None:
@ -325,13 +336,14 @@ class LoggerDocco(object):
def parse_file(self, filepath): def parse_file(self, filepath):
with open(filepath) as f: with open(filepath) as f:
# print("Opened (%s)" % filepath) # print("Opened (%s)" % filepath)
lines = f.readlines() lines = f.readlines()
f.close() f.close()
def debug(x): def debug(x):
pass pass
# if filepath == "/home/pbarker/rc/ardupilot/libraries/AP_HAL/AnalogIn.h": # if filepath == "/home/pbarker/rc/ardupilot/libraries/AP_HAL/AnalogIn.h":
# debug = print # debug = print
state_outside = "outside" state_outside = "outside"
state_inside = "inside" state_inside = "inside"
messagedef = "" messagedef = ""
@ -346,7 +358,7 @@ class LoggerDocco(object):
messagedef = "" messagedef = ""
if state == state_outside: if state == state_outside:
# Check for start of a message definition # Check for start of a message definition
messagedef = self.search_messagedef_start(line,messagedef) messagedef = self.search_messagedef_start(line, messagedef)
# Check for fmt/unit/mult #define # Check for fmt/unit/mult #define
u = re_fmt_define.search(line) u = re_fmt_define.search(line)
@ -425,7 +437,7 @@ class LoggerDocco(object):
new_doccos = [] new_doccos = []
for docco in self.doccos: for docco in self.doccos:
if isinstance(docco.name, list): if isinstance(docco.name, list):
for name,desc in zip(docco.name, docco.description): for name, desc in zip(docco.name, docco.description):
tmpdocco = copy.copy(docco) tmpdocco = copy.copy(docco)
tmpdocco.name = name tmpdocco.name = name
tmpdocco.description = desc tmpdocco.description = desc
@ -463,7 +475,7 @@ class LoggerDocco(object):
mults = self.msg_mults_list[docco.name] mults = self.msg_mults_list[docco.name]
# Apply the units/mults to the docco # Apply the units/mults to the docco
if units is not None and mults is not None: if units is not None and mults is not None:
docco.set_units(units,mults) docco.set_units(units, mults)
elif units is not None or mults is not None: elif units is not None or mults is not None:
print(f"Cannot find matching units/mults for message {docco.name}") print(f"Cannot find matching units/mults for message {docco.name}")