ardupilot/Tools/autotest/autotest.py

837 lines
26 KiB
Python
Raw Normal View History

#!/usr/bin/env python
"""
APM automatic test suite
Andrew Tridgell, October 2011
"""
2016-11-08 07:06:05 -04:00
from __future__ import print_function
import atexit
import fnmatch
import glob
import optparse
import os
2018-12-13 21:40:01 -04:00
import re
import shutil
import signal
import subprocess
import sys
import time
import traceback
import apmrover2
import arducopter
import arduplane
import ardusub
import antennatracker
import quadplane
2018-07-20 08:53:31 -03:00
import balancebot
import examples
from pysim import util
from pymavlink import mavutil
from pymavlink.generator import mavtemplate
def buildlogs_dirpath():
return os.getenv("BUILDLOGS", util.reltopdir("../buildlogs"))
def buildlogs_path(path):
"""return a string representing path in the buildlogs directory"""
bits = [buildlogs_dirpath()]
if isinstance(path, list):
bits.extend(path)
else:
bits.append(path)
return os.path.join(*bits)
def get_default_params(atype, binary):
"""Get default parameters."""
# use rover simulator so SITL is not starved of input
HOME = mavutil.location(40.071374969556928,
-105.22978898137808,
1583.702759,
246)
if "plane" in binary or "rover" in binary:
frame = "rover"
else:
frame = "+"
home = "%f,%f,%u,%u" % (HOME.lat, HOME.lng, HOME.alt, HOME.heading)
sitl = util.start_SITL(binary,
wipe=True,
model=frame,
home=home,
speedup=10,
unhide_parameters=True)
mavproxy = util.start_MAVProxy_SITL(atype)
print("Dumping defaults")
idx = mavproxy.expect(['Saved [0-9]+ parameters to (\S+)'])
if idx == 0:
# we need to restart it after eeprom erase
util.pexpect_close(mavproxy)
util.pexpect_close(sitl)
sitl = util.start_SITL(binary, model=frame, home=home, speedup=10)
mavproxy = util.start_MAVProxy_SITL(atype)
mavproxy.expect('Saved [0-9]+ parameters to (\S+)')
parmfile = mavproxy.match.group(1)
dest = buildlogs_path('%s-defaults.parm' % atype)
shutil.copy(parmfile, dest)
util.pexpect_close(mavproxy)
util.pexpect_close(sitl)
print("Saved defaults for %s to %s" % (atype, dest))
return True
def build_all_filepath():
return util.reltopdir('Tools/scripts/build_all.sh')
def build_all():
"""Run the build_all.sh script."""
print("Running build_all.sh")
if util.run_cmd(build_all_filepath(), directory=util.reltopdir('.')) != 0:
print("Failed build_all.sh")
return False
return True
def build_binaries():
"""Run the build_binaries.py script."""
print("Running build_binaries.py")
# copy the script as it changes git branch, which can change the
# script while running
orig = util.reltopdir('Tools/scripts/build_binaries.py')
copy = util.reltopdir('./build_binaries.py')
shutil.copy2(orig, copy)
# also copy generate_manifest library:
orig_gm = util.reltopdir('Tools/scripts/generate_manifest.py')
copy_gm = util.reltopdir('./generate_manifest.py')
shutil.copy2(orig_gm, copy_gm)
2019-05-03 22:20:37 -03:00
# and gen_stable.py
orig_gs = util.reltopdir('Tools/scripts/gen_stable.py')
copy_gs = util.reltopdir('./gen_stable.py')
shutil.copy2(orig_gs, copy_gs)
if util.run_cmd(copy, directory=util.reltopdir('.')) != 0:
print("Failed build_binaries.py")
return False
return True
def build_examples():
"""Build examples."""
2018-11-20 06:25:55 -04:00
for target in 'fmuv2', 'Pixhawk1', 'navio', 'linux':
print("Running build.examples for %s" % target)
try:
util.build_examples(target)
except Exception as e:
print("Failed build_examples on board=%s" % target)
print(str(e))
return False
return True
def build_unit_tests():
"""Build tests."""
for target in ['linux']:
print("Running build.unit_tests for %s" % target)
try:
util.build_tests(target)
except Exception as e:
print("Failed build.unit_tests on board=%s" % target)
print(str(e))
return False
return True
def run_unit_test(test):
print("Running (%s)" % test)
subprocess.check_call([test])
def run_unit_tests():
binary_dir = util.reltopdir(os.path.join('build',
'linux',
'tests',
))
tests = glob.glob("%s/*" % binary_dir)
success = True
for test in tests:
try:
run_unit_test(test)
except Exception as e:
print("Exception running (%s): %s" % (test, e.message))
success = False
return success
def param_parse_filepath():
return util.reltopdir('Tools/autotest/param_metadata/param_parse.py')
def all_vehicles():
return ('ArduPlane',
'ArduCopter',
'APMrover2',
'AntennaTracker',
'ArduSub')
def build_parameters():
"""Run the param_parse.py script."""
print("Running param_parse.py")
for vehicle in all_vehicles():
if util.run_cmd([param_parse_filepath(), '--vehicle', vehicle],
directory=util.reltopdir('.')) != 0:
print("Failed param_parse.py (%s)" % vehicle)
return False
return True
def mavtogpx_filepath():
return util.reltopdir("modules/mavlink/pymavlink/tools/mavtogpx.py")
2011-11-09 05:27:00 -04:00
def convert_gpx():
"""Convert any tlog files to GPX and KML."""
mavlog = glob.glob(buildlogs_path("*.tlog"))
passed = True
2011-11-09 05:27:00 -04:00
for m in mavlog:
util.run_cmd(mavtogpx_filepath() + " --nofixcheck " + m)
2011-11-09 05:27:00 -04:00
gpx = m + '.gpx'
kml = m + '.kml'
try:
util.run_cmd('gpsbabel -i gpx -f %s '
'-o kml,units=m,floating=1,extrude=1 -F %s' %
(gpx, kml))
except subprocess.CalledProcessError as e:
passed = False
try:
util.run_cmd('zip %s.kmz %s.kml' % (m, m))
except subprocess.CalledProcessError as e:
passed = False
util.run_cmd("mavflightview.py --imagefile=%s.png %s" % (m, m))
return passed
2011-11-09 05:27:00 -04:00
def test_prerequisites():
"""Check we have the right directories and tools to run tests."""
print("Testing prerequisites")
util.mkdir_p(buildlogs_dirpath())
return True
2011-11-12 19:11:25 -04:00
def alarm_handler(signum, frame):
"""Handle test timeout."""
global results, opts
try:
print("Alarm handler called")
results.add('TIMEOUT',
'<span class="failed-text">FAILED</span>',
opts.timeout)
2012-06-15 17:47:55 -03:00
util.pexpect_close_all()
2012-07-09 20:13:16 -03:00
convert_gpx()
write_fullresults()
os.killpg(0, signal.SIGKILL)
except Exception:
pass
sys.exit(1)
def should_run_step(step):
"""See if a step should be skipped."""
for skip in skipsteps:
if fnmatch.fnmatch(step.lower(), skip.lower()):
return False
return True
__bin_names = {
"ArduCopter": "arducopter",
"ArduPlane": "arduplane",
"APMrover2": "ardurover",
"AntennaTracker": "antennatracker",
"CopterAVC": "arducopter-heli",
"QuadPlane": "arduplane",
2018-07-20 08:53:31 -03:00
"ArduSub": "ardusub",
"balancebot": "ardurover",
"BalanceBot": "ardurover",
}
def binary_path(step, debug=False):
try:
vehicle = step.split(".")[1]
except Exception:
return None
if vehicle in __bin_names:
binary_name = __bin_names[vehicle]
else:
2016-05-27 18:29:23 -03:00
# cope with builds that don't have a specific binary
return None
binary = util.reltopdir(os.path.join('build',
'sitl',
'bin',
binary_name))
if not os.path.exists(binary):
if os.path.exists(binary + ".exe"):
binary += ".exe"
else:
raise ValueError("Binary (%s) does not exist" % (binary,))
return binary
2018-12-13 21:40:01 -04:00
def split_specific_test_step(step):
print('step=%s' % str(step))
m = re.match("((fly|drive|dive|test)[.][^.]+)[.](.*)", step)
2018-12-13 21:40:01 -04:00
if m is None:
return None
return ( (m.group(1), m.group(3)) )
def find_specific_test_to_run(step):
t = split_specific_test_step(step)
if t is None:
return None
(testname, test) = t
return "%s.%s" % (testname, test)
tester_class_map = {
"fly.ArduCopter": arducopter.AutoTestCopter,
"fly.ArduPlane": arduplane.AutoTestPlane,
"fly.QuadPlane": quadplane.AutoTestQuadPlane,
"drive.APMrover2": apmrover2.AutoTestRover,
"drive.balancebot": balancebot.AutoTestBalanceBot,
"fly.CopterAVC": arducopter.AutoTestHeli,
"dive.ArduSub": ardusub.AutoTestSub,
"test.AntennaTracker": antennatracker.AutoTestTracker,
}
2018-12-13 21:40:01 -04:00
def run_specific_test(step, *args, **kwargs):
t = split_specific_test_step(step)
if t is None:
return []
(testname, test) = t
tester_class = tester_class_map[testname]
tester = tester_class(*args, **kwargs)
print("Got %s" % str(tester))
for a in tester.tests():
print("Got %s" % (a[0]))
if a[0] == test:
return tester.run_tests([a])
print("Failed to find test %s on %s" % (test, testname))
sys.exit(1)
def run_step(step):
"""Run one step."""
# remove old logs
util.run_cmd('/bin/rm -f logs/*.BIN logs/LASTLOG.TXT')
if step == "prerequisites":
return test_prerequisites()
build_opts = {
"j": opts.j,
"debug": opts.debug,
"clean": not opts.no_clean,
"configure": not opts.no_configure,
"extra_configure_args": opts.waf_configure_args,
}
vehicle_binary = None
if step == 'build.ArduPlane':
vehicle_binary = 'bin/arduplane'
if step == 'build.APMrover2':
vehicle_binary = 'bin/ardurover'
if step == 'build.ArduCopter':
vehicle_binary = 'bin/arducopter'
if step == 'build.AntennaTracker':
vehicle_binary = 'bin/antennatracker'
if step == 'build.Helicopter':
vehicle_binary = 'bin/arducopter-heli'
2017-02-24 19:22:38 -04:00
if step == 'build.ArduSub':
vehicle_binary = 'bin/ardusub'
if vehicle_binary is not None:
return util.build_SITL(vehicle_binary, **build_opts)
binary = binary_path(step, debug=opts.debug)
if step.startswith("defaults"):
vehicle = step[9:]
return get_default_params(vehicle, binary)
fly_opts = {
"viewerip": opts.viewerip,
"use_map": opts.map,
"valgrind": opts.valgrind,
"gdb": opts.gdb,
"lldb": opts.lldb,
"gdbserver": opts.gdbserver,
"breakpoints": opts.breakpoint,
"disable_breakpoints": opts.disable_breakpoints,
"frame": opts.frame,
"_show_test_timings": opts.show_test_timings,
}
if opts.speedup is not None:
fly_opts["speedup"] = opts.speedup
# handle "fly.ArduCopter" etc:
if step in tester_class_map:
t = tester_class_map[step](binary, **fly_opts)
return (t.autotest(), t)
2018-12-13 21:40:01 -04:00
specific_test_to_run = find_specific_test_to_run(step)
if specific_test_to_run is not None:
return run_specific_test(specific_test_to_run, binary, **fly_opts)
2018-12-13 21:40:01 -04:00
if step == 'build.All':
return build_all()
if step == 'build.Binaries':
return build_binaries()
if step == 'build.examples':
return build_examples()
if step == 'run.examples':
return examples.run_examples(debug=opts.debug, valgrind=False, gdb=False)
if step == 'build.Parameters':
return build_parameters()
2011-11-09 05:27:00 -04:00
if step == 'convertgpx':
return convert_gpx()
if step == 'build.unit_tests':
return build_unit_tests()
if step == 'run.unit_tests':
return run_unit_tests()
raise RuntimeError("Unknown step %s" % step)
2011-11-09 08:25:08 -04:00
class TestResult(object):
"""Test result class."""
2011-11-09 08:25:08 -04:00
def __init__(self, name, result, elapsed):
self.name = name
self.result = result
self.elapsed = "%.1f" % elapsed
2011-11-09 08:25:08 -04:00
class TestFile(object):
"""Test result file."""
2011-11-09 08:25:08 -04:00
def __init__(self, name, fname):
self.name = name
self.fname = fname
2011-11-09 08:25:08 -04:00
class TestResults(object):
"""Test results class."""
2011-11-09 08:25:08 -04:00
def __init__(self):
self.date = time.asctime()
self.githash = util.run_cmd('git rev-parse HEAD',
output=True,
directory=util.reltopdir('.')).strip()
2011-11-09 08:25:08 -04:00
self.tests = []
self.files = []
self.images = []
2011-11-09 08:25:08 -04:00
def add(self, name, result, elapsed):
"""Add a result."""
2011-11-09 08:25:08 -04:00
self.tests.append(TestResult(name, result, elapsed))
def addfile(self, name, fname):
"""Add a result file."""
2011-11-09 08:25:08 -04:00
self.files.append(TestFile(name, fname))
def addimage(self, name, fname):
"""Add a result image."""
self.images.append(TestFile(name, fname))
2011-11-09 08:25:08 -04:00
def addglob(self, name, pattern):
"""Add a set of files."""
for f in glob.glob(buildlogs_path(pattern)):
2011-11-09 08:25:08 -04:00
self.addfile(name, os.path.basename(f))
def addglobimage(self, name, pattern):
"""Add a set of images."""
for f in glob.glob(buildlogs_path(pattern)):
self.addimage(name, os.path.basename(f))
2019-02-07 12:11:45 -04:00
def generate_badge(self):
"""
Gets the badge template, populates and saves the result to buildlogs
path.
"""
passed_tests = len([t for t in self.tests if "PASSED" in t.result])
total_tests = len(self.tests)
badge_color = "#4c1" if passed_tests == total_tests else "#e05d44"
badge_text = "{0}/{1}".format(passed_tests, total_tests)
# Text length so it is not stretched by svg
text_length = len(badge_text) * 70
# Load template file
template_path = 'Tools/autotest/web/autotest-badge-template.svg'
with open(util.reltopdir(template_path), "r") as f:
template = f.read()
# Add our results to the template
badge = template.format(color=badge_color,
text=badge_text,
text_length=text_length)
with open(buildlogs_path("autotest-badge.svg"), "w") as f:
f.write(badge)
2011-11-09 08:25:08 -04:00
def write_webresults(results_to_write):
"""Write webpage results."""
2011-11-09 08:25:08 -04:00
t = mavtemplate.MAVTemplate()
for h in glob.glob(util.reltopdir('Tools/autotest/web/*.html')):
html = util.loadfile(h)
f = open(buildlogs_path(os.path.basename(h)), mode='w')
t.write(f, html, results_to_write)
f.close()
2011-11-09 09:31:50 -04:00
for f in glob.glob(util.reltopdir('Tools/autotest/web/*.png')):
shutil.copy(f, buildlogs_path(os.path.basename(f)))
2019-02-07 12:11:45 -04:00
results_to_write.generate_badge()
2011-11-09 08:25:08 -04:00
def write_fullresults():
"""Write out full results set."""
global results
results.addglob("Google Earth track", '*.kmz')
results.addfile('Full Logs', 'autotest-output.txt')
results.addglob('DataFlash Log', '*-log.bin')
results.addglob("MAVLink log", '*.tlog')
results.addglob("GPX track", '*.gpx')
# results common to all vehicles:
vehicle_files = [('{vehicle} build log', '{vehicle}.txt'),
('{vehicle} code size', '{vehicle}.sizes.txt'),
('{vehicle} stack sizes', '{vehicle}.framesizes.txt'),
('{vehicle} defaults', '{vehicle}-defaults.parm'),
('{vehicle} core', '{vehicle}.core'),
('{vehicle} ELF', '{vehicle}.elf'), ]
vehicle_globs = [('{vehicle} log', '{vehicle}-*.BIN'), ]
for vehicle in all_vehicles():
subs = {'vehicle': vehicle}
for vehicle_file in vehicle_files:
description = vehicle_file[0].format(**subs)
filename = vehicle_file[1].format(**subs)
results.addfile(description, filename)
for vehicle_glob in vehicle_globs:
description = vehicle_glob[0].format(**subs)
glob = vehicle_glob[1].format(**subs)
results.addglob(description, glob)
results.addglob("CopterAVC log", 'CopterAVC-*.BIN')
results.addfile("CopterAVC core", 'CopterAVC.core')
2013-08-14 20:08:08 -03:00
results.addglob('APM:Libraries documentation', 'docs/libraries/index.html')
results.addglob('APM:Plane documentation', 'docs/ArduPlane/index.html')
results.addglob('APM:Copter documentation', 'docs/ArduCopter/index.html')
results.addglob('APM:Rover documentation', 'docs/APMrover2/index.html')
2017-02-24 19:22:38 -04:00
results.addglob('APM:Sub documentation', 'docs/ArduSub/index.html')
results.addglobimage("Flight Track", '*.png')
write_webresults(results)
def check_logs(step):
"""Check for log files from a step."""
print("check step: ", step)
if step.startswith('fly.'):
vehicle = step[4:]
elif step.startswith('drive.'):
vehicle = step[6:]
elif step.startswith('dive.'):
vehicle = step[5:]
else:
return
logs = glob.glob("logs/*.BIN")
for log in logs:
bname = os.path.basename(log)
newname = buildlogs_path("%s-%s" % (vehicle, bname))
print("Renaming %s to %s" % (log, newname))
shutil.move(log, newname)
corefile = "core"
if os.path.exists(corefile):
newname = buildlogs_path("%s.core" % vehicle)
print("Renaming %s to %s" % (corefile, newname))
shutil.move(corefile, newname)
2017-12-11 17:45:56 -04:00
try:
util.run_cmd('/bin/cp build/sitl/bin/* %s' % buildlogs_dirpath(),
directory=util.reltopdir('.'))
except Exception:
print("Unable to save binary")
def run_tests(steps):
"""Run a list of steps."""
global results
2011-11-09 08:25:08 -04:00
passed = True
2011-11-09 02:16:20 -04:00
failed = []
failed_testinstances = dict()
for step in steps:
util.pexpect_close_all()
2011-11-09 08:25:08 -04:00
t1 = time.time()
print(">>>> RUNNING STEP: %s at %s" % (step, time.asctime()))
try:
success = run_step(step)
testinstance = None
if type(success) == tuple:
(success, testinstance) = success
if success:
results.add(step, '<span class="passed-text">PASSED</span>',
time.time() - t1)
print(">>>> PASSED STEP: %s at %s" % (step, time.asctime()))
check_logs(step)
else:
print(">>>> FAILED STEP: %s at %s" % (step, time.asctime()))
passed = False
2011-11-09 02:16:20 -04:00
failed.append(step)
if testinstance is not None:
if failed_testinstances.get(step) is None:
failed_testinstances[step] = []
failed_testinstances[step].append(testinstance)
results.add(step, '<span class="failed-text">FAILED</span>',
time.time() - t1)
except Exception as msg:
passed = False
2011-11-09 02:16:20 -04:00
failed.append(step)
print(">>>> FAILED STEP: %s at %s (%s)" %
(step, time.asctime(), msg))
2011-11-09 08:43:25 -04:00
traceback.print_exc(file=sys.stdout)
results.add(step,
'<span class="failed-text">FAILED</span>',
time.time() - t1)
check_logs(step)
2011-11-09 02:16:20 -04:00
if not passed:
keys = failed_testinstances.keys()
if len(keys):
print("Failure Summary:")
for key in keys:
print(" %s:" % key)
for testinstance in failed_testinstances[key]:
for failure in testinstance.fail_list:
(desc, exception, debug_filename) = failure
print(" %s (%s) (see %s)" % (desc, exception, debug_filename))
2011-11-09 02:16:20 -04:00
print("FAILED %u tests: %s" % (len(failed), failed))
2011-11-09 08:25:08 -04:00
util.pexpect_close_all()
write_fullresults()
2011-11-09 08:25:08 -04:00
return passed
if __name__ == "__main__":
''' main program '''
os.environ['PYTHONUNBUFFERED'] = '1'
os.putenv('TMPDIR', util.reltopdir('tmp'))
parser = optparse.OptionParser("autotest")
parser.add_option("--skip",
type='string',
default='',
help='list of steps to skip (comma separated)')
parser.add_option("--list",
action='store_true',
default=False,
help='list the available steps')
parser.add_option("--viewerip",
default=None,
help='IP address to send MAVLink and fg packets to')
parser.add_option("--map",
action='store_true',
default=False,
help='show map')
parser.add_option("--experimental",
default=False,
action='store_true',
help='enable experimental tests')
parser.add_option("--timeout",
default=3600,
type='int',
help='maximum runtime in seconds')
parser.add_option("--frame",
type='string',
default=None,
help='specify frame type')
parser.add_option("--show-test-timings",
action="store_true",
default=False,
help="show how long each test took to run")
group_build = optparse.OptionGroup(parser, "Build options")
group_build.add_option("--no-configure",
default=False,
action='store_true',
help='do not configure before building',
dest="no_configure")
group_build.add_option("", "--waf-configure-args",
action="append",
dest="waf_configure_args",
type="string",
default=[],
help="extra arguments passed to waf in configure")
group_build.add_option("-j", default=None, type='int', help='build CPUs')
group_build.add_option("--no-clean",
default=False,
action='store_true',
help='do not clean before building',
dest="no_clean")
group_build.add_option("--debug",
default=False,
action='store_true',
help='make built binaries debug binaries')
parser.add_option_group(group_build)
group_sim = optparse.OptionGroup(parser, "Simulation options")
group_sim.add_option("--speedup",
default=None,
type='int',
help='speedup to run the simulations at')
group_sim.add_option("--valgrind",
default=False,
action='store_true',
help='run ArduPilot binaries under valgrind')
group_sim.add_option("--gdb",
default=False,
action='store_true',
help='run ArduPilot binaries under gdb')
group_sim.add_option("--gdbserver",
default=False,
action='store_true',
help='run ArduPilot binaries under gdbserver')
group_sim.add_option("--lldb",
default=False,
action='store_true',
help='run ArduPilot binaries under lldb')
group_sim.add_option("-B", "--breakpoint",
type='string',
action="append",
default=[],
help="add a breakpoint at given location in debugger")
group_sim.add_option("--disable-breakpoints",
default=False,
action='store_true',
help="disable all breakpoints before starting")
parser.add_option_group(group_sim)
opts, args = parser.parse_args()
steps = [
'prerequisites',
'build.Binaries',
'build.All',
'build.Parameters',
'build.unit_tests',
'run.unit_tests',
'build.examples',
'run.examples',
'build.ArduPlane',
'defaults.ArduPlane',
'fly.ArduPlane',
'fly.QuadPlane',
'build.APMrover2',
'defaults.APMrover2',
'drive.APMrover2',
2018-07-20 08:53:31 -03:00
'drive.balancebot',
'build.ArduCopter',
'defaults.ArduCopter',
'fly.ArduCopter',
'build.Helicopter',
'fly.CopterAVC',
'build.AntennaTracker',
'defaults.AntennaTracker',
'test.AntennaTracker',
'build.ArduSub',
'defaults.ArduSub',
'dive.ArduSub',
'convertgpx',
]
skipsteps = opts.skip.split(',')
# ensure we catch timeouts
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(opts.timeout)
if opts.list:
for step in steps:
print(step)
sys.exit(0)
util.mkdir_p(buildlogs_dirpath())
lckfile = buildlogs_path('autotest.lck')
print("lckfile=%s" % repr(lckfile))
lck = util.lock_file(lckfile)
if lck is None:
print("autotest is locked - exiting. lckfile=(%s)" % (lckfile,))
sys.exit(0)
atexit.register(util.pexpect_close_all)
if len(args) > 0:
# allow a wildcard list of steps
matched = []
for a in args:
matches = [step for step in steps
if fnmatch.fnmatch(step.lower(), a.lower())]
2018-12-13 21:40:01 -04:00
x = find_specific_test_to_run(a)
if x is not None:
matches.append(x)
if not len(matches):
print("No steps matched {}".format(a))
sys.exit(1)
matched.extend(matches)
steps = matched
# skip steps according to --skip option:
steps_to_run = [s for s in steps if should_run_step(s)]
results = TestResults()
try:
if not run_tests(steps_to_run):
sys.exit(1)
except KeyboardInterrupt:
util.pexpect_close_all()
sys.exit(1)
except Exception:
# make sure we kill off any children
util.pexpect_close_all()
raise