2018-03-03 06:42:52 -04:00
#!/usr/bin/env python
2019-03-13 02:53:56 -03:00
''' A helper script for bisecting common problems when working with ArduPilot
2018-03-03 06:42:52 -04:00
Bisect between a commit which builds and one which doesn ' t,
finding the first commit which broke the build with a
specific failure :
2018-12-31 00:23:31 -04:00
git bisect reset
2018-03-03 06:42:52 -04:00
git bisect good a7647e77d9
git bisect bad 153 ad9539866f8d93a99e9998118bb090d2f747f
cp - a Tools / autotest / bisect - helper . py / tmp
git bisect run / tmp / bisect - helper . py - - build \
- - build - failure - string = \
" reference to ' OpticalFlow ' is ambiguous "
Work out who killed bebop :
cp - a Tools / autotest / bisect - helper . py / tmp
2018-12-31 00:23:31 -04:00
git bisect reset
2018-03-03 06:42:52 -04:00
git bisect good a7647e77d9 & &
git bisect bad 153 ad9539866f8d93a99e9998118bb090d2f747f & &
git bisect run / tmp / bisect - helper . py - - build \
- - waf - configure - arg = " --board bebop "
2018-12-31 00:23:31 -04:00
# Use a failing test to work out which commit broke things:
cp Tools / autotest / bisect - helper . py / tmp
git bisect reset
git bisect start
git bisect bad
git bisect good HEAD ~ 1024
time git bisect run / tmp / bisect - helper . py - - autotest - - autotest - vehicle = Plane - - autotest - test = NeedEKFToArm - - autotest - branch = wip / bisection - using - named - test
2020-01-29 21:01:24 -04:00
Work out who overflowed Omnbusf4pro :
cp - a Tools Tools2
GOOD = c4ce6fa3851f93df34393c376fee5b37e0a270d2
BAD = f00bf77af75f828334f735580d6b19698b639a74
BFS = " overflowed by "
git bisect reset
git bisect start
git bisect good $ GOOD & &
git bisect bad $ BAD & &
git bisect run Tools2 / autotest / bisect - helper . py - - build \
- - waf - configure - arg = " --board OmniBusF4Pro " \
- - build - failure - string = " $BFS "
2019-03-13 02:53:56 -03:00
# Use a flapping test to work out which commit broke things. The
# "autotest-branch" is the branch containing the flapping test (which
# may be master)
rm / tmp / bisect - debug / * ; git commit - m " stuff " - a ; cp Tools / autotest / bisect - helper . py / tmp ; git bisect reset ; git bisect start ; git bisect bad d24e569b20 ; git bisect good 3 f6fd49507f286ad8f6ccc9e29b110d5e9fc9207 ^
time git bisect run / tmp / bisect - helper . py - - autotest - - autotest - vehicle = Copter - - autotest - test = Replay - - autotest - branch = wip / bisection - using - flapping - test - - autotest - test - passes = 40 - - autotest - failure - require - string = " Mismatch in field XKF1.Pitch " - - autotest - failure - ignore - string = " HALSITL::SITL_State::_check_rc_input "
2018-03-03 06:42:52 -04:00
'''
import optparse
import os
import subprocess
import shlex
import sys
import time
2019-03-13 02:53:56 -03:00
import traceback
2018-03-03 06:42:52 -04:00
2019-03-13 02:53:56 -03:00
def get_exception_stacktrace ( e ) :
if sys . version_info [ 0 ] > = 3 :
ret = " %s \n " % e
ret + = ' ' . join ( traceback . format_exception ( etype = type ( e ) ,
value = e ,
tb = e . __traceback__ ) )
return ret
return traceback . format_exc ( e )
2018-03-03 06:42:52 -04:00
class Bisect ( object ) :
def __init__ ( self , opts ) :
self . opts = opts
2019-03-13 02:53:56 -03:00
def exit_skip_code ( self ) :
return 125
def exit_pass_code ( self ) :
return 0
def exit_fail_code ( self ) :
return 1
def exit_abort_code ( self ) :
return 129
2018-03-03 06:42:52 -04:00
def exit_skip ( self ) :
self . progress ( " SKIP " )
2019-03-13 02:53:56 -03:00
sys . exit ( self . exit_skip_code ( ) )
2018-03-03 06:42:52 -04:00
def exit_pass ( self ) :
self . progress ( " PASS " )
2019-03-13 02:53:56 -03:00
sys . exit ( self . exit_pass_code ( ) )
2018-03-03 06:42:52 -04:00
def exit_fail ( self ) :
self . progress ( " FAIL " )
2019-03-13 02:53:56 -03:00
sys . exit ( self . exit_fail_code ( ) )
def exit_abort ( self ) :
''' call when this harness has failed (e.g. to reset to required
state ) '''
self . progress ( " ABORT " )
sys . exit ( self . exit_abort_code ( ) )
2018-03-03 06:42:52 -04:00
def progress ( self , string ) :
''' pretty-print progress '''
print ( " BH: %s " % string )
def run_program ( self , prefix , cmd_list ) :
''' copied in from build_binaries.py '''
''' run cmd_list, spewing and setting output in self '''
self . progress ( " Running ( %s ) " % " " . join ( cmd_list ) )
p = subprocess . Popen ( cmd_list ,
bufsize = 1 ,
stdin = None ,
close_fds = True ,
stdout = subprocess . PIPE ,
stderr = subprocess . STDOUT )
self . program_output = " "
while True :
x = p . stdout . readline ( )
if len ( x ) == 0 :
returncode = os . waitpid ( p . pid , 0 )
if returncode :
break
# select not available on Windows... probably...
time . sleep ( 0.1 )
continue
2019-03-13 02:53:56 -03:00
if type ( x ) == bytes :
x = x . decode ( ' utf-8 ' )
2018-03-03 06:42:52 -04:00
self . program_output + = x
x = x . rstrip ( )
print ( " %s : %s " % ( prefix , x ) )
( _ , status ) = returncode
if status != 0 :
self . progress ( " Process failed ( %s ) " %
str ( returncode ) )
raise subprocess . CalledProcessError (
returncode , cmd_list )
def build ( self ) :
''' run ArduCopter build. May exit with skip or fail '''
self . run_program ( " WAF-clean " , [ " ./waf " , " clean " ] )
cmd_configure = [ " ./waf " , " configure " ]
pieces = [ shlex . split ( x )
2018-12-31 00:23:31 -04:00
for x in self . opts . waf_configure_args ]
2018-03-03 06:42:52 -04:00
for piece in pieces :
cmd_configure . extend ( piece )
self . run_program ( " WAF-configure " , cmd_configure )
cmd_build = [ " ./waf " , " build " ]
pieces = [ shlex . split ( x )
2018-12-31 00:23:31 -04:00
for x in self . opts . waf_build_args ]
2018-03-03 06:42:52 -04:00
for piece in pieces :
cmd_build . extend ( piece )
try :
self . run_program ( " WAF-build " , cmd_build )
except subprocess . CalledProcessError as e :
# well, it definitely failed....
2018-12-31 00:23:31 -04:00
if self . opts . build_failure_string is not None :
if self . opts . build_failure_string in self . program_output :
2018-03-03 06:42:52 -04:00
self . progress ( " Found relevant build failure " )
self . exit_fail ( )
# it failed, but not for the reason we're looking
# for...
self . exit_skip ( )
else :
self . exit_fail ( )
class BisectBuild ( Bisect ) :
def __init__ ( self , opts ) :
super ( BisectBuild , self ) . __init__ ( opts )
def run ( self ) :
self . build ( ) # may exit with skip or fail
self . exit_pass ( )
class BisectCITest ( Bisect ) :
def __init__ ( self , opts ) :
super ( BisectCITest , self ) . __init__ ( opts )
2018-12-31 00:23:31 -04:00
def autotest_script ( self ) :
return os . path . join ( " Tools " , " autotest " , " autotest.py " )
2019-03-13 02:53:56 -03:00
def git_reset ( self ) :
try :
self . run_program ( " Reset autotest directory " , [ " git " , " reset " , " --hard " ] )
except subprocess . CalledProcessError as e :
self . exit_abort ( )
def get_current_hash ( self ) :
self . run_program ( " Get current hash " , [ " git " , " rev-parse " , " HEAD " ] )
x = self . program_output
return x . strip ( )
2018-12-31 00:23:31 -04:00
def run ( self ) :
2019-03-13 02:53:56 -03:00
current_hash = self . get_current_hash ( )
self . debug_dir = os . path . join ( " /tmp " , " bisect-debug " )
if not os . path . exists ( self . debug_dir ) :
os . mkdir ( self . debug_dir )
2018-12-31 00:23:31 -04:00
if self . opts . autotest_branch is None :
raise ValueError ( " expected autotest branch " )
try :
self . run_program ( " Update submodules " ,
[ " git " , " submodule " , " update " , " --init " , " --recursive " ] )
except subprocess . CalledProcessError as e :
2019-03-13 02:53:56 -03:00
self . exit_abort ( )
2018-12-31 00:23:31 -04:00
try :
self . run_program ( " Check autotest directory out from master " ,
[ " git " , " checkout " , self . opts . autotest_branch , " Tools/autotest " ] )
except subprocess . CalledProcessError as e :
2019-03-13 02:53:56 -03:00
self . exit_abort ( )
2018-12-31 00:23:31 -04:00
2019-03-13 02:53:56 -03:00
self . progress ( " Build " )
2018-12-31 00:23:31 -04:00
cmd = [ self . autotest_script ( ) ]
2019-03-13 02:53:56 -03:00
if self . opts . autotest_valgrind :
cmd . append ( " --debug " )
2018-12-31 00:23:31 -04:00
cmd . append ( " build. %s " % self . opts . autotest_vehicle )
2019-03-13 02:53:56 -03:00
print ( " build cmd: %s " % str ( cmd ) )
2018-12-31 00:23:31 -04:00
try :
2019-03-13 02:53:56 -03:00
self . run_program ( " Run autotest (build) " , cmd )
2018-12-31 00:23:31 -04:00
except subprocess . CalledProcessError as e :
2019-03-13 02:53:56 -03:00
self . git_reset ( )
self . exit_skip ( )
2018-12-31 00:23:31 -04:00
2019-03-13 02:53:56 -03:00
cmd = [ self . autotest_script ( ) ]
if self . opts . autotest_valgrind :
cmd . append ( " --valgrind " )
cmd . append ( " test. %s . %s " % ( self . opts . autotest_vehicle , self . opts . autotest_test ) )
2018-12-31 00:23:31 -04:00
2019-03-13 02:53:56 -03:00
code = self . exit_pass_code ( )
for i in range ( 0 , self . opts . autotest_test_passes ) :
ignore = False
try :
self . run_program (
" Run autotest ( %u / %u ) " % ( i + 1 , self . opts . autotest_test_passes ) ,
cmd )
except subprocess . CalledProcessError as e :
for ignore_string in self . opts . autotest_failure_ignore_string :
if ignore_string in self . program_output :
self . progress ( " Found ignore string ( %s ) in program output " % ignore_string )
ignore = True
if not ignore and self . opts . autotest_failure_require_string is not None :
if self . opts . autotest_failure_require_string not in self . program_output :
# it failed, but not for the reason we're looking
# for...
self . progress ( " Did not find test failure string ( %s ); skipping " % self . opts . autotest_failure_require_string )
code = self . exit_skip_code ( )
break
if not ignore :
code = self . exit_fail_code ( )
with open ( os . path . join ( self . debug_dir , " run- %s - %u .txt " % ( current_hash , i + 1 ) ) , " w " ) as f :
f . write ( self . program_output )
if code == self . exit_fail_code ( ) :
with open ( " /tmp/fail-counts " , " a " ) as f :
print ( " Failed on run %u " % ( i + 1 , ) , file = f )
if ignore :
self . progress ( " Ignoring this run " )
continue
if code != self . exit_pass_code ( ) :
break
2018-12-31 00:23:31 -04:00
2019-03-13 02:53:56 -03:00
self . git_reset ( )
sys . exit ( code )
2018-12-31 00:23:31 -04:00
2018-03-03 06:42:52 -04:00
if __name__ == ' __main__ ' :
parser = optparse . OptionParser ( " bisect.py " )
parser . add_option ( " --build " ,
action = ' store_true ' ,
default = False ,
help = " Help bisect a build failure " )
parser . add_option ( " --build-failure-string " ,
type = ' string ' ,
default = None ,
help = " If supplied, must be present in "
" build output to count as a failure " )
2018-12-31 00:23:31 -04:00
group_autotest = optparse . OptionGroup ( parser , " Run-AutoTest Options " )
group_autotest . add_option ( " --autotest " ,
action = ' store_true ' ,
default = False ,
help = " Bisect a failure with an autotest test " )
group_autotest . add_option ( " " , " --autotest-vehicle " ,
dest = " autotest_vehicle " ,
type = " string " ,
default = " ArduCopter " ,
help = " Which vehicle to run tests for " )
group_autotest . add_option ( " " , " --autotest-test " ,
dest = " autotest_test " ,
type = " string " ,
default = " NavDelayAbsTime " ,
help = " Test to run to find failure " )
2019-03-13 02:53:56 -03:00
group_autotest . add_option ( " " , " --autotest-valgrind " ,
dest = " autotest_valgrind " ,
action = ' store_true ' ,
default = False ,
help = " Run autotest under valgrind " )
group_autotest . add_option ( " " , " --autotest-test-passes " ,
dest = " autotest_test_passes " ,
type = int ,
default = 1 ,
help = " Number of times to run test before declaring it is good " )
2018-12-31 00:23:31 -04:00
group_autotest . add_option ( " " , " --autotest-branch " ,
dest = " autotest_branch " ,
type = " string " ,
help = " Branch on which the test exists. The autotest directory will be reset to this branch " )
2019-03-13 02:53:56 -03:00
group_autotest . add_option ( " --autotest-failure-require-string " ,
type = ' string ' ,
default = None ,
help = " If supplied, must be present in "
" test output to count as a failure " )
group_autotest . add_option ( " --autotest-failure-ignore-string " ,
type = ' string ' ,
default = [ ] ,
action = " append " ,
help = " If supplied and present in "
" test output run will be ignored " )
2018-12-31 00:23:31 -04:00
2018-03-03 06:42:52 -04:00
group_build = optparse . OptionGroup ( parser , " Build options " )
group_build . add_option ( " " , " --waf-configure-arg " ,
action = " append " ,
dest = " waf_configure_args " ,
type = " string " ,
default = [ " --board skyviper-v2450 " ] ,
help = " extra arguments to pass to "
" waf in configure step " )
group_build . add_option ( " " , " --waf-build-arg " ,
action = " append " ,
dest = " waf_build_args " ,
type = " string " ,
default = [ " --target bin/arducopter " ] ,
help = " extra arguments to pass "
" to waf in its build step " )
parser . add_option_group ( group_build )
( opts , args ) = parser . parse_args ( )
if opts . build :
bisecter = BisectBuild ( opts )
2018-12-31 00:23:31 -04:00
elif opts . autotest :
2018-03-03 06:42:52 -04:00
bisecter = BisectCITest ( opts )
2018-12-31 00:23:31 -04:00
else :
raise ValueError ( " Not told how to bisect " )
2018-03-03 06:42:52 -04:00
try :
bisecter . run ( )
except Exception as e :
print ( " Caught exception in bisect-helper: %s " % str ( e ) )
2019-03-13 02:53:56 -03:00
print ( get_exception_stacktrace ( e ) )
2018-03-03 06:42:52 -04:00
sys . exit ( 129 ) # should abort the bisect process