autotest: add option to retry tests / create and use Test definition object

Tools/autotest/arducopter.py

@@ -19,6 +19,7 @@ from pysim import vehicleinfo
 
 from common import AutoTest
 from common import NotAchievedException, AutoTestTimeoutException, PreconditionFailedException
+from common import Test
 
 # get location of scripts
 testdir = os.path.dirname(os.path.realpath(__file__))
@@ -6273,61 +6274,62 @@ class AutoTestCopter(AutoTest):
     def tests2b(self): #this block currently around 9.5mins here
         '''return list of all tests'''
         ret = ([
-            ("MotorVibration",
+            Test("MotorVibration",
-             "Fly motor vibration test",
+                 "Fly motor vibration test",
-             self.fly_motor_vibration), 
+                 self.fly_motor_vibration), 
 
-            ("DynamicNotches",
+            Test("DynamicNotches",
-             "Fly Dynamic Notches",
+                 "Fly Dynamic Notches",
-             self.fly_dynamic_notches), 
+                 self.fly_dynamic_notches), 
 
-            ("GyroFFT",
+            Test("GyroFFT",
-             "Fly Gyro FFT",
+                 "Fly Gyro FFT",
-             self.fly_gyro_fft),
+                 self.fly_gyro_fft,
- 
+                 attempts=4),
-            ("GyroFFTHarmonic",
-             "Fly Gyro FFT Harmonic Matching",
-             self.fly_gyro_fft_harmonic),
 
-            ("CompassReordering",
+            Test("GyroFFTHarmonic",
-             "Test Compass reordering when priorities are changed",
+                 "Fly Gyro FFT Harmonic Matching",
-             self.test_mag_reordering),  # 40sec?
+                 self.fly_gyro_fft_harmonic),
 
-            ("CRSF",
+            Test("CompassReordering",
-             "Test RC CRSF",
+                 "Test Compass reordering when priorities are changed",
-             self.test_crsf), #20secs ish
+                 self.test_mag_reordering),  # 40sec?
 
-            ("MotorTest",
+            Test("CRSF",
-             "Run Motor Tests",
+                 "Test RC CRSF",
-             self.test_motortest), #20secs ish
+                 self.test_crsf), #20secs ish
 
-            ("AltEstimation",
+            Test("MotorTest",
-             "Test that Alt Estimation is mandatory for ALT_HOLD",
+                 "Run Motor Tests",
-             self.test_alt_estimate_prearm), #20secs ish
+                 self.test_motortest), #20secs ish
 
-            ("EKFSource",
+            Test("AltEstimation",
-             "Check EKF Source Prearms work",
+                 "Test that Alt Estimation is mandatory for ALT_HOLD",
-             self.test_ekf_source),
+                 self.test_alt_estimate_prearm), #20secs ish
 
-            ("DataFlash",
+            Test("EKFSource",
-             "Test DataFlash Block backend",
+                 "Check EKF Source Prearms work",
-             self.test_dataflash_sitl),
+                 self.test_ekf_source),
 
-            ("DataFlashErase",
+            Test("DataFlash",
-             "Test DataFlash Block backend erase",
+                 "Test DataFlash Block backend",
-             self.test_dataflash_erase),
+                 self.test_dataflash_sitl),
 
-            ("Callisto",
+            Test("DataFlashErase",
-             "Test Callisto",
+                 "Test DataFlash Block backend erase",
-             self.test_callisto),
+                 self.test_dataflash_erase),
 
-            ("Replay",
+            Test("Callisto",
-             "Test Replay",
+                 "Test Callisto",
-             self.test_replay),
+                 self.test_callisto),
 
-            ("LogUpload",
+            Test("Replay",
-             "Log upload",
+                 "Test Replay",
-             self.log_upload),
+                 self.test_replay),
+
+            Test("LogUpload",
+                 "Log upload",
+                 self.log_upload),
         ])
         return ret
 

Tools/autotest/autotest.py

@@ -372,8 +372,12 @@ def run_specific_test(step, *args, **kwargs):
 
     print("Got %s" % str(tester))
     for a in tester.tests():
-        print("Got %s" % (a[0]))
+        if hasattr(a, 'name'):
-        if a[0] == test:
+            name = a.name
+        else:
+            name = a[0]
+        print("Got %s" % (name))
+        if name == test:
             return tester.run_tests([a])
     print("Failed to find test %s on %s" % (test, testname))
     sys.exit(1)

Tools/autotest/common.py

@@ -1104,6 +1104,14 @@ class LocationInt(object):
         self.alt = alt
         self.yaw = yaw
 
+class Test(object):
+    '''a test definition - information about a test'''
+    def __init__(self, name, description, function, attempts=1):
+        self.name = name
+        self.description = description
+        self.function = function
+        self.attempts = attempts
+
 class AutoTest(ABC):
     """Base abstract class.
     It implements the common function for all vehicle types.
@@ -4890,10 +4898,27 @@ Also, ignores heartbeats not from our target system'''
             util.run_cmd('/bin/cp build/sitl/bin/* %s' % to_dir,
                          directory=util.reltopdir('.'))
 
-    def run_one_test(self, name, desc, test_function, interact=False):
+    def run_one_test(self, test, interact=False):
         '''new-style run-one-test used by run_tests'''
-        test_output_filename = self.buildlogs_path("%s-%s.txt" %
+        for i in range(0, test.attempts-1):
-                                                   (self.log_name(), name))
+            if self.run_one_test_attempt(test, interact=interact, attempt=i+2, do_fail_list=False):
+                return
+            self.progress("Run attempt failed.  Retrying")
+        self.run_one_test_attempt(test, interact=interact, attempt=1)
+
+    def run_one_test_attempt(self, test, interact=False, attempt=1, do_fail_list=True):
+        '''called by run_one_test to actually run the test in a retry loop'''
+        name = test.name
+        desc = test.description
+        test_function = test.function
+        if attempt != 1:
+            self.progress("RETRYING %s" % name)
+            test_output_filename = self.buildlogs_path("%s-%s-retry-%u.txt" %
+                                                       (self.log_name(), name, attempt-1))
+        else:
+            test_output_filename = self.buildlogs_path("%s-%s.txt" %
+                                                       (self.log_name(), name))
+
         tee = TeeBoth(test_output_filename, 'w', self.mavproxy_logfile)
 
         prettyname = "%s (%s)" % (name, desc)
@@ -4958,7 +4983,8 @@ Also, ignores heartbeats not from our target system'''
         else:
             self.progress('FAILED: "%s": %s (see %s)' %
                           (prettyname, repr(ex), test_output_filename))
-            self.fail_list.append((prettyname, ex, test_output_filename))
+            if do_fail_list:
+                self.fail_list.append((prettyname, ex, test_output_filename))
             if interact:
                 self.progress("Starting MAVProxy interaction as directed")
                 self.mavproxy.interact()
@@ -4970,6 +4996,8 @@ Also, ignores heartbeats not from our target system'''
 
         tee.close()
 
+        return passed
+
     def check_test_syntax(self, test_file):
         """Check mistake on autotest function syntax."""
         self.start_test("Check for syntax mistake in autotest lambda")
@@ -7591,8 +7619,7 @@ switch value'''
         pass
 
     def test_skipped(self, test, reason):
-        (name, desc, func) = test
+        self.progress("##### %s is skipped: %s" % (test, reason))
-        self.progress("##### %s is skipped: %s" % (name, reason))
         self.skip_list.append((test, reason))
 
     def last_onboard_log(self):
@@ -7652,8 +7679,7 @@ switch value'''
                 self.clear_mission(mavutil.mavlink.MAV_MISSION_TYPE_ALL)
 
             for test in tests:
-                (name, desc, func) = test
+                self.run_one_test(test)
-                self.run_one_test(name, desc, func)
 
         except pexpect.TIMEOUT:
             self.progress("Failed with timeout")
@@ -7673,8 +7699,7 @@ switch value'''
             self.progress("Skipped tests:")
             for skipped in self.skip_list:
                 (test, reason) = skipped
-                (name, desc, func) = test
+                print("  %s (see %s)" % (test.name, reason))
-                print("  %s (see %s)" % (name, reason))
 
         if len(self.fail_list):
             self.progress("Failing tests:")
@@ -8985,43 +9010,43 @@ switch value'''
 
     def tests(self):
         return [
-            ("PIDTuning",
+            Test("PIDTuning",
-             "Test PID Tuning",
+                 "Test PID Tuning",
-             self.test_pid_tuning),
+                 self.test_pid_tuning),
 
-            ("ArmFeatures", "Arm features", self.test_arm_feature),
+            Test("ArmFeatures", "Arm features", self.test_arm_feature),
 
-            ("SetHome",
+            Test("SetHome",
-            "Test Set Home",
+                 "Test Set Home",
-             self.fly_test_set_home),
+                 self.fly_test_set_home),
 
-            ("ConfigErrorLoop",
+            Test("ConfigErrorLoop",
-             "Test Config Error Loop",
+                 "Test Config Error Loop",
-             self.test_config_error_loop),
+                 self.test_config_error_loop),
 
-            ("CPUFailsafe",
+            Test("CPUFailsafe",
-             "Ensure we do something appropriate when the main loop stops",
+                 "Ensure we do something appropriate when the main loop stops",
-             self.CPUFailsafe),
+                 self.CPUFailsafe),
 
-            ("Parameters",
+            Test("Parameters",
-             "Test Parameter Set/Get",
+                 "Test Parameter Set/Get",
-             self.test_parameters),
+                 self.test_parameters),
 
-            ("LoggerDocumentation",
+            Test("LoggerDocumentation",
-             "Test Onboard Logging Generation",
+                 "Test Onboard Logging Generation",
-             self.test_onboard_logging_generation),
+                 self.test_onboard_logging_generation),
 
-            ("Logging",
+            Test("Logging",
-             "Test Onboard Logging",
+                 "Teqst Onboard Logging",
-             self.test_onboard_logging),
+                 self.test_onboard_logging),
 
-            ("GetCapabilities",
+            Test("GetCapabilities",
-             "Get Capabilities",
+                 "Get Capabilities",
-             self.test_get_autopilot_capabilities),
+                 self.test_get_autopilot_capabilities),
 
-            ("InitialMode",
+            Test("InitialMode",
-             "Test initial mode switching",
+                 "Test initial mode switching",
-             self.test_initial_mode),
+                 self.test_initial_mode),
         ]
 
     def post_tests_announcements(self):
@@ -9041,13 +9066,20 @@ switch value'''
 
     def autotest(self):
         """Autotest used by ArduPilot autotest CI."""
-        all_tests = self.tests()
+        all_tests = []
+        for test in self.tests():
+            if type(test) == Test:
+                all_tests.append(test)
+                continue
+            (name, desc, func) = test
+            actual_test = Test(name, desc, func)
+            all_tests.append(actual_test)
+
         disabled = self.disabled_tests()
         tests = []
         for test in all_tests:
-            (name, desc, func) = test
+            if test.name in disabled:
-            if name in disabled:
+                self.test_skipped(test, disabled[test.name])
-                self.test_skipped(test, disabled[name])
                 continue
             tests.append(test)