diff --git a/Tools/scripts/build_binaries.py b/Tools/scripts/build_binaries.py index 95e6097281..26db8f7e5b 100755 --- a/Tools/scripts/build_binaries.py +++ b/Tools/scripts/build_binaries.py @@ -20,12 +20,15 @@ import gzip # local imports import generate_manifest, gen_stable - +import build_binaries_history class build_binaries(object): def __init__(self, tags): self.tags = tags self.dirty = False + binaries_history_filepath = os.path.join(self.buildlogs_dirpath(), + "build_binaries_history.sqlite") + self.history = build_binaries_history.BuildBinariesHistory(binaries_history_filepath) def progress(self, string): '''pretty-print progress''' @@ -114,7 +117,11 @@ is bob we will attempt to checkout bob-AVR''' if ctag == "latest": vtag = "master" else: - vtag = "%s-%s" % (vehicle, ctag) + tagvehicle = vehicle + if tagvehicle == "Rover": + # FIXME: Rover tags in git still named APMrover2 :-( + tagvehicle = "APMrover2" + vtag = "%s-%s" % (tagvehicle, ctag) branches = [] if cframe is not None: @@ -331,9 +338,13 @@ is bob we will attempt to checkout bob-AVR''' self.progress("Building %s %s binaries (cwd=%s)" % (vehicle, tag, os.getcwd())) - for board in boards: + board_count = len(boards) + count = 0 + for board in sorted(boards, key=str.lower): now = datetime.datetime.now() - self.progress("Building board: %s at %s" % (board, str(now))) + count += 1 + self.progress("[%u/%u] Building board: %s at %s" % + (count, board_count, board, str(now))) for frame in frames: if frame is not None: self.progress("Considering frame %s for board %s" % @@ -373,6 +384,8 @@ is bob we will attempt to checkout bob-AVR''' self.remove_tmpdir() + githash = self.run_git(["rev-parse", "HEAD"]).rstrip() + t0 = time.time() self.progress("Configuring for %s in %s" % @@ -396,11 +409,16 @@ is bob we will attempt to checkout bob-AVR''' (vehicle, board, framesuffix, tag)) self.progress(msg) self.error_strings.append(msg) + # record some history about this build + t1 = time.time() + time_taken_to_build = t1-t0 + self.history.record_build(githash, tag, vehicle, board, frame, None, t0, time_taken_to_build) continue t1 = time.time() + time_taken_to_build = t1-t0 self.progress("Building %s %s %s %s took %u seconds" % - (vehicle, tag, board, frame, t1-t0)) + (vehicle, tag, board, frame, time_taken_to_build)) bare_path = os.path.join(self.buildroot, board, @@ -415,8 +433,10 @@ is bob we will attempt to checkout bob-AVR''' filepath = "".join([bare_path, extension]) if os.path.exists(filepath): files_to_copy.append(filepath) + if not os.path.exists(bare_path): + raise Exception("No elf file?!") # only copy the elf if we don't have other files to copy - if os.path.exists(bare_path) and len(files_to_copy) == 0: + if len(files_to_copy) == 0: files_to_copy.append(bare_path) for path in files_to_copy: @@ -428,6 +448,9 @@ is bob we will attempt to checkout bob-AVR''' self.touch_filepath(os.path.join(self.binaries, vehicle_binaries_subdir, tag)) + # record some history about this build + self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) + if not self.checkout(vehicle, tag, "PX4", None): self.checkout(vehicle, "latest") return @@ -541,6 +564,7 @@ is bob we will attempt to checkout bob-AVR''' "MatekF405-STD", "MatekF405-Wing", "MatekF765-Wing", + "MatekH743", "OMNIBUSF7V2", "sparky2", "omnibusf4", @@ -559,13 +583,17 @@ is bob we will attempt to checkout bob-AVR''' "PH4-mini", "CUAVv5", "CUAVv5Nano", + "CUAV-Nora", + "CUAV-X7", "mRoX21", "Pixracer", "F4BY", "mRoX21-777", "mRoControlZeroF7", + "mRoNexus", "F35Lightning", "speedybeef4", + "SuccexF4", "DrotekP3Pro", "VRBrain-v51", "VRBrain-v52", @@ -576,6 +604,7 @@ is bob we will attempt to checkout bob-AVR''' "Durandal", "CubeOrange", "CubeYellow", + "R9Pilot", # SITL targets "SITL_x86_64_linux_gnu", "SITL_arm_linux_gnueabihf", @@ -586,6 +615,10 @@ is bob we will attempt to checkout bob-AVR''' return ["f103-GPS", "f103-ADSB", "f103-RangeFinder", + "f303-GPS", + "f303-Universal", + "f303-M10025", + "f303-M10070", "CUAV_GPS", "ZubaxGNSS", ] @@ -628,11 +661,11 @@ is bob we will attempt to checkout bob-AVR''' '''build Rover binaries''' boards = self.common_boards() self.build_vehicle(tag, - "APMrover2", + "Rover", boards, "Rover", "ardurover", - "APMrover2") + "Rover") def build_ardusub(self, tag): '''build Sub binaries''' @@ -657,7 +690,7 @@ is bob we will attempt to checkout bob-AVR''' def generate_manifest(self): '''generate manigest files for GCS to download''' self.progress("Generating manifest") - base_url = 'http://firmware.ardupilot.org' + base_url = 'https://firmware.ardupilot.org' generator = generate_manifest.ManifestGenerator(self.binaries, base_url) content = generator.json() @@ -756,12 +789,14 @@ is bob we will attempt to checkout bob-AVR''' "binaries.build") for tag in self.tags: + t0 = time.time() self.build_arducopter(tag) self.build_arduplane(tag) self.build_rover(tag) self.build_antennatracker(tag) self.build_ardusub(tag) self.build_AP_Periph(tag) + self.history.record_run(githash, tag, t0, time.time()-t0) if os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) diff --git a/Tools/scripts/configure_all.py b/Tools/scripts/configure_all.py index 93e1b40038..a8d9b04632 100755 --- a/Tools/scripts/configure_all.py +++ b/Tools/scripts/configure_all.py @@ -16,21 +16,27 @@ parser.add_argument('--build', action='store_true', default=False, help='build a parser.add_argument('--build-target', default='copter', help='build target') parser.add_argument('--stop', action='store_true', default=False, help='stop on build fail') parser.add_argument('--no-bl', action='store_true', default=False, help="don't check bootloader builds") +parser.add_argument('--Werror', action='store_true', default=False, help="build with -Werror") parser.add_argument('--pattern', default='*') +parser.add_argument('--start', default=None, type=int, help='continue from specified build number') parser.add_argument('--python', default='python') args = parser.parse_args() os.environ['PYTHONUNBUFFERED'] = '1' failures = [] +done = [] +board_list = [] def get_board_list(): '''add boards based on existance of hwdef-bl.dat in subdirectories for ChibiOS''' board_list = [] + # these are base builds, and don't build directly + omit = ['f103-periph', 'f303-periph'] dirname, dirlist, filenames = next(os.walk('libraries/AP_HAL_ChibiOS/hwdef')) for d in dirlist: hwdef = os.path.join(dirname, d, 'hwdef.dat') - if os.path.exists(hwdef): + if os.path.exists(hwdef) and not d in omit: board_list.append(d) return board_list @@ -38,17 +44,30 @@ def run_program(cmd_list, build): print("Running (%s)" % " ".join(cmd_list)) retcode = subprocess.call(cmd_list) if retcode != 0: - print("Build failed: %s %s" % (build, ' '.join(cmd_list))) + print("FAILED BUILD: %s %s" % (build, ' '.join(cmd_list))) global failures failures.append(build) if args.stop: sys.exit(1) -for board in get_board_list(): +for board in sorted(get_board_list()): if not fnmatch.fnmatch(board, args.pattern): continue - print("Configuring for %s" % board) - run_program([args.python, "waf", "configure", "--board", board], "configure: " + board) + board_list.append(board) + +if args.start is not None: + if args.start < 1 or args.start >= len(board_list): + print("Invalid start %u for %u boards" % (args.start, len(board_list))) + sys.exit(1) + board_list = board_list[args.start-1:] + +for board in board_list: + done.append(board) + print("Configuring for %s [%u/%u failed=%u]" % (board, len(done), len(board_list), len(failures))) + config_opts = ["--board", board] + if args.Werror: + config_opts += ["--Werror"] + run_program([args.python, "waf", "configure"] + config_opts, "configure: " + board) if args.build: if board == "iomcu": target = "iofirmware" @@ -56,7 +75,10 @@ for board in get_board_list(): target = "AP_Periph" else: target = args.build_target - run_program([args.python, "waf", target], "build: " + board) + if target.find('/') != -1: + run_program([args.python, "waf", "--target", target], "build: " + board) + else: + run_program([args.python, "waf", target], "build: " + board) if args.no_bl: continue # check for bootloader def diff --git a/Tools/scripts/decode_devid.py b/Tools/scripts/decode_devid.py index 7c7bda0eab..054570b158 100755 --- a/Tools/scripts/decode_devid.py +++ b/Tools/scripts/decode_devid.py @@ -93,6 +93,13 @@ if opts.compass: if opts.imu: decoded_devname = imu_types.get(devtype, "UNKNOWN") -print("bus_type:%s(%u) bus:%u address:%u(0x%x) devtype:%u(0x%x) %s" % ( - bustypes.get(bus_type,"UNKNOWN"), bus_type, - bus, address, address, devtype, devtype, decoded_devname)) + +if bus_type == 3: + #uavcan devtype represents sensor_id + print("bus_type:%s(%u) bus:%u address:%u(0x%x) sensor_id:%u(0x%x) %s" % ( + bustypes.get(bus_type,"UNKNOWN"), bus_type, + bus, address, address, devtype-1, devtype-1, decoded_devname)) +else: + print("bus_type:%s(%u) bus:%u address:%u(0x%x) devtype:%u(0x%x) %s" % ( + bustypes.get(bus_type,"UNKNOWN"), bus_type, + bus, address, address, devtype, devtype, decoded_devname)) diff --git a/Tools/scripts/generate_manifest.py b/Tools/scripts/generate_manifest.py index c3a947a759..b054f6e75a 100755 --- a/Tools/scripts/generate_manifest.py +++ b/Tools/scripts/generate_manifest.py @@ -20,6 +20,7 @@ brand_map = { 'PH4-mini' : ('Pixhawk 4 Mini', 'Holybro'), 'KakuteF4' : ('KakuteF4', 'Holybro'), 'KakuteF7' : ('KakuteF7', 'Holybro'), + 'KakuteF7Mini' : ('KakuteF7Mini', 'Holybro'), 'CubeBlack' : ('CubeBlack', 'Hex/ProfiCNC'), 'CubeYellow' : ('CubeYellow', 'Hex/ProfiCNC'), 'CubeOrange' : ('CubeOrange', 'Hex/ProfiCNC'), @@ -28,6 +29,8 @@ brand_map = { 'CubeGreen-solo' : ('CubeGreen Solo', 'Hex/ProfiCNC'), 'CUAVv5' : ('CUAVv5', 'CUAV'), 'CUAVv5Nano' : ('CUAVv5 Nano', 'CUAV'), + 'CUAV-Nora' : ('CUAV Nora', 'CUAV'), + 'CUAV-X7' : ('CUAV X7', 'CUAV'), 'DrotekP3Pro' : ('Pixhawk 3 Pro', 'Drotek'), 'MatekF405' : ('Matek F405', 'Matek'), 'MatekF405-STD' : ('Matek F405 STD', 'Matek'), @@ -37,6 +40,7 @@ brand_map = { 'Pixracer' : ('PixRacer', 'mRobotics'), 'mRoX21' : ('mRo X2.1', 'mRobotics'), 'mRoX21-777' : ('mRo X2.1-777', 'mRobotics'), + 'mRoNexus' : ('mRo Nexus', 'mRobotics'), 'TBS-Colibri-F7' : ('Colibri F7', 'TBS'), 'sparky2' : ('Sparky2', 'TauLabs'), 'mindpx-v2' : ('MindPX V2', 'AirMind'), @@ -188,7 +192,7 @@ class ManifestGenerator(): } if 'USBID' in apj_json: # newer APJ files have USBID in the json data - firmware['USBID'] = apj_json['USBID'] + firmware['USBID'] = [apj_json['USBID']] elif platform in USBID_MAP: firmware['USBID'] = USBID_MAP[platform] else: @@ -224,6 +228,11 @@ class ManifestGenerator(): (brand_name, manufacturer) = brand_map[platform] firmware['brand_name'] = brand_name firmware['manufacturer'] = manufacturer + # copy over some extra information if available + extra_tags = [ 'image_size' ] + for tag in extra_tags: + if tag in apj_json: + firmware[tag] = apj_json[tag] def add_USB_IDs(self, firmware): '''add USB IDs to a firmware''' @@ -329,7 +338,7 @@ class ManifestGenerator(): filepath = os.path.join(some_dir, filename) firmware_format = self.firmware_format_for_filepath(filepath) - if firmware_format not in [ "ELF", "abin", "apj", "hex", "px4" ]: + if firmware_format not in [ "ELF", "abin", "apj", "hex", "px4", "bin" ]: print("Unknown firmware format (%s)" % firmware_format) firmware = Firmware() @@ -471,7 +480,7 @@ class ManifestGenerator(): file=sys.stderr) structure = self.walk_directory(self.basedir) - return json.dumps(structure, indent=4) + return json.dumps(structure, indent=4, separators=(',', ': ')) def usage(): @@ -484,7 +493,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description='generate manifest.json') parser.add_argument('--outfile', type=str, default=None, help='output file, default stdout') - parser.add_argument('--baseurl', type=str, default="http://firmware.ardupilot.org", help='base binaries directory') + parser.add_argument('--baseurl', type=str, default="https://firmware.ardupilot.org", help='base binaries directory') parser.add_argument('basedir', type=str, default="-", help='base binaries directory') args = parser.parse_args()