From 0dd9edb0474a2498ae0c5cb4d14552a479d0a4db Mon Sep 17 00:00:00 2001 From: Andy Piper Date: Thu, 31 Aug 2023 08:10:56 +0100 Subject: [PATCH] scripts: update size_compare from master --- Tools/scripts/size_compare_branches.py | 479 ++++++++++++++++++++----- 1 file changed, 384 insertions(+), 95 deletions(-) diff --git a/Tools/scripts/size_compare_branches.py b/Tools/scripts/size_compare_branches.py index 9e719910c1..886e387876 100755 --- a/Tools/scripts/size_compare_branches.py +++ b/Tools/scripts/size_compare_branches.py @@ -17,16 +17,24 @@ Output is placed into ../ELF_DIFF_[VEHICLE_NAME] ''' import copy +import fnmatch import optparse import os +import pathlib import shutil import string import subprocess import sys import tempfile +import threading import time import board_list +try: + import queue as Queue +except ImportError: + import Queue + if sys.version_info[0] < 3: running_python3 = False else: @@ -54,15 +62,20 @@ class SizeCompareBranches(object): bin_dir=None, run_elf_diff=True, all_vehicles=False, + exclude_board_glob=[], all_boards=False, use_merge_base=True, waf_consistent_builds=True, show_empty=True, + show_unchanged=True, extra_hwdef=[], extra_hwdef_branch=[], - extra_hwdef_master=[]): + extra_hwdef_master=[], + parallel_copies=None, + jobs=None): + if branch is None: - branch = self.find_current_git_branch() + branch = self.find_current_git_branch_or_sha1() self.master_branch = master_branch self.branch = branch @@ -78,6 +91,9 @@ class SizeCompareBranches(object): self.use_merge_base = use_merge_base self.waf_consistent_builds = waf_consistent_builds self.show_empty = show_empty + self.show_unchanged = show_unchanged + self.parallel_copies = parallel_copies + self.jobs = jobs if self.bin_dir is None: self.bin_dir = self.find_bin_dir() @@ -116,10 +132,23 @@ class SizeCompareBranches(object): if v not in self.vehicle_map.keys(): raise ValueError("Bad vehicle (%s); choose from %s" % (v, ",".join(self.vehicle_map.keys()))) + # remove boards based on --exclude-board-glob + new_self_board = [] + for board_name in self.board: + exclude = False + for exclude_glob in exclude_board_glob: + if fnmatch.fnmatch(board_name, exclude_glob): + exclude = True + break + if not exclude: + new_self_board.append(board_name) + self.board = new_self_board + # some boards we don't have a -bl.dat for, so skip them. # TODO: find a way to get this information from board_list: self.bootloader_blacklist = set([ 'CubeOrange-SimOnHardWare', + 'CubeOrangePlus-SimOnHardWare', 'fmuv2', 'fmuv3-bdshot', 'iomcu', @@ -130,7 +159,9 @@ class SizeCompareBranches(object): 'skyviper-f412-rev1', 'skyviper-journey', 'Pixhawk1-1M-bdshot', + 'Pixhawk1-bdshot', 'SITL_arm_linux_gnueabihf', + 'RADIX2HD', ]) # blacklist all linux boards for bootloader build: @@ -170,6 +201,9 @@ class SizeCompareBranches(object): return [ 'esp32buzz', 'esp32empty', + 'esp32tomte76', + 'esp32nick', + 'esp32s3devkit', 'esp32icarous', 'esp32diy', ] @@ -183,15 +217,21 @@ class SizeCompareBranches(object): # vast amounts of stuff copied into here from build_binaries.py - def run_program(self, prefix, cmd_list, show_output=True, env=None): - if show_output: - self.progress("Running (%s)" % " ".join(cmd_list)) + def run_program(self, prefix, cmd_list, show_output=True, env=None, show_output_on_error=True, show_command=None, cwd="."): + if show_command is None: + show_command = True + if show_command: + cmd = " ".join(cmd_list) + if cwd is None: + cwd = "." + self.progress(f"Running ({cmd}) in ({cwd})") p = subprocess.Popen( cmd_list, stdin=None, stdout=subprocess.PIPE, close_fds=True, stderr=subprocess.STDOUT, + cwd=cwd, env=env) output = "" while True: @@ -209,18 +249,33 @@ class SizeCompareBranches(object): x = "".join([chr(c) for c in x]) output += x x = x.rstrip() + some_output = "%s: %s" % (prefix, x) if show_output: - print("%s: %s" % (prefix, x)) + print(some_output) + else: + output += some_output (_, status) = returncode if status != 0: + if not show_output and show_output_on_error: + # we were told not to show output, but we just + # failed... so show output... + print(output) self.progress("Process failed (%s)" % str(returncode)) raise subprocess.CalledProcessError( returncode, cmd_list) return output - def find_current_git_branch(self): - output = self.run_git(["symbolic-ref", "--short", "HEAD"]) + def find_current_git_branch_or_sha1(self): + try: + output = self.run_git(["symbolic-ref", "--short", "HEAD"]) + output = output.strip() + return output + except subprocess.CalledProcessError: + pass + + # probably in a detached-head state. Get a sha1 instead: + output = self.run_git(["rev-parse", "--short", "HEAD"]) output = output.strip() return output @@ -229,13 +284,13 @@ class SizeCompareBranches(object): output = output.strip() return output - def run_git(self, args): + def run_git(self, args, show_output=True, source_dir=None): '''run git with args git_args; returns git's output''' cmd_list = ["git"] cmd_list.extend(args) - return self.run_program("SCB-GIT", cmd_list) + return self.run_program("SCB-GIT", cmd_list, show_output=show_output, cwd=source_dir) - def run_waf(self, args, compiler=None): + def run_waf(self, args, compiler=None, show_output=True, source_dir=None): # try to modify the environment so we can consistent builds: consistent_build_envs = { "CHIBIOS_GIT_VERSION": "12345678", @@ -264,16 +319,19 @@ class SizeCompareBranches(object): env["CXX"] = "ccache arm-none-eabi-g++" else: raise Exception("BB-WAF: Missing compiler %s" % gcc_path) - self.run_program("SCB-WAF", cmd_list, env=env) + self.run_program("SCB-WAF", cmd_list, env=env, show_output=show_output, cwd=source_dir) def progress(self, string): '''pretty-print progress''' print("SCB: %s" % string) - def build_branch_into_dir(self, board, branch, vehicle, outdir, extra_hwdef=None): - self.run_git(["checkout", branch]) - self.run_git(["submodule", "update", "--recursive"]) - shutil.rmtree("build", ignore_errors=True) + def build_branch_into_dir(self, board, branch, vehicle, outdir, source_dir=None, extra_hwdef=None, jobs=None): + self.run_git(["checkout", branch], show_output=False, source_dir=source_dir) + self.run_git(["submodule", "update", "--recursive"], show_output=False, source_dir=source_dir) + build_dir = "build" + if source_dir is not None: + build_dir = os.path.join(source_dir, "build") + shutil.rmtree(build_dir, ignore_errors=True) waf_configure_args = ["configure", "--board", board] if self.waf_consistent_builds: waf_configure_args.append("--consistent-builds") @@ -281,14 +339,22 @@ class SizeCompareBranches(object): if extra_hwdef is not None: waf_configure_args.extend(["--extra-hwdef", extra_hwdef]) - self.run_waf(waf_configure_args) + if self.run_elf_diff: + waf_configure_args.extend(["--debug-symbols"]) + + if jobs is None: + jobs = self.jobs + if jobs is not None: + waf_configure_args.extend(["-j", str(jobs)]) + + self.run_waf(waf_configure_args, show_output=False, source_dir=source_dir) # we can't run `./waf copter blimp plane` without error, so do # them one-at-a-time: for v in vehicle: if v == 'bootloader': # need special configuration directive continue - self.run_waf([v]) + self.run_waf([v], show_output=False, source_dir=source_dir) for v in vehicle: if v != 'bootloader': continue @@ -297,19 +363,233 @@ class SizeCompareBranches(object): # need special configuration directive bootloader_waf_configure_args = copy.copy(waf_configure_args) bootloader_waf_configure_args.append('--bootloader') - self.run_waf(bootloader_waf_configure_args) - self.run_waf([v]) - self.run_program("rsync", ["rsync", "-aP", "build/", outdir]) + # hopefully temporary hack so you can build bootloader + # after building other vehicles without a clean: + dsdl_generated_path = os.path.join('build', board, "modules", "DroneCAN", "libcanard", "dsdlc_generated") + self.progress("HACK: Removing (%s)" % dsdl_generated_path) + if source_dir is not None: + dsdl_generated_path = os.path.join(source_dir, dsdl_generated_path) + shutil.rmtree(dsdl_generated_path, ignore_errors=True) + self.run_waf(bootloader_waf_configure_args, show_output=False, source_dir=source_dir) + self.run_waf([v], show_output=False, source_dir=source_dir) + self.run_program("rsync", ["rsync", "-ap", "build/", outdir], cwd=source_dir) + if source_dir is not None: + pathlib.Path(outdir, "scb_sourcepath.txt").write_text(source_dir) + + def vehicles_to_build_for_board_info(self, board_info): + vehicles_to_build = [] + for vehicle in self.vehicle: + if vehicle == 'AP_Periph': + if not board_info.is_ap_periph: + continue + else: + if board_info.is_ap_periph: + continue + # the bootloader target isn't an autobuild target, so + # it gets special treatment here: + if vehicle != 'bootloader' and vehicle.lower() not in [x.lower() for x in board_info.autobuild_targets]: + continue + vehicles_to_build.append(vehicle) + + return vehicles_to_build + + def parallel_thread_main(self, thread_number): + # initialisation; make a copy of the source directory + my_source_dir = os.path.join(self.tmpdir, f"thread-{thread_number}-source") + self.run_program("rsync", [ + "rsync", + "--exclude=build/", + "-ap", + "./", + my_source_dir + ]) + + while True: + try: + task = self.parallel_tasks.pop(0) + except IndexError: + break + jobs = None + if self.jobs is not None: + jobs = int(self.jobs / self.num_threads_remaining) + if jobs <= 0: + jobs = 1 + try: + self.run_build_task(task, source_dir=my_source_dir, jobs=jobs) + except Exception as ex: + self.thread_exit_result_queue.put(f"{task}") + raise ex + + def check_result_queue(self): + while True: + try: + result = self.thread_exit_result_queue.get_nowait() + except Queue.Empty: + break + if result is None: + continue + self.failure_exceptions.append(result) + + def run_build_tasks_in_parallel(self, tasks): + n_threads = self.parallel_copies + if len(tasks) < n_threads: + n_threads = len(tasks) + self.num_threads_remaining = n_threads + + # shared list for the threads: + self.parallel_tasks = copy.copy(tasks) # make this an argument instead?! + threads = [] + self.thread_exit_result_queue = Queue.Queue() + for i in range(0, n_threads): + t = threading.Thread( + target=self.parallel_thread_main, + name=f'task-builder-{i}', + args=[i], + ) + t.start() + threads.append(t) + tstart = time.time() + self.failure_exceptions = [] + + while len(threads): + + self.check_result_queue() + + new_threads = [] + for thread in threads: + thread.join(0) + if thread.is_alive(): + new_threads.append(thread) + threads = new_threads + self.num_threads_remaining = len(threads) + self.progress( + f"remaining-tasks={len(self.parallel_tasks)} " + + f"remaining-threads={len(threads)} failed-threads={len(self.failure_exceptions)} elapsed={int(time.time() - tstart)}s") # noqa + + # write out a progress CSV: + task_results = [] + for task in tasks: + task_results.append(self.gather_results_for_task(task)) + # progress CSV: + with open("/tmp/some.csv", "w") as f: + f.write(self.csv_for_results(self.compare_task_results(task_results, no_elf_diff=True))) + + time.sleep(1) + self.progress("All threads returned") + + self.check_result_queue() + + if len(self.failure_exceptions): + self.progress("Some threads failed:") + for ex in self.failure_exceptions: + print("Thread failure: %s" % str(ex)) def run_all(self): '''run tests for boards and vehicles passed in constructor''' - results = {} + tmpdir = tempfile.mkdtemp() + self.tmpdir = tmpdir + + self.master_commit = self.master_branch + if self.use_merge_base: + self.master_commit = self.find_git_branch_merge_base(self.branch, self.master_branch) + self.progress("Using merge base (%s)" % self.master_commit) + + # create an array of tasks to run: + tasks = [] for board in self.board: - vehicle_results = self.run_board(board) - results[board] = vehicle_results - with open("/tmp/some.csv", "w") as f: - f.write(self.csv_for_results(results)) + board_info = self.boards_by_name[board] + + vehicles_to_build = self.vehicles_to_build_for_board_info(board_info) + + outdir_1 = os.path.join(tmpdir, "out-master-%s" % (board,)) + tasks.append((board, self.master_commit, outdir_1, vehicles_to_build, self.extra_hwdef_master)) + outdir_2 = os.path.join(tmpdir, "out-branch-%s" % (board,)) + tasks.append((board, self.branch, outdir_2, vehicles_to_build, self.extra_hwdef_branch)) + self.tasks = tasks + + if self.parallel_copies is not None: + self.run_build_tasks_in_parallel(tasks) + task_results = [] + for task in tasks: + task_results.append(self.gather_results_for_task(task)) + else: + # traditional build everything in sequence: + task_results = [] + for task in tasks: + self.run_build_task(task) + task_results.append(self.gather_results_for_task(task)) + + # progress CSV: + with open("/tmp/some.csv", "w") as f: + f.write(self.csv_for_results(self.compare_task_results(task_results, no_elf_diff=True))) + + return self.compare_task_results(task_results) + + def elf_diff_results(self, result_master, result_branch): + master_branch = result_master["branch"] + branch = result_branch["branch"] + for vehicle in result_master["vehicle"].keys(): + elf_filename = result_master["vehicle"][vehicle]["elf_filename"] + master_elf_dir = result_master["vehicle"][vehicle]["elf_dir"] + new_elf_dir = result_branch["vehicle"][vehicle]["elf_dir"] + board = result_master["board"] + self.progress("Starting compare (~10 minutes!)") + elf_diff_commandline = [ + "time", + "python3", + "-m", "elf_diff", + "--bin_dir", self.bin_dir, + '--bin_prefix=arm-none-eabi-', + "--old_alias", "%s %s" % (master_branch, elf_filename), + "--new_alias", "%s %s" % (branch, elf_filename), + "--html_dir", "../ELF_DIFF_%s_%s" % (board, vehicle), + ] + + try: + master_source_prefix = result_master["vehicle"][vehicle]["source_path"] + branch_source_prefix = result_branch["vehicle"][vehicle]["source_path"] + elf_diff_commandline.extend([ + "--old_source_prefix", master_source_prefix, + "--new_source_prefix", branch_source_prefix, + ]) + except KeyError: + pass + + elf_diff_commandline.extend([ + os.path.join(master_elf_dir, elf_filename), + os.path.join(new_elf_dir, elf_filename) + ]) + + self.run_program("SCB", elf_diff_commandline) + + def compare_task_results(self, task_results, no_elf_diff=False): + # pair off results, master and branch: + pairs = {} + for res in task_results: + board = res["board"] + if board not in pairs: + pairs[board] = {} + if res["branch"] == self.master_commit: + pairs[board]["master"] = res + elif res["branch"] == self.branch: + pairs[board]["branch"] = res + else: + raise ValueError(res["branch"]) + + results = {} + for pair in pairs.values(): + if "master" not in pair or "branch" not in pair: + # probably incomplete: + continue + master = pair["master"] + board = master["board"] + try: + results[board] = self.compare_results(master, pair["branch"]) + if self.run_elf_diff and not no_elf_diff: + self.elf_diff_results(master, pair["branch"]) + except FileNotFoundError: + pass return results @@ -342,6 +622,11 @@ class SizeCompareBranches(object): if not self.show_empty: if len(list(filter(lambda x : x != "", line[1:]))) == 0: continue + # do not add to ret value if all output binaries are identical: + if not self.show_unchanged: + starcount = len(list(filter(lambda x : x == "*", line[1:]))) + if len(line[1:]) == starcount: + continue ret += ",".join(line) + "\n" return ret @@ -375,96 +660,75 @@ class SizeCompareBranches(object): return f.name - def run_board(self, board): - ret = {} - board_info = self.boards_by_name[board] - - vehicles_to_build = [] - for vehicle in self.vehicle: - if vehicle == 'AP_Periph': - if not board_info.is_ap_periph: - continue - else: - if board_info.is_ap_periph: - continue - # the bootloader target isn't an autobuild target, so - # it gets special treatment here: - if vehicle != 'bootloader' and vehicle.lower() not in [x.lower() for x in board_info.autobuild_targets]: - continue - vehicles_to_build.append(vehicle) - if len(vehicles_to_build) == 0: - return ret - - tmpdir = tempfile.mkdtemp() - outdir_1 = os.path.join(tmpdir, "out-master-%s" % (board,)) - outdir_2 = os.path.join(tmpdir, "out-branch-%s" % (board,)) - - self.progress("Building branch 1 (%s)" % self.master_branch) - master_commit = self.master_branch - if self.use_merge_base: - master_commit = self.find_git_branch_merge_base(self.branch, self.master_branch) - self.progress("Using merge base (%s)" % master_commit) - shutil.rmtree(outdir_1, ignore_errors=True) + def run_build_task(self, task, source_dir=None, jobs=None): + (board, commitish, outdir, vehicles_to_build, extra_hwdef_file) = task + self.progress(f"Building {task}") + shutil.rmtree(outdir, ignore_errors=True) self.build_branch_into_dir( board, - master_commit, + commitish, vehicles_to_build, - outdir_1, - extra_hwdef=self.extra_hwdef_file(self.extra_hwdef_master) + outdir, + source_dir=source_dir, + extra_hwdef=self.extra_hwdef_file(extra_hwdef_file), + jobs=jobs, ) - self.progress("Building branch 2 (%s)" % self.branch) - shutil.rmtree(outdir_2, ignore_errors=True) - self.build_branch_into_dir( - board, - self.branch, - vehicles_to_build, - outdir_2, - self.extra_hwdef_file(self.extra_hwdef_branch) - ) + def gather_results_for_task(self, task): + (board, commitish, outdir, vehicles_to_build, extra_hwdef_file) = task + + result = { + "board": board, + "branch": commitish, + "vehicle": {}, + } + + have_source_trees = self.parallel_copies is not None and len(self.tasks) <= self.parallel_copies for vehicle in vehicles_to_build: if vehicle == 'bootloader' and board in self.bootloader_blacklist: continue - elf_filename = self.vehicle_map[vehicle] - bin_filename = self.vehicle_map[vehicle] + '.bin' + result["vehicle"][vehicle] = {} + v = result["vehicle"][vehicle] + v["bin_filename"] = self.vehicle_map[vehicle] + '.bin' - if self.run_elf_diff: - master_elf_dirname = "bin" - new_elf_dirname = "bin" - if vehicle == 'bootloader': - # elfs for bootloaders are in the bootloader directory... - master_elf_dirname = "bootloader" - new_elf_dirname = "bootloader" - master_elf_dir = os.path.join(outdir_1, board, master_elf_dirname) - new_elf_dir = os.path.join(outdir_2, board, new_elf_dirname) - self.progress("Starting compare (~10 minutes!)") - elf_diff_commandline = [ - "time", - "python3", - "-m", "elf_diff", - "--bin_dir", self.bin_dir, - '--bin_prefix=arm-none-eabi-', - "--old_alias", "%s %s" % (self.master_branch, elf_filename), - "--new_alias", "%s %s" % (self.branch, elf_filename), - "--html_dir", "../ELF_DIFF_%s_%s" % (board, vehicle), - os.path.join(master_elf_dir, elf_filename), - os.path.join(new_elf_dir, elf_filename) - ] + elf_dirname = "bin" + if vehicle == 'bootloader': + # elfs for bootloaders are in the bootloader directory... + elf_dirname = "bootloader" + elf_basedir = outdir + if have_source_trees: + try: + v["source_path"] = pathlib.Path(outdir, "scb_sourcepath.txt").read_text() + elf_basedir = os.path.join(v["source_path"], 'build') + self.progress("Have source trees") + except FileNotFoundError: + pass + v["bin_dir"] = os.path.join(elf_basedir, board, "bin") + elf_dir = os.path.join(elf_basedir, board, elf_dirname) + v["elf_dir"] = elf_dir + v["elf_filename"] = self.vehicle_map[vehicle] - self.run_program("SCB", elf_diff_commandline) + return result - master_bin_dir = os.path.join(outdir_1, board, "bin") - new_bin_dir = os.path.join(outdir_2, board, "bin") + def compare_results(self, result_master, result_branch): + ret = {} + for vehicle in result_master["vehicle"].keys(): + # check for the difference in size (and identicality) + # of the two binaries: + master_bin_dir = result_master["vehicle"][vehicle]["bin_dir"] + new_bin_dir = result_branch["vehicle"][vehicle]["bin_dir"] try: + bin_filename = result_master["vehicle"][vehicle]["bin_filename"] master_path = os.path.join(master_bin_dir, bin_filename) new_path = os.path.join(new_bin_dir, bin_filename) master_size = os.path.getsize(master_path) new_size = os.path.getsize(new_path) except FileNotFoundError: + elf_filename = result_master["vehicle"][vehicle]["elf_filename"] master_path = os.path.join(master_bin_dir, elf_filename) new_path = os.path.join(new_bin_dir, elf_filename) master_size = os.path.getsize(master_path) @@ -472,6 +736,7 @@ class SizeCompareBranches(object): identical = self.files_are_identical(master_path, new_path) + board = result_master["board"] ret[vehicle] = SizeCompareBranchesResult(board, vehicle, new_size - master_size, identical) return ret @@ -514,6 +779,11 @@ if __name__ == '__main__': action='store_true', default=False, help="Show result lines even if no builds were done for the board") + parser.add_option("", + "--hide-unchanged", + action='store_true', + default=False, + help="Hide binary-size-change results for any board where output binary is unchanged") parser.add_option("", "--board", action='append', @@ -539,11 +809,26 @@ if __name__ == '__main__': action='store_true', default=False, help="Build all boards") + parser.add_option("", + "--exclude-board-glob", + default=[], + action="append", + help="exclude any board which matches this pattern") parser.add_option("", "--all-vehicles", action='store_true', default=False, help="Build all vehicles") + parser.add_option("", + "--parallel-copies", + type=int, + default=None, + help="Copy source dir this many times, build from those copies in parallel") + parser.add_option("-j", + "--jobs", + type=int, + default=None, + help="Passed to waf configure -j; number of build jobs. If running with --parallel-copies, this is divided by the number of remaining threads before being passed.") # noqa cmd_opts, cmd_args = parser.parse_args() vehicle = [] @@ -569,8 +854,12 @@ if __name__ == '__main__': run_elf_diff=(cmd_opts.elf_diff), all_vehicles=cmd_opts.all_vehicles, all_boards=cmd_opts.all_boards, + exclude_board_glob=cmd_opts.exclude_board_glob, use_merge_base=not cmd_opts.no_merge_base, waf_consistent_builds=not cmd_opts.no_waf_consistent_builds, show_empty=cmd_opts.show_empty, + show_unchanged=not cmd_opts.hide_unchanged, + parallel_copies=cmd_opts.parallel_copies, + jobs=cmd_opts.jobs, ) x.run()