ardupilot/Tools/scripts/size_compare_branches.py

#!/usr/bin/env python3

'''
Wrapper around elf_diff (https://github.com/noseglasses/elf_diff)
to create a html report comparing an ArduPilot build across two
branches

pip3 install --user elf_diff weasyprint

AP_FLAKE8_CLEAN

How to use?
Starting in the ardupilot directory.
~/ardupilot $ python Tools/scripts/size_compare_branches.py --branch=[PR_BRANCH_NAME] --vehicle=copter

Output is placed into ../ELF_DIFF_[VEHICLE_NAME]
'''

import copy
import optparse
import os
import shutil
import string
import subprocess
import sys
import tempfile
import time
import board_list

if sys.version_info[0] < 3:
    running_python3 = False
else:
    running_python3 = True


class SizeCompareBranchesResult(object):
    '''object to return results from a comparison'''

    def __init__(self, board, vehicle, bytes_delta, identical):
        self.board = board
        self.vehicle = vehicle
        self.bytes_delta = bytes_delta
        self.identical = identical


class SizeCompareBranches(object):
    '''script to build and compare branches using elf_diff'''

    def __init__(self,
                 branch=None,
                 master_branch="master",
                 board=["MatekF405-Wing"],
                 vehicle=["plane"],
                 bin_dir=None,
                 run_elf_diff=True,
                 all_vehicles=False,
                 all_boards=False,
                 use_merge_base=True,
                 waf_consistent_builds=True,
                 show_empty=True,
                 extra_hwdef=[],
                 extra_hwdef_branch=[],
                 extra_hwdef_master=[]):
        if branch is None:
            branch = self.find_current_git_branch()

        self.master_branch = master_branch
        self.branch = branch
        self.board = board
        self.vehicle = vehicle
        self.bin_dir = bin_dir
        self.run_elf_diff = run_elf_diff
        self.extra_hwdef = extra_hwdef
        self.extra_hwdef_branch = extra_hwdef_branch
        self.extra_hwdef_master = extra_hwdef_master
        self.all_vehicles = all_vehicles
        self.all_boards = all_boards
        self.use_merge_base = use_merge_base
        self.waf_consistent_builds = waf_consistent_builds
        self.show_empty = show_empty

        if self.bin_dir is None:
            self.bin_dir = self.find_bin_dir()

        self.boards_by_name = {}
        for board in board_list.BoardList().boards:
            self.boards_by_name[board.name] = board

        # map from vehicle names to binary names
        self.vehicle_map = {
            "rover"     : "ardurover",
            "copter"    : "arducopter",
            "plane"     : "arduplane",
            "sub"       : "ardusub",
            "heli"      : "arducopter-heli",
            "blimp"     : "blimp",
            "antennatracker" : "antennatracker",
            "AP_Periph" : "AP_Periph",
            "bootloader": "AP_Bootloader",
            "iofirmware": "iofirmware_highpolh",  # FIXME: lowpolh?
        }

        if all_boards:
            self.board = sorted(list(self.boards_by_name.keys()), key=lambda x: x.lower())
        else:
            # validate boards
            all_boards = set(self.boards_by_name.keys())
            for b in self.board:
                if b not in all_boards:
                    raise ValueError("Bad board %s" % str(b))

        if all_vehicles:
            self.vehicle = sorted(list(self.vehicle_map.keys()), key=lambda x: x.lower())
        else:
            for v in self.vehicle:
                if v not in self.vehicle_map.keys():
                    raise ValueError("Bad vehicle (%s); choose from %s" % (v, ",".join(self.vehicle_map.keys())))

        # some boards we don't have a -bl.dat for, so skip them.
        # TODO: find a way to get this information from board_list:
        self.bootloader_blacklist = set([
            'CubeOrange-SimOnHardWare',
            'fmuv2',
            'fmuv3-bdshot',
            'iomcu',
            'iomcu',
            'iomcu_f103_8MHz',
            'luminousbee4',
            'skyviper-v2450',
            'skyviper-f412-rev1',
            'skyviper-journey',
            'Pixhawk1-1M-bdshot',
            'SITL_arm_linux_gnueabihf',
        ])

        # blacklist all linux boards for bootloader build:
        self.bootloader_blacklist.update(self.linux_board_names())
        # ... and esp32 boards:
        self.bootloader_blacklist.update(self.esp32_board_names())

    def linux_board_names(self):
        '''return a list of all Linux board names; FIXME: get this dynamically'''
        # grep 'class.*[(]linux' Tools/ardupilotwaf/boards.py  | perl -pe "s/class (.*)\(linux\).*/            '\\1',/"
        return [
            'navigator',
            'erleboard',
            'navio',
            'navio2',
            'edge',
            'zynq',
            'ocpoc_zynq',
            'bbbmini',
            'blue',
            'pocket',
            'pxf',
            'bebop',
            'vnav',
            'disco',
            'erlebrain2',
            'bhat',
            'dark',
            'pxfmini',
            'aero',
            'rst_zynq',
            'obal',
            'SITL_x86_64_linux_gnu',
        ]

    def esp32_board_names(self):
        return [
            'esp32buzz',
            'esp32empty',
            'esp32icarous',
            'esp32diy',
        ]

    def find_bin_dir(self):
        '''attempt to find where the arm-none-eabi tools are'''
        binary = shutil.which("arm-none-eabi-g++")
        if binary is None:
            raise Exception("No arm-none-eabi-g++?")
        return os.path.dirname(binary)

    # vast amounts of stuff copied into here from build_binaries.py

    def run_program(self, prefix, cmd_list, show_output=True, env=None):
        if show_output:
            self.progress("Running (%s)" % " ".join(cmd_list))
        p = subprocess.Popen(
            cmd_list,
            stdin=None,
            stdout=subprocess.PIPE,
            close_fds=True,
            stderr=subprocess.STDOUT,
            env=env)
        output = ""
        while True:
            x = p.stdout.readline()
            if len(x) == 0:
                returncode = os.waitpid(p.pid, 0)
                if returncode:
                    break
                    # select not available on Windows... probably...
                time.sleep(0.1)
                continue
            if running_python3:
                x = bytearray(x)
                x = filter(lambda x : chr(x) in string.printable, x)
                x = "".join([chr(c) for c in x])
            output += x
            x = x.rstrip()
            if show_output:
                print("%s: %s" % (prefix, x))
        (_, status) = returncode
        if status != 0:
            self.progress("Process failed (%s)" %
                          str(returncode))
            raise subprocess.CalledProcessError(
                returncode, cmd_list)
        return output

    def find_current_git_branch(self):
        output = self.run_git(["symbolic-ref", "--short", "HEAD"])
        output = output.strip()
        return output

    def find_git_branch_merge_base(self, branch, master_branch):
        output = self.run_git(["merge-base", branch, master_branch])
        output = output.strip()
        return output

    def run_git(self, args):
        '''run git with args git_args; returns git's output'''
        cmd_list = ["git"]
        cmd_list.extend(args)
        return self.run_program("SCB-GIT", cmd_list)

    def run_waf(self, args, compiler=None):
        # try to modify the environment so we can consistent builds:
        consistent_build_envs = {
            "CHIBIOS_GIT_VERSION": "12345678",
            "GIT_VERSION": "abcdef",
            "GIT_VERSION_INT": "15",
        }
        for (n, v) in consistent_build_envs.items():
            os.environ[n] = v

        if os.path.exists("waf"):
            waf = "./waf"
        else:
            waf = os.path.join(".", "modules", "waf", "waf-light")
        cmd_list = [waf]
        cmd_list.extend(args)
        env = None
        if compiler is not None:
            # default to $HOME/arm-gcc, but allow for any path with AP_GCC_HOME environment variable
            gcc_home = os.environ.get("AP_GCC_HOME", os.path.join(os.environ["HOME"], "arm-gcc"))
            gcc_path = os.path.join(gcc_home, compiler, "bin")
            if os.path.exists(gcc_path):
                # setup PATH to point at the right compiler, and setup to use ccache
                env = os.environ.copy()
                env["PATH"] = gcc_path + ":" + env["PATH"]
                env["CC"] = "ccache arm-none-eabi-gcc"
                env["CXX"] = "ccache arm-none-eabi-g++"
            else:
                raise Exception("BB-WAF: Missing compiler %s" % gcc_path)
        self.run_program("SCB-WAF", cmd_list, env=env)

    def progress(self, string):
        '''pretty-print progress'''
        print("SCB: %s" % string)

    def build_branch_into_dir(self, board, branch, vehicle, outdir, extra_hwdef=None):
        self.run_git(["checkout", branch])
        self.run_git(["submodule", "update", "--recursive"])
        shutil.rmtree("build", ignore_errors=True)
        waf_configure_args = ["configure", "--board", board]
        if self.waf_consistent_builds:
            waf_configure_args.append("--consistent-builds")

        if extra_hwdef is not None:
            waf_configure_args.extend(["--extra-hwdef", extra_hwdef])

        self.run_waf(waf_configure_args)
        # we can't run `./waf copter blimp plane` without error, so do
        # them one-at-a-time:
        for v in vehicle:
            if v == 'bootloader':
                # need special configuration directive
                continue
            self.run_waf([v])
        for v in vehicle:
            if v != 'bootloader':
                continue
            if board in self.bootloader_blacklist:
                continue
            # need special configuration directive
            bootloader_waf_configure_args = copy.copy(waf_configure_args)
            bootloader_waf_configure_args.append('--bootloader')
            self.run_waf(bootloader_waf_configure_args)
            self.run_waf([v])
        self.run_program("rsync", ["rsync", "-aP", "build/", outdir])

    def run_all(self):
        '''run tests for boards and vehicles passed in constructor'''

        results = {}
        for board in self.board:
            vehicle_results = self.run_board(board)
            results[board] = vehicle_results
            with open("/tmp/some.csv", "w") as f:
                f.write(self.csv_for_results(results))

        return results

    def emit_csv_for_results(self, results):
        '''emit dictionary of dictionaries as a CSV'''
        print(self.csv_for_results(results))

    def csv_for_results(self, results):
        '''return a string with csv for results'''
        boards = sorted(results.keys())
        all_vehicles = set()
        for board in boards:
            all_vehicles.update(list(results[board].keys()))
        sorted_all_vehicles = sorted(list(all_vehicles))
        ret = ""
        ret += ",".join(["Board"] + sorted_all_vehicles) + "\n"
        for board in boards:
            line = [board]
            board_results = results[board]
            for vehicle in sorted_all_vehicles:
                bytes_delta = ""
                if vehicle in board_results:
                    result = board_results[vehicle]
                    if result.identical:
                        bytes_delta = "*"
                    else:
                        bytes_delta = result.bytes_delta
                line.append(str(bytes_delta))
            # do not add to ret value if we're not showing empty results:
            if not self.show_empty:
                if len(list(filter(lambda x : x != "", line[1:]))) == 0:
                    continue
            ret += ",".join(line) + "\n"
        return ret

    def run(self):
        results = self.run_all()
        self.emit_csv_for_results(results)

    def files_are_identical(self, file1, file2):
        '''returns true if the files have the same content'''
        return open(file1, "rb").read() == open(file2, "rb").read()

    def extra_hwdef_file(self, more):
        # create a combined list of hwdefs:
        extra_hwdefs = []
        extra_hwdefs.extend(self.extra_hwdef)
        extra_hwdefs.extend(more)
        extra_hwdefs = list(filter(lambda x : x is not None, extra_hwdefs))
        if len(extra_hwdefs) == 0:
            return None

        # slurp all content into a variable:
        content = bytearray()
        for extra_hwdef in extra_hwdefs:
            with open(extra_hwdef, "r+b") as f:
                content += f.read()

        # spew content to single file:
        f = tempfile.NamedTemporaryFile(delete=False)
        f.write(content)
        f.close()

        return f.name

    def run_board(self, board):
        ret = {}
        board_info = self.boards_by_name[board]

        vehicles_to_build = []
        for vehicle in self.vehicle:
            if vehicle == 'AP_Periph':
                if not board_info.is_ap_periph:
                    continue
            else:
                if board_info.is_ap_periph:
                    continue
                # the bootloader target isn't an autobuild target, so
                # it gets special treatment here:
                if vehicle != 'bootloader' and vehicle.lower() not in [x.lower() for x in board_info.autobuild_targets]:
                    continue
            vehicles_to_build.append(vehicle)
        if len(vehicles_to_build) == 0:
            return ret

        tmpdir = tempfile.mkdtemp()
        outdir_1 = os.path.join(tmpdir, "out-master-%s" % (board,))
        outdir_2 = os.path.join(tmpdir, "out-branch-%s" % (board,))

        self.progress("Building branch 1 (%s)" % self.master_branch)
        master_commit = self.master_branch
        if self.use_merge_base:
            master_commit = self.find_git_branch_merge_base(self.branch, self.master_branch)
            self.progress("Using merge base (%s)" % master_commit)
        shutil.rmtree(outdir_1, ignore_errors=True)

        self.build_branch_into_dir(
            board,
            master_commit,
            vehicles_to_build,
            outdir_1,
            extra_hwdef=self.extra_hwdef_file(self.extra_hwdef_master)
        )

        self.progress("Building branch 2 (%s)" % self.branch)
        shutil.rmtree(outdir_2, ignore_errors=True)
        self.build_branch_into_dir(
            board,
            self.branch,
            vehicles_to_build,
            outdir_2,
            self.extra_hwdef_file(self.extra_hwdef_branch)
        )

        for vehicle in vehicles_to_build:
            if vehicle == 'bootloader' and board in self.bootloader_blacklist:
                continue

            elf_filename = self.vehicle_map[vehicle]
            bin_filename = self.vehicle_map[vehicle] + '.bin'

            if self.run_elf_diff:
                master_elf_dirname = "bin"
                new_elf_dirname = "bin"
                if vehicle == 'bootloader':
                    # elfs for bootloaders are in the bootloader directory...
                    master_elf_dirname = "bootloader"
                    new_elf_dirname = "bootloader"
                master_elf_dir = os.path.join(outdir_1, board, master_elf_dirname)
                new_elf_dir = os.path.join(outdir_2, board, new_elf_dirname)
                self.progress("Starting compare (~10 minutes!)")
                elf_diff_commandline = [
                    "time",
                    "python3",
                    "-m", "elf_diff",
                    "--bin_dir", self.bin_dir,
                    '--bin_prefix=arm-none-eabi-',
                    "--old_alias", "%s %s" % (self.master_branch, elf_filename),
                    "--new_alias", "%s %s" % (self.branch, elf_filename),
                    "--html_dir", "../ELF_DIFF_%s_%s" % (board, vehicle),
                    os.path.join(master_elf_dir, elf_filename),
                    os.path.join(new_elf_dir, elf_filename)
                ]

                self.run_program("SCB", elf_diff_commandline)

            master_bin_dir = os.path.join(outdir_1, board, "bin")
            new_bin_dir = os.path.join(outdir_2, board, "bin")

            try:
                master_path = os.path.join(master_bin_dir, bin_filename)
                new_path = os.path.join(new_bin_dir, bin_filename)
                master_size = os.path.getsize(master_path)
                new_size = os.path.getsize(new_path)
            except FileNotFoundError:
                master_path = os.path.join(master_bin_dir, elf_filename)
                new_path = os.path.join(new_bin_dir, elf_filename)
                master_size = os.path.getsize(master_path)
                new_size = os.path.getsize(new_path)

            identical = self.files_are_identical(master_path, new_path)

            ret[vehicle] = SizeCompareBranchesResult(board, vehicle, new_size - master_size, identical)

        return ret


if __name__ == '__main__':
    parser = optparse.OptionParser("size_compare_branches.py")
    parser.add_option("",
                      "--elf-diff",
                      action="store_true",
                      default=False,
                      help="run elf_diff on output files")
    parser.add_option("",
                      "--master-branch",
                      type="string",
                      default="master",
                      help="master branch to use")
    parser.add_option("",
                      "--no-merge-base",
                      action="store_true",
                      default=False,
                      help="do not use the merge-base for testing, do a direct comparison between branches")
    parser.add_option("",
                      "--no-waf-consistent-builds",
                      action="store_true",
                      default=False,
                      help="do not use the --consistent-builds waf command-line option (for older branches)")
    parser.add_option("",
                      "--branch",
                      type="string",
                      default=None,
                      help="branch to compare")
    parser.add_option("",
                      "--vehicle",
                      action='append',
                      default=[],
                      help="vehicle to build for")
    parser.add_option("",
                      "--show-empty",
                      action='store_true',
                      default=False,
                      help="Show result lines even if no builds were done for the board")
    parser.add_option("",
                      "--board",
                      action='append',
                      default=[],
                      help="board to build for")
    parser.add_option("",
                      "--extra-hwdef",
                      default=[],
                      action="append",
                      help="configure with this extra hwdef file")
    parser.add_option("",
                      "--extra-hwdef-branch",
                      default=[],
                      action="append",
                      help="configure with this extra hwdef file only on new branch")
    parser.add_option("",
                      "--extra-hwdef-master",
                      default=[],
                      action="append",
                      help="configure with this extra hwdef file only on merge/master branch")
    parser.add_option("",
                      "--all-boards",
                      action='store_true',
                      default=False,
                      help="Build all boards")
    parser.add_option("",
                      "--all-vehicles",
                      action='store_true',
                      default=False,
                      help="Build all vehicles")
    cmd_opts, cmd_args = parser.parse_args()

    vehicle = []
    for v in cmd_opts.vehicle:
        vehicle.extend(v.split(','))
    if len(vehicle) == 0:
        vehicle.append("plane")

    board = []
    for b in cmd_opts.board:
        board.extend(b.split(','))
    if len(board) == 0:
        board.append("MatekF405-Wing")

    x = SizeCompareBranches(
        branch=cmd_opts.branch,
        master_branch=cmd_opts.master_branch,
        board=board,
        vehicle=vehicle,
        extra_hwdef=cmd_opts.extra_hwdef,
        extra_hwdef_branch=cmd_opts.extra_hwdef_branch,
        extra_hwdef_master=cmd_opts.extra_hwdef_master,
        run_elf_diff=(cmd_opts.elf_diff),
        all_vehicles=cmd_opts.all_vehicles,
        all_boards=cmd_opts.all_boards,
        use_merge_base=not cmd_opts.no_merge_base,
        waf_consistent_builds=not cmd_opts.no_waf_consistent_builds,
        show_empty=cmd_opts.show_empty,
    )
    x.run()