From d464db9c700dbfc4990b86558cff28788a8867d8 Mon Sep 17 00:00:00 2001 From: Peter Barker Date: Sat, 7 Mar 2020 13:41:41 +1100 Subject: [PATCH] Tools: create log of build_binaries.py builds --- Tools/autotest/autotest.py | 22 ++----- Tools/scripts/build_binaries.py | 23 ++++++- Tools/scripts/build_binaries_history.py | 87 +++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 18 deletions(-) create mode 100644 Tools/scripts/build_binaries_history.py diff --git a/Tools/autotest/autotest.py b/Tools/autotest/autotest.py index b3b8fab023..901baa6ff5 100755 --- a/Tools/autotest/autotest.py +++ b/Tools/autotest/autotest.py @@ -100,23 +100,15 @@ def build_all(): def build_binaries(): """Run the build_binaries.py script.""" print("Running build_binaries.py") - # copy the script as it changes git branch, which can change the - # script while running - orig = util.reltopdir('Tools/scripts/build_binaries.py') - copy = util.reltopdir('./build_binaries.py') - shutil.copy2(orig, copy) - # also copy generate_manifest library: - orig_gm = util.reltopdir('Tools/scripts/generate_manifest.py') - copy_gm = util.reltopdir('./generate_manifest.py') - shutil.copy2(orig_gm, copy_gm) - - # and gen_stable.py - orig_gs = util.reltopdir('Tools/scripts/gen_stable.py') - copy_gs = util.reltopdir('./gen_stable.py') - shutil.copy2(orig_gs, copy_gs) + # copy the script (and various libraries used by the script) as it + # changes git branch, which can change the script while running + for thing in "build_binaries.py", "generate_manifest.py", "gen_stable.py", "build_binaries_history.py": + orig = util.reltopdir('Tools/scripts/%s' % thing) + copy = util.reltopdir('./%s' % thing) + shutil.copy2(orig, copy) - if util.run_cmd(copy, directory=util.reltopdir('.')) != 0: + if util.run_cmd("./build_binaries.py", directory=util.reltopdir('.')) != 0: print("Failed build_binaries.py") return False return True diff --git a/Tools/scripts/build_binaries.py b/Tools/scripts/build_binaries.py index 54c5b58893..0656cdbcf4 100755 --- a/Tools/scripts/build_binaries.py +++ b/Tools/scripts/build_binaries.py @@ -20,12 +20,15 @@ import gzip # local imports import generate_manifest, gen_stable - +import build_binaries_history class build_binaries(object): def __init__(self, tags): self.tags = tags self.dirty = False + binaries_history_filepath = os.path.join(self.buildlogs_dirpath(), + "build_binaries_history.sqlite") + self.history = build_binaries_history.BuildBinariesHistory(binaries_history_filepath) def progress(self, string): '''pretty-print progress''' @@ -377,6 +380,8 @@ is bob we will attempt to checkout bob-AVR''' self.remove_tmpdir() + githash = self.run_git(["rev-parse", "HEAD"]).rstrip() + t0 = time.time() self.progress("Configuring for %s in %s" % @@ -400,11 +405,16 @@ is bob we will attempt to checkout bob-AVR''' (vehicle, board, framesuffix, tag)) self.progress(msg) self.error_strings.append(msg) + # record some history about this build + t1 = time.time() + time_taken_to_build = t1-t0 + self.history.record_build(githash, tag, vehicle, board, frame, None, t0, time_taken_to_build) continue t1 = time.time() + time_taken_to_build = t1-t0 self.progress("Building %s %s %s %s took %u seconds" % - (vehicle, tag, board, frame, t1-t0)) + (vehicle, tag, board, frame, time_taken_to_build)) bare_path = os.path.join(self.buildroot, board, @@ -419,8 +429,10 @@ is bob we will attempt to checkout bob-AVR''' filepath = "".join([bare_path, extension]) if os.path.exists(filepath): files_to_copy.append(filepath) + if not os.path.exists(bare_path): + raise Exception("No elf file?!") # only copy the elf if we don't have other files to copy - if os.path.exists(bare_path) and len(files_to_copy) == 0: + if len(files_to_copy) == 0: files_to_copy.append(bare_path) for path in files_to_copy: @@ -432,6 +444,9 @@ is bob we will attempt to checkout bob-AVR''' self.touch_filepath(os.path.join(self.binaries, vehicle_binaries_subdir, tag)) + # record some history about this build + self.history.record_build(githash, tag, vehicle, board, frame, bare_path, t0, time_taken_to_build) + if not self.checkout(vehicle, tag, "PX4", None): self.checkout(vehicle, "latest") return @@ -766,12 +781,14 @@ is bob we will attempt to checkout bob-AVR''' "binaries.build") for tag in self.tags: + t0 = time.time() self.build_arducopter(tag) self.build_arduplane(tag) self.build_rover(tag) self.build_antennatracker(tag) self.build_ardusub(tag) self.build_AP_Periph(tag) + self.history.record_run(githash, tag, t0, time.time()-t0) if os.path.exists(self.tmpdir): shutil.rmtree(self.tmpdir) diff --git a/Tools/scripts/build_binaries_history.py b/Tools/scripts/build_binaries_history.py new file mode 100644 index 0000000000..2ed9b0286d --- /dev/null +++ b/Tools/scripts/build_binaries_history.py @@ -0,0 +1,87 @@ +#!/usr/bin/python + +from __future__ import print_function + +import os +import sqlite3 + + +class BuildBinariesHistory(): + def __init__(self, db_filepath): + self.db_filepath = db_filepath + self.assure_db_present() + + def progress(self, msg): + print("BBHIST: %s" % msg) + + def conn(self): + return sqlite3.connect(self.db_filepath) + + def create_schema(self, c): + '''create our tables and whatnot''' + schema_version = 1 + c.execute("create table version (version integer)") + c.execute("insert into version (version) values (?)", (schema_version,)) + # at some stage we should probably directly associate build with runs.... + c.execute("create table build (hash text, tag text, vehicle text, board text, " + "frame text, text integer, data integer, bss integer, start_time real, duration real)") + c.execute("create table run (hash text, tag text, start_time real, duration real)") + + def sizes_for_file(self, filepath): + cmd = "size %s" % (filepath,) + stuff = os.popen(cmd).read() + lines = stuff.split("\n") + sizes = lines[1].split("\t") + text = int(sizes[0]) + data = int(sizes[1]) + bss = int(sizes[2]) + print("text=%u" % text) + print("data=%u" % data) + print("bss=%u" % bss) + return (text, data, bss) + + def assure_db_present(self): + c = self.conn() + need_schema_create = False + try: + version_cursor = c.execute("select version from version") + except sqlite3.OperationalError as e: + if "no such table" in str(e): # FIXME: do better here? what's in "e"? + print("need schema create") + need_schema_create = True + + if need_schema_create: + self.create_schema(c) + version_cursor = c.execute("select version from version") + + version_results = version_cursor.fetchall() + + if len(version_results) == 0: + raise IOError("No version number?") + if len(version_results) > 1: + raise IOError("More than one version result?") + first = version_results[0] + want_version = 1 + got_version = first[0] + if got_version != want_version: + raise IOError("Bad version number (want=%u got=%u" % + (want_version, got_version)) + self.progress("Got history version %u" % got_version) + + def record_build(self, hash, tag, vehicle, board, frame, bare_path, start_time, duration): + if bare_path is None: + (text, data, bss) = (None, None, None) + else: + (text, data, bss) = self.sizes_for_file(bare_path) + c = self.conn() + c.execute("replace into build (hash, tag, vehicle, board, frame, text, data, bss, start_time, duration) " + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (hash, tag, vehicle, board, frame, text, data, bss, start_time, duration)) + c.commit() + + def record_run(self, hash, tag, start_time, duration): + c = self.conn() + c.execute("replace into run (hash, tag, start_time, duration) " + "values (?, ?, ?, ?)", + (hash, tag, start_time, duration)) + c.commit()