From d7977dfcc2f8e2c8b9509e01e62cfa3688bacfaa Mon Sep 17 00:00:00 2001 From: Peter Barker Date: Mon, 29 Feb 2016 16:18:41 +1100 Subject: [PATCH] Tools/scripts/generate-manifest.py: json description of binaries directory --- Tools/scripts/generate-manifest.py | 271 +++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100755 Tools/scripts/generate-manifest.py diff --git a/Tools/scripts/generate-manifest.py b/Tools/scripts/generate-manifest.py new file mode 100755 index 0000000000..1fb46c994f --- /dev/null +++ b/Tools/scripts/generate-manifest.py @@ -0,0 +1,271 @@ +#!/usr/bin/python + +from __future__ import print_function + +import sys +import json +import os +import re + +class Firmware(): + def __init__(self, date=None, platform=None, vehicletype=None, filepath=None, git_sha=None, frame=None): + self.atts = dict() + self.atts["date"] = date + self.atts["platform"] = platform + self.atts["vehicletype"] = vehicletype + self.atts["filepath"] = filepath + self.atts["git_sha"] = git_sha + self.atts["frame"] = frame + self.atts["release-type"] = None + self.atts["firmware-version"] = None + + def __getitem__(self,what): + return self.atts[what] + + def __setitem__(self,name,value): + self.atts[name] = value + +class ManifestGenerator(): + '''Return a JSON string describing "binary" directory contents under basedir''' + + def __init__(self,basedir,baseurl): + self.basedir = basedir + self.baseurl = baseurl + + def frame_map(self, frame): + '''translate from ArduPilot frame type terminology into mavlink terminology''' + frame_to_mavlink_dict = { + "quad": "QUADROTOR", + "hexa": "HEXAROTOR", + "y6": "ARDUPILOT_Y6", + "tri": "TRICOPTER", + "octa": "OCTOROTOR", + "octa-quad": "ARDUPILOT_OCTAQUAD", + "heli": "HELICOPTER", + "Plane": "FIXED_WING", + "AntennaTracker": "ANTENNA_TRACKER", + "Rover": "GROUND_ROVER", + "PX4IO": "ARDUPILOT_PX4IO", + } + if frame in frame_to_mavlink_dict: + return frame_to_mavlink_dict[frame] + + return frame + + def releasetype_map(self, releasetype): + '''translate from ArduPilot release type terminology into mavlink terminology''' + if releasetype == 'stable': return 'OFFICIAL' + return releasetype.upper() + + def looks_like_binaries_directory(self, dir): + '''returns True if dir looks like it is a build_binaries.sh output directory''' + for entry in os.listdir(dir): + if entry in {"AntennaTracker", "Copter", "Plane", "Rover"}: + return True + return False + + def git_sha_from_git_version(self, filepath): + '''parses get-version.txt (as emitted by build_binaries.sh, returns git sha from it''' + content = open(filepath).read() + sha_regex = re.compile("commit (?P[0-9a-f]+)") + m = sha_regex.search(content) + if m is None: + raise Exception("filepath (%s) does not appear to contain a git sha" % (filepath,)) + return m.group("sha") + + def add_firmware_data_from_dir(self, dir, firmware_data, vehicletype, releasetype="dev"): + '''accumulate additional information about firmwares from a directory''' + platform_frame_regex = re.compile("(?PPX4|navio|pxf)(-(?P.+))?") + variant_firmware_regex = re.compile("[^-]+-(?Pv\d+)[.px4]") + for platformdir in os.listdir(dir): + some_dir = os.path.join(dir, platformdir) + try: + git_sha = self.git_sha_from_git_version(os.path.join(some_dir, "git-version.txt")) + except Exception as e: + continue + try: + firmware_version = open(os.path.join(some_dir, "firmware-version.txt")).read() + firmware_version = firmware_version.strip() + except Exception as e: + # this exception is swallowed.... the current archive + # is incomplete. + firmware_version = None + + m = platform_frame_regex.match(platformdir) + if m is not None: + # the model type (quad/tri) is + # encoded in the platform name + # (e.g. navio-octa) + platform = m.group("board") # e.g. navio + frame = m.group("frame") # e.g. octa + if frame is None: + frame = vehicletype + else: + frame = vehicletype # e.g. Plane + platform = platformdir # e.g. apm2 + + for file in os.listdir(some_dir): + if file == "git-version.txt": + continue + if file == "firmware-version.txt": + continue + + m = variant_firmware_regex.match(file) + if m: + # the platform variant is + # encoded in the firmware filename + # (e.g. the "v1" in + # ArduCopter-v1.px4) + variant = m.group("variant") + file_platform = "-".join([platform,variant]) + else: + file_platform = platform + + firmware_format = "".join(file.split(".")[-1:]) + + if not vehicletype in firmware_data: + firmware_data[vehicletype] = dict() + if not file_platform in firmware_data[vehicletype]: + firmware_data[vehicletype][file_platform] = dict() + if not git_sha in firmware_data[vehicletype][file_platform]: + firmware_data[vehicletype][file_platform][git_sha] = dict() + + if not firmware_format in firmware_data[vehicletype][file_platform][git_sha]: + firmware_data[vehicletype][file_platform][git_sha][firmware_format] = dict() + if not frame in firmware_data[vehicletype][file_platform][git_sha][firmware_format]: + firmware_data[vehicletype][file_platform][git_sha][firmware_format][frame] = Firmware() + + firmware = firmware_data[vehicletype][file_platform][git_sha][firmware_format][frame] + + # translate from supplied "release type" into both a + # "latest" flag andan actual release type. Also sort + # out which filepath we should use: + firmware["latest" ] = 0 + if releasetype == "dev": + if firmware["filepath"] is None: + firmware["filepath"] = os.path.join(some_dir, file) + if firmware["release-type"] is None: + firmware["release-type"] = "dev" + elif releasetype == "latest": + firmware["latest"] = 1 + firmware["filepath"] = os.path.join(some_dir, file) + if firmware["release-type"] is None: + firmware["release-type"] = "dev" + else: + if (not firmware["latest"]): + firmware["filepath"] = os.path.join(some_dir, file) + firmware["release-type"] = releasetype + + firmware["platform"] = file_platform + firmware["vehicletype"] = vehicletype + firmware["git_sha"] = git_sha + firmware["frame"] = frame + firmware["timestamp"] = os.path.getctime(firmware["filepath"]) + firmware["format"] = firmware_format + firmware["firmware-version"] = firmware_version + + def xfirmwares_to_firmwares(self, xfirmwares): + '''takes hash structure of firmwares, returns list of them''' + if isinstance(xfirmwares, dict): + ret = [] + for value in xfirmwares.values(): + o = self.xfirmwares_to_firmwares(value) + for oo in o: + ret.append(oo) + return ret + else: + return [xfirmwares] + + known_release_types = { + "beta" : 1, + "latest" : 1, + "stable" : 1 + } + + def parse_fw_version(self, version): + (version_numbers,release_type) = version.split("-") + (major,minor,patch) = version_numbers.split(".") + return (major,minor,patch,version) + + def walk_directory(self, basedir): + '''walks directory structure created by build_binaries, returns Python structure representing releases in that structure''' + year_month_regex = re.compile("(?P\d{4})-(?P\d{2})") + + xfirmwares = dict() + + # used to listdir basedir here, but since this is also a web document root, there's a lot of other stuff accumulated... + vehicletypes = [ 'AntennaTracker', 'Copter', 'Plane', 'PX4IO', 'Rover' ] + for vehicletype in vehicletypes: + vdir = os.listdir(os.path.join(basedir, vehicletype)) + for firstlevel in vdir: + if year_month_regex.match(firstlevel): + # this is a dated directory e.g. binaries/Copter/2016-02 + year_month_path = os.path.join(basedir, vehicletype, firstlevel) + for fulldate in os.listdir(year_month_path): + self.add_firmware_data_from_dir(os.path.join(year_month_path, fulldate), xfirmwares, vehicletype) + else: + # assume this is a release directory such as + # "beta", or the "latest" directory (treated as a + # release and handled specially later) + tag = firstlevel + if tag not in self.known_release_types: + print("Unknown tag (%s) in directory (%s)" % + (tag, vdir)) + tag_path = os.path.join(basedir, vehicletype, tag) + self.add_firmware_data_from_dir(tag_path, xfirmwares, vehicletype, releasetype=tag) + + firmwares = self.xfirmwares_to_firmwares(xfirmwares) + + # convert from ardupilot-naming conventions to common JSON format: + firmware_json = [] + for firmware in firmwares: + filepath = firmware["filepath"] + # replace the base directory with the base URL + urlifier = re.compile("^" + re.escape(basedir)) + url = re.sub(urlifier, self.baseurl, filepath) + some_json = dict({ + "mav-autopilot": "ARDUPILOTMEGA", +# "vehicletype": firmware["vehicletype"], + "platform": firmware["platform"], + "git-sha": firmware["git_sha"], + "url": url, + "mav-type": self.frame_map(firmware["frame"]), + "mav-firmware-version-type": self.releasetype_map(firmware["release-type"]), + "latest": firmware["latest"], + "format": firmware["format"], + }) + if firmware["firmware-version"]: + (major,minor,patch,release_type) = self.parse_fw_version(firmware["firmware-version"]) + some_json["mav-firmware-version"] = ".".join([major,minor,patch]) + some_json["mav-firmware-version-major"] = major + some_json["mav-firmware-version-minor"] = minor + some_json["mav-firmware-version-patch"] = patch + + firmware_json.append(some_json) + + ret = { + "format-version": "1.0.0", # semantic versioning + "firmware": firmware_json + } + + return ret + + def json(self): + '''walk directory supplied in constructor, return json string''' + if not self.looks_like_binaries_directory(self.basedir): + print("Warning: this does not look like a binaries directory", file=sys.stderr) + + structure = self.walk_directory(self.basedir) + return json.dumps(structure, indent=4) + +def usage(): + return '''Usage: +generate-manifest.py basedir baseurl''' + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(usage()) + sys.exit(1) + + generator = ManifestGenerator(sys.argv[1], sys.argv[2]) + print(generator.json())