diff --git a/Tools/scripts/generate_manifest.py b/Tools/scripts/generate_manifest.py index d3eaf73b0e..bf48e6f13e 100755 --- a/Tools/scripts/generate_manifest.py +++ b/Tools/scripts/generate_manifest.py @@ -7,8 +7,15 @@ import json import os import re + class Firmware(): - def __init__(self, date=None, platform=None, vehicletype=None, filepath=None, git_sha=None, frame=None): + def __init__(self, + date=None, + platform=None, + vehicletype=None, + filepath=None, + git_sha=None, + frame=None): self.atts = dict() self.atts["date"] = date self.atts["platform"] = platform @@ -19,21 +26,24 @@ class Firmware(): self.atts["release-type"] = None self.atts["firmware-version"] = None - def __getitem__(self,what): + def __getitem__(self, what): return self.atts[what] - def __setitem__(self,name,value): + def __setitem__(self, name, value): self.atts[name] = value -class ManifestGenerator(): - '''Return a JSON string describing "binary" directory contents under basedir''' - def __init__(self,basedir,baseurl): +class ManifestGenerator(): + '''Return a JSON string describing "binary" directory contents under + basedir''' + + def __init__(self, basedir, baseurl): self.basedir = basedir self.baseurl = baseurl def frame_map(self, frame): - '''translate from ArduPilot frame type terminology into mavlink terminology''' + '''translate from ArduPilot frame type terminology into mavlink + terminology''' frame_to_mavlink_dict = { "quad": "QUADROTOR", "hexa": "HEXAROTOR", @@ -53,29 +63,39 @@ class ManifestGenerator(): return frame def releasetype_map(self, releasetype): - '''translate from ArduPilot release type terminology into mavlink terminology''' - if releasetype == 'stable': return 'OFFICIAL' + '''translate from ArduPilot release type terminology into mavlink + terminology''' + if releasetype == 'stable': + return 'OFFICIAL' return releasetype.upper() def looks_like_binaries_directory(self, dir): - '''returns True if dir looks like it is a build_binaries.py output directory''' + '''returns True if dir looks like it is a build_binaries.py output + directory''' for entry in os.listdir(dir): if entry in {"AntennaTracker", "Copter", "Plane", "Rover", "Sub"}: return True return False def git_sha_from_git_version(self, filepath): - '''parses get-version.txt (as emitted by build_binaries.py, returns git sha from it''' + '''parses get-version.txt (as emitted by build_binaries.py, returns + git sha from it''' content = open(filepath).read() sha_regex = re.compile("commit (?P[0-9a-f]+)") m = sha_regex.search(content) if m is None: - raise Exception("filepath (%s) does not appear to contain a git sha" % (filepath,)) + raise Exception( + "filepath (%s) does not contain a git sha" % (filepath,)) return m.group("sha") - def add_firmware_data_from_dir(self, dir, firmware_data, vehicletype, releasetype="dev"): - '''accumulate additional information about firmwares from a directory''' - platform_frame_regex = re.compile("(?PPX4|navio|pxf)(-(?P.+))?") + def add_firmware_data_from_dir(self, + dir, + firmware_data, + vehicletype, + releasetype="dev"): + '''accumulate additional information about firmwares from directory''' + platform_frame_regex = re.compile( + "(?PPX4|navio|pxf)(-(?P.+))?") variant_firmware_regex = re.compile("[^-]+-(?Pv\d+)[.px4]") if not os.path.isdir(dir): return @@ -87,16 +107,19 @@ class ManifestGenerator(): for platformdir in dlist: some_dir = os.path.join(dir, platformdir) try: - git_sha = self.git_sha_from_git_version(os.path.join(some_dir, "git-version.txt")) + git_sha = self.git_sha_from_git_version( + os.path.join(some_dir, "git-version.txt")) except Exception as e: continue - firmware_version_file = os.path.join(some_dir, "firmware-version.txt") + firmware_version_file = os.path.join(some_dir, + "firmware-version.txt") try: firmware_version = open(firmware_version_file).read() firmware_version = firmware_version.strip() - (version_numbers,release_type) = firmware_version.split("-") + (version_numbers, release_type) = firmware_version.split("-") except ValueError as e: - # print("malformed firmware-version.txt at (%s)" % (firmware_version_file,), file=sys.stderr) + # print("malformed firmware-version.txt at (%s)" % + # (firmware_version_file,), file=sys.stderr) firmware_version = None except Exception as e: # this exception is swallowed.... the current archive @@ -108,13 +131,13 @@ class ManifestGenerator(): # the model type (quad/tri) is # encoded in the platform name # (e.g. navio-octa) - platform = m.group("board") # e.g. navio - frame = m.group("frame") # e.g. octa + platform = m.group("board") # e.g. navio + frame = m.group("frame") # e.g. octa if frame is None: frame = vehicletype else: - frame = vehicletype # e.g. Plane - platform = platformdir # e.g. apm2 + frame = vehicletype # e.g. Plane + platform = platformdir # e.g. apm2 for file in os.listdir(some_dir): if file == "git-version.txt": @@ -131,32 +154,31 @@ class ManifestGenerator(): # (e.g. the "v1" in # ArduCopter-v1.px4) variant = m.group("variant") - file_platform = "-".join([platform,variant]) + file_platform = "-".join([platform, variant]) else: file_platform = platform firmware_format = "".join(file.split(".")[-1:]) - if not vehicletype in firmware_data: + if vehicletype not in firmware_data: firmware_data[vehicletype] = dict() - if not file_platform in firmware_data[vehicletype]: + if file_platform not in firmware_data[vehicletype]: firmware_data[vehicletype][file_platform] = dict() - if not git_sha in firmware_data[vehicletype][file_platform]: + if git_sha not in firmware_data[vehicletype][file_platform]: firmware_data[vehicletype][file_platform][git_sha] = dict() - if not firmware_format in firmware_data[vehicletype][file_platform][git_sha]: - firmware_data[vehicletype][file_platform][git_sha][firmware_format] = dict() - if not releasetype in firmware_data[vehicletype][file_platform][git_sha][firmware_format]: - firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype] = dict() - if not frame in firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype]: - firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype][frame] = Firmware() + sha_dict = firmware_data[vehicletype][file_platform][git_sha] + if firmware_format not in sha_dict: + sha_dict[firmware_format] = dict() + if frame not in sha_dict[firmware_format]: + sha_dict[firmware_format][frame] = Firmware() - firmware = firmware_data[vehicletype][file_platform][git_sha][firmware_format][releasetype][frame] + firmware = sha_dict[firmware_format][frame] # translate from supplied "release type" into both a # "latest" flag andan actual release type. Also sort # out which filepath we should use: - firmware["latest" ] = 0 + firmware["latest"] = 0 if releasetype == "dev": if firmware["filepath"] is None: firmware["filepath"] = os.path.join(some_dir, file) @@ -185,7 +207,7 @@ class ManifestGenerator(): if isinstance(xfirmwares, dict): ret = [] for value in xfirmwares.values(): - o = self.xfirmwares_to_firmwares(value) + o = self.xfirmwares_to_firmwares(value) for oo in o: ret.append(oo) return ret @@ -193,24 +215,26 @@ class ManifestGenerator(): return [xfirmwares] known_release_types = { - "beta" : 1, - "latest" : 1, - "stable" : 1 + "beta": 1, + "latest": 1, + "stable": 1 } def parse_fw_version(self, version): - (version_numbers,release_type) = version.split("-") - (major,minor,patch) = version_numbers.split(".") - return (major,minor,patch,version) + (version_numbers, release_type) = version.split("-") + (major, minor, patch) = version_numbers.split(".") + return (major, minor, patch, version) def walk_directory(self, basedir): - '''walks directory structure created by build_binaries, returns Python structure representing releases in that structure''' + '''walks directory structure created by build_binaries, returns Python + structure representing releases in that structure''' year_month_regex = re.compile("(?P\d{4})-(?P\d{2})") xfirmwares = dict() - # used to listdir basedir here, but since this is also a web document root, there's a lot of other stuff accumulated... - vehicletypes = [ 'AntennaTracker', 'Copter', 'Plane', 'Rover', 'Sub' ] + # used to listdir basedir here, but since this is also a web + # document root, there's a lot of other stuff accumulated... + vehicletypes = ['AntennaTracker', 'Copter', 'Plane', 'Rover', 'Sub'] for vehicletype in vehicletypes: try: vdir = os.listdir(os.path.join(basedir, vehicletype)) @@ -226,12 +250,17 @@ class ManifestGenerator(): # this is a dated directory e.g. binaries/Copter/2016-02 # we do not include dated directories in the manifest ATM: continue - year_month_path = os.path.join(basedir, vehicletype, firstlevel) + year_month_path = os.path.join(basedir, + vehicletype, + firstlevel) for fulldate in os.listdir(year_month_path): if fulldate in ["files.html", ".makehtml"]: # generated file which should be ignored continue - self.add_firmware_data_from_dir(os.path.join(year_month_path, fulldate), xfirmwares, vehicletype) + self.add_firmware_data_from_dir( + os.path.join(year_month_path, fulldate), + xfirmwares, + vehicletype) else: # assume this is a release directory such as # "beta", or the "latest" directory (treated as a @@ -243,7 +272,10 @@ class ManifestGenerator(): tag_path = os.path.join(basedir, vehicletype, tag) if not os.path.isdir(tag_path): continue - self.add_firmware_data_from_dir(tag_path, xfirmwares, vehicletype, releasetype=tag) + self.add_firmware_data_from_dir(tag_path, + xfirmwares, + vehicletype, + releasetype=tag) firmwares = self.xfirmwares_to_firmwares(xfirmwares) @@ -254,20 +286,24 @@ class ManifestGenerator(): # replace the base directory with the base URL urlifier = re.compile("^" + re.escape(basedir)) url = re.sub(urlifier, self.baseurl, filepath) + version_type = self.releasetype_map(firmware["release-type"]) some_json = dict({ "mav-autopilot": "ARDUPILOTMEGA", -# "vehicletype": firmware["vehicletype"], + # "vehicletype": firmware["vehicletype"], "platform": firmware["platform"], "git-sha": firmware["git_sha"], "url": url, "mav-type": self.frame_map(firmware["frame"]), - "mav-firmware-version-type": self.releasetype_map(firmware["release-type"]), + "mav-firmware-version-type": version_type, "latest": firmware["latest"], "format": firmware["format"], }) if firmware["firmware-version"]: - (major,minor,patch,release_type) = self.parse_fw_version(firmware["firmware-version"]) - some_json["mav-firmware-version"] = ".".join([major,minor,patch]) + (major, minor, patch, release_type) = self.parse_fw_version( + firmware["firmware-version"]) + some_json["mav-firmware-version"] = ".".join([major, + minor, + patch]) some_json["mav-firmware-version-major"] = major some_json["mav-firmware-version-minor"] = minor some_json["mav-firmware-version-patch"] = patch @@ -275,7 +311,7 @@ class ManifestGenerator(): firmware_json.append(some_json) ret = { - "format-version": "1.0.0", # semantic versioning + "format-version": "1.0.0", # semantic versioning "firmware": firmware_json } @@ -284,15 +320,18 @@ class ManifestGenerator(): def json(self): '''walk directory supplied in constructor, return json string''' if not self.looks_like_binaries_directory(self.basedir): - print("Warning: this does not look like a binaries directory", file=sys.stderr) + print("Warning: this does not look like a binaries directory", + file=sys.stderr) structure = self.walk_directory(self.basedir) return json.dumps(structure, indent=4) + def usage(): return '''Usage: generate-manifest.py basedir baseurl''' + if __name__ == "__main__": if len(sys.argv) != 3: print(usage())