Tools: generate_manifest.py: allow generation of features.json file

Tools/scripts/build_binaries.py

@@ -20,7 +20,6 @@ import string
 import subprocess
 import sys
 import traceback
-import gzip
 
 # local imports
 import generate_manifest
@@ -621,21 +620,9 @@ is bob we will attempt to checkout bob-AVR'''
         base_url = 'https://firmware.ardupilot.org'
         generator = generate_manifest.ManifestGenerator(self.binaries,
                                                         base_url)
-        content = generator.json()
+        generator.run()
-        new_json_filepath = os.path.join(self.binaries, "manifest.json.new")
+
-        self.write_string_to_filepath(content, new_json_filepath)
+        generator.write_manifest_json(os.path.join(self.binaries, "manifest.json"))
-        # provide a pre-compressed manifest.  For reference, a 7M manifest
-        # "gzip -9"s to 300k in 1 second, "xz -e"s to 80k in 26 seconds
-        new_json_filepath_gz = os.path.join(self.binaries,
-                                            "manifest.json.gz.new")
-        with gzip.open(new_json_filepath_gz, 'wb') as gf:
-            if running_python3:
-                content = bytes(content, 'ascii')
-            gf.write(content)
-        json_filepath = os.path.join(self.binaries, "manifest.json")
-        json_filepath_gz = os.path.join(self.binaries, "manifest.json.gz")
-        shutil.move(new_json_filepath, json_filepath)
-        shutil.move(new_json_filepath_gz, json_filepath_gz)
         self.progress("Manifest generation successful")
 
         self.progress("Generating stable releases")

Tools/scripts/generate_manifest.py

@@ -2,18 +2,26 @@
 
 from __future__ import print_function
 
-import sys
+import copy
+import fnmatch
+import gen_stable
+import gzip
 import json
 import os
 import re
-import fnmatch
-import gen_stable
 import subprocess
+import shutil
+import sys
 
 if sys.version_info[0] < 3:
     running_python3 = False
+    running_python310 = False
+elif sys.version_info[1] < 10:
+    running_python3 = True
+    running_python310 = False
 else:
     running_python3 = True
+    running_python310 = True
 
 FIRMWARE_TYPES = ["AntennaTracker", "Copter", "Plane", "Rover", "Sub", "AP_Periph", "Blimp"]
 RELEASE_TYPES = ["beta", "latest", "stable", "stable-*", "dirty"]
@@ -394,8 +402,14 @@ class ManifestGenerator():
                 frame = vehicletype  # e.g. Plane
                 platform = platformdir  # e.g. apm2
 
+            # also gather information from any features.txt files present:
+            features_text = None
+            features_filepath = os.path.join(some_dir, "features.txt")
+            if os.path.exists(features_filepath):
+                features_text = sorted(open(features_filepath).read().rstrip().split("\n"))
+
             for filename in os.listdir(some_dir):
-                if filename in ["git-version.txt", "firmware-version.txt", "files.html"]:
+                if filename in ["git-version.txt", "firmware-version.txt", "files.html", "features.txt"]:
                     continue
                 if filename.startswith("."):
                     continue
@@ -446,6 +460,8 @@ class ManifestGenerator():
                 firmware["format"] = firmware_format
                 firmware["firmware-version"] = firmware_version
 
+                firmware["features"] = features_text
+
                 firmware_data.append(firmware)
 
     def valid_release_type(self, tag):
@@ -506,6 +522,8 @@ class ManifestGenerator():
 
         # convert from ardupilot-naming conventions to common JSON format:
         firmware_json = []
+        features_json = []  # a structure containing summarised features per firmware
+
         for firmware in firmwares:
             filepath = firmware["filepath"]
             # replace the base directory with the base URL
@@ -524,6 +542,7 @@ class ManifestGenerator():
                 "latest": firmware["latest"],
                 "format": firmware["format"],
             })
+
             if firmware["firmware-version"]:
                 try:
                     (major, minor, patch, release_type) = self.parse_fw_version(
@@ -543,22 +562,83 @@ class ManifestGenerator():
             #print(some_json['url'])
             firmware_json.append(some_json)
 
+            # now the features the firmware supports...
+            try:
+                features = firmware["features"]
+                # check apj here in case we're creating bin and apj etc:
+                if (firmware["format"] == "apj" and
+                    features is not None and
+                    bool(firmware["latest"])):
+                    x = dict({
+                        "vehicletype": firmware["vehicletype"],
+                        "platform": firmware["platform"],
+                        "git-sha": firmware["git_sha"],
+                        "latest": firmware["latest"],
+                    })
+                    x["features"] = features
+                    features_json.append(x)
+
+            except KeyError:
+                pass
+
         ret = {
             "format-version": "1.0.0",  # semantic versioning
             "firmware": firmware_json
         }
 
-        return ret
+        features_ret = {
+            "format-version": "1.0.0",  # semantic versioning
+            "features": features_json
+        }
 
-    def json(self):
+        return ret, features_ret
-        '''walk directory supplied in constructor, return json string'''
+
+    def run(self):
+        '''walk directory supplied in constructor, record results in self'''
         if not self.looks_like_binaries_directory(self.basedir):
             print("Warning: this does not look like a binaries directory",
                   file=sys.stderr)
 
-        structure = self.walk_directory(self.basedir)
+        self.structure, self.features_structure = self.walk_directory(self.basedir)
-        return json.dumps(structure, indent=4, separators=(',', ': '))
 
+    def json(self):
+        '''returns JSON string for version information for all firmwares'''
+        if getattr(self, 'structure', None) is None:
+            self.run()
+        return json.dumps(self.structure, indent=4, separators=(',', ': '))
+
+    def json_features(self):
+        '''returns JSON string for supported features for all firmwares.
+        run() method must have been called already'''
+        return json.dumps(self.features_structure, indent=4, separators=(',', ': '))
+
+    def write_string_to_filepath(self, string, filepath):
+        '''writes the entirety of string to filepath'''
+        with open(filepath, "w") as x:
+            x.write(string)
+
+    def write_json(self, content, path):
+        '''write content to path, also creating a compress .gz version'''
+        new_json_filepath = path + ".new"
+        self.write_string_to_filepath(content, new_json_filepath)
+        # provide a pre-compressed version.  For reference, a 7M manifest
+        # "gzip -9"s to 300k in 1 second, "xz -e"s to 80k in 26 seconds
+        new_json_filepath_gz = path + ".gz.new"
+        with gzip.open(new_json_filepath_gz, 'wb') as gf:
+            if running_python3:
+                content = bytes(content, 'ascii')
+            gf.write(content)
+            gf.close()
+        shutil.move(new_json_filepath, path)
+        shutil.move(new_json_filepath_gz, path + ".gz")
+
+    def write_manifest_json(self, path):
+        '''write generated JSON content to path'''
+        self.write_json(self.json(), path)
+
+    def write_features_json(self, path):
+        '''write generated features JSON content to path'''
+        self.write_json(self.json_features(), path)
 
 def usage():
     return '''Usage:
@@ -570,6 +650,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='generate manifest.json')
 
     parser.add_argument('--outfile', type=str, default=None, help='output file, default stdout')
+    parser.add_argument('--outfile-features-json', type=str, default=None, help='output file for features json file')
     parser.add_argument('--baseurl', type=str, default="https://firmware.ardupilot.org", help='base binaries directory')
     parser.add_argument('basedir', type=str, default="-", help='base binaries directory')
 
@@ -579,12 +660,13 @@ if __name__ == "__main__":
     gen_stable.make_all_stable(args.basedir)
 
     generator = ManifestGenerator(args.basedir, args.baseurl)
+    generator.run()
+
+    content = generator.json()
     if args.outfile is None:
-        print(generator.json())
+        print(content)
     else:
-        f = open(args.outfile, "w")
+        generator.write_manifest_json(args.outfile)
-        content = generator.json()
+
-        if running_python3:
+    if args.outfile_features_json is not None:
-            content = bytes(content, 'ascii')
+        generator.write_features_json(args.outfile_features_json)
-        f.write(content)
-        f.close()