Tools: generate_manifest.py: allow generation of features.json file
This commit is contained in:
Peter Barker
Peter Barker
parent
71d1c050b3
commit
459af5ec9b
@ -20,7 +20,6 @@ import string
|
||||
import subprocess
|
||||
import sys
|
||||
import traceback
|
||||
import gzip
|
||||
|
||||
# local imports
|
||||
import generate_manifest
|
||||
@ -621,21 +620,9 @@ is bob we will attempt to checkout bob-AVR'''
|
||||
base_url = 'https://firmware.ardupilot.org'
|
||||
generator = generate_manifest.ManifestGenerator(self.binaries,
|
||||
base_url)
|
||||
content = generator.json()
|
||||
new_json_filepath = os.path.join(self.binaries, "manifest.json.new")
|
||||
self.write_string_to_filepath(content, new_json_filepath)
|
||||
# provide a pre-compressed manifest. For reference, a 7M manifest
|
||||
# "gzip -9"s to 300k in 1 second, "xz -e"s to 80k in 26 seconds
|
||||
new_json_filepath_gz = os.path.join(self.binaries,
|
||||
"manifest.json.gz.new")
|
||||
with gzip.open(new_json_filepath_gz, 'wb') as gf:
|
||||
if running_python3:
|
||||
content = bytes(content, 'ascii')
|
||||
gf.write(content)
|
||||
json_filepath = os.path.join(self.binaries, "manifest.json")
|
||||
json_filepath_gz = os.path.join(self.binaries, "manifest.json.gz")
|
||||
shutil.move(new_json_filepath, json_filepath)
|
||||
shutil.move(new_json_filepath_gz, json_filepath_gz)
|
||||
generator.run()
|
||||
|
||||
generator.write_manifest_json(os.path.join(self.binaries, "manifest.json"))
|
||||
self.progress("Manifest generation successful")
|
||||
|
||||
self.progress("Generating stable releases")
|
||||
|
||||
@ -2,18 +2,26 @@
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
import copy
|
||||
import fnmatch
|
||||
import gen_stable
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import fnmatch
|
||||
import gen_stable
|
||||
import subprocess
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
if sys.version_info[0] < 3:
|
||||
running_python3 = False
|
||||
running_python310 = False
|
||||
elif sys.version_info[1] < 10:
|
||||
running_python3 = True
|
||||
running_python310 = False
|
||||
else:
|
||||
running_python3 = True
|
||||
running_python310 = True
|
||||
|
||||
FIRMWARE_TYPES = ["AntennaTracker", "Copter", "Plane", "Rover", "Sub", "AP_Periph", "Blimp"]
|
||||
RELEASE_TYPES = ["beta", "latest", "stable", "stable-*", "dirty"]
|
||||
@ -394,8 +402,14 @@ class ManifestGenerator():
|
||||
frame = vehicletype # e.g. Plane
|
||||
platform = platformdir # e.g. apm2
|
||||
|
||||
# also gather information from any features.txt files present:
|
||||
features_text = None
|
||||
features_filepath = os.path.join(some_dir, "features.txt")
|
||||
if os.path.exists(features_filepath):
|
||||
features_text = sorted(open(features_filepath).read().rstrip().split("\n"))
|
||||
|
||||
for filename in os.listdir(some_dir):
|
||||
if filename in ["git-version.txt", "firmware-version.txt", "files.html"]:
|
||||
if filename in ["git-version.txt", "firmware-version.txt", "files.html", "features.txt"]:
|
||||
continue
|
||||
if filename.startswith("."):
|
||||
continue
|
||||
@ -446,6 +460,8 @@ class ManifestGenerator():
|
||||
firmware["format"] = firmware_format
|
||||
firmware["firmware-version"] = firmware_version
|
||||
|
||||
firmware["features"] = features_text
|
||||
|
||||
firmware_data.append(firmware)
|
||||
|
||||
def valid_release_type(self, tag):
|
||||
@ -506,6 +522,8 @@ class ManifestGenerator():
|
||||
|
||||
# convert from ardupilot-naming conventions to common JSON format:
|
||||
firmware_json = []
|
||||
features_json = [] # a structure containing summarised features per firmware
|
||||
|
||||
for firmware in firmwares:
|
||||
filepath = firmware["filepath"]
|
||||
# replace the base directory with the base URL
|
||||
@ -524,6 +542,7 @@ class ManifestGenerator():
|
||||
"latest": firmware["latest"],
|
||||
"format": firmware["format"],
|
||||
})
|
||||
|
||||
if firmware["firmware-version"]:
|
||||
try:
|
||||
(major, minor, patch, release_type) = self.parse_fw_version(
|
||||
@ -543,22 +562,83 @@ class ManifestGenerator():
|
||||
#print(some_json['url'])
|
||||
firmware_json.append(some_json)
|
||||
|
||||
# now the features the firmware supports...
|
||||
try:
|
||||
features = firmware["features"]
|
||||
# check apj here in case we're creating bin and apj etc:
|
||||
if (firmware["format"] == "apj" and
|
||||
features is not None and
|
||||
bool(firmware["latest"])):
|
||||
x = dict({
|
||||
"vehicletype": firmware["vehicletype"],
|
||||
"platform": firmware["platform"],
|
||||
"git-sha": firmware["git_sha"],
|
||||
"latest": firmware["latest"],
|
||||
})
|
||||
x["features"] = features
|
||||
features_json.append(x)
|
||||
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
ret = {
|
||||
"format-version": "1.0.0", # semantic versioning
|
||||
"firmware": firmware_json
|
||||
}
|
||||
|
||||
return ret
|
||||
features_ret = {
|
||||
"format-version": "1.0.0", # semantic versioning
|
||||
"features": features_json
|
||||
}
|
||||
|
||||
def json(self):
|
||||
'''walk directory supplied in constructor, return json string'''
|
||||
return ret, features_ret
|
||||
|
||||
def run(self):
|
||||
'''walk directory supplied in constructor, record results in self'''
|
||||
if not self.looks_like_binaries_directory(self.basedir):
|
||||
print("Warning: this does not look like a binaries directory",
|
||||
file=sys.stderr)
|
||||
|
||||
structure = self.walk_directory(self.basedir)
|
||||
return json.dumps(structure, indent=4, separators=(',', ': '))
|
||||
self.structure, self.features_structure = self.walk_directory(self.basedir)
|
||||
|
||||
def json(self):
|
||||
'''returns JSON string for version information for all firmwares'''
|
||||
if getattr(self, 'structure', None) is None:
|
||||
self.run()
|
||||
return json.dumps(self.structure, indent=4, separators=(',', ': '))
|
||||
|
||||
def json_features(self):
|
||||
'''returns JSON string for supported features for all firmwares.
|
||||
run() method must have been called already'''
|
||||
return json.dumps(self.features_structure, indent=4, separators=(',', ': '))
|
||||
|
||||
def write_string_to_filepath(self, string, filepath):
|
||||
'''writes the entirety of string to filepath'''
|
||||
with open(filepath, "w") as x:
|
||||
x.write(string)
|
||||
|
||||
def write_json(self, content, path):
|
||||
'''write content to path, also creating a compress .gz version'''
|
||||
new_json_filepath = path + ".new"
|
||||
self.write_string_to_filepath(content, new_json_filepath)
|
||||
# provide a pre-compressed version. For reference, a 7M manifest
|
||||
# "gzip -9"s to 300k in 1 second, "xz -e"s to 80k in 26 seconds
|
||||
new_json_filepath_gz = path + ".gz.new"
|
||||
with gzip.open(new_json_filepath_gz, 'wb') as gf:
|
||||
if running_python3:
|
||||
content = bytes(content, 'ascii')
|
||||
gf.write(content)
|
||||
gf.close()
|
||||
shutil.move(new_json_filepath, path)
|
||||
shutil.move(new_json_filepath_gz, path + ".gz")
|
||||
|
||||
def write_manifest_json(self, path):
|
||||
'''write generated JSON content to path'''
|
||||
self.write_json(self.json(), path)
|
||||
|
||||
def write_features_json(self, path):
|
||||
'''write generated features JSON content to path'''
|
||||
self.write_json(self.json_features(), path)
|
||||
|
||||
def usage():
|
||||
return '''Usage:
|
||||
@ -570,6 +650,7 @@ if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='generate manifest.json')
|
||||
|
||||
parser.add_argument('--outfile', type=str, default=None, help='output file, default stdout')
|
||||
parser.add_argument('--outfile-features-json', type=str, default=None, help='output file for features json file')
|
||||
parser.add_argument('--baseurl', type=str, default="https://firmware.ardupilot.org", help='base binaries directory')
|
||||
parser.add_argument('basedir', type=str, default="-", help='base binaries directory')
|
||||
|
||||
@ -579,12 +660,13 @@ if __name__ == "__main__":
|
||||
gen_stable.make_all_stable(args.basedir)
|
||||
|
||||
generator = ManifestGenerator(args.basedir, args.baseurl)
|
||||
generator.run()
|
||||
|
||||
content = generator.json()
|
||||
if args.outfile is None:
|
||||
print(generator.json())
|
||||
print(content)
|
||||
else:
|
||||
f = open(args.outfile, "w")
|
||||
content = generator.json()
|
||||
if running_python3:
|
||||
content = bytes(content, 'ascii')
|
||||
f.write(content)
|
||||
f.close()
|
||||
generator.write_manifest_json(args.outfile)
|
||||
|
||||
if args.outfile_features_json is not None:
|
||||
generator.write_features_json(args.outfile_features_json)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user