github ci: push parameter metadata to s3

.github/workflows/deploy_all.yml

@@ -0,0 +1,77 @@
+name: Deploy metadata for all targets
+
+on:
+  push:
+    branches:
+    - 'master'
+    - 'release/*'
+    - 'pr-metadata-test'
+
+jobs:
+  enumerate_targets:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        token: ${{secrets.ACCESS_TOKEN}}
+    - id: set-matrix
+      run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)"
+  build:
+    runs-on: ubuntu-latest
+    needs: enumerate_targets
+    strategy:
+      matrix: ${{fromJson(needs.enumerate_targets.outputs.matrix)}}
+    container: px4io/px4-dev-${{ matrix.container }}:2021-02-04
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        token: ${{secrets.ACCESS_TOKEN}}
+
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+    - name: ccache cache files
+      uses: actions/cache@v2
+      with:
+        path: ~/.ccache
+        key: ${{matrix.target}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
+        restore-keys: ${{matrix.target}}-ccache-
+    - name: setup ccache
+      run: |
+          mkdir -p ~/.ccache
+          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
+          echo "compression = true" >> ~/.ccache/ccache.conf
+          echo "compression_level = 5" >> ~/.ccache/ccache.conf
+          echo "max_size = 100M" >> ~/.ccache/ccache.conf
+          ccache -s
+          ccache -z
+
+    - name: make ${{matrix.target}}
+      run: make ${{matrix.target}}
+    - name: ccache post-run
+      run: ccache -s
+
+    - name: parameter metadata
+      run: |
+        make ${{matrix.target}} ver_gen
+        ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV
+        cd build/${{ matrix.target }}
+        mkdir _metadata || true
+        cp parameters.* _metadata
+
+    - uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read
+      env:
+        AWS_S3_BUCKET: 'px4-travis'
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_REGION: 'us-west-1'
+        SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/'
+        DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/'
+

.github/workflows/metadata.yml

@@ -4,9 +4,8 @@ on:
   push:
     branches:
     - 'master'
-  pull_request:
-    branches:
-    - '*'
+    - 'release/*'
+    - 'pr-metadata-test'
 
 jobs:
 
@@ -51,9 +50,18 @@ jobs:
     - name: parameter metadata
       run: |
         make parameters_metadata
-        cd build/px4_sitl_default/docs
-        ls -ls *
-      # TODO: deploy 'parameters.md, parameters.xml to S3 px4-travis:Firmware/master/
+        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
+
+    - uses: jakejarvis/s3-sync-action@master
+      with:
+        args: --acl public-read
+      env:
+        AWS_S3_BUCKET: 'px4-travis'
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_REGION: 'us-west-1'
+        SOURCE_DIR: 'build/px4_sitl_default/docs/'
+        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
 
   uorb_graph:
     runs-on: ubuntu-latest

Makefile

@@ -318,7 +318,7 @@ coverity_scan: px4_sitl_default
 .PHONY: parameters_metadata airframe_metadata module_documentation px4_metadata doxygen
 
 parameters_metadata:
-	@$(MAKE) --no-print-directory px4_sitl_default metadata_parameters
+	@$(MAKE) --no-print-directory px4_sitl_default metadata_parameters ver_gen
 
 airframe_metadata:
 	@$(MAKE) --no-print-directory px4_sitl_default metadata_airframes

Tools/generate_board_targets_json.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+""" Script to generate a JSON config with all build targets (for CI) """
+
+import argparse
+import os
+import sys
+import json
+import re
+
+source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
+
+parser = argparse.ArgumentParser(description='Generate build targets')
+
+parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
+                    help='Verbose Output')
+parser.add_argument('-p', '--pretty', dest='pretty', action='store_true',
+                    help='Pretty output instead of a single line')
+
+args = parser.parse_args()
+verbose = args.verbose
+
+build_configs = []
+excluded_manufacturers = ['atlflight']
+excluded_platforms = ['qurt']
+excluded_labels = ['stackcheck', 'nolockstep', 'replay', 'test']
+
+def process_target(cmake_file, target_name):
+    ret = None
+    is_board_def = False
+    platform = None
+    toolchain = None
+    for line in open(cmake_file, 'r'):
+        if 'px4_add_board' in line:
+            is_board_def = True
+        if not is_board_def:
+            continue
+
+        re_platform = re.search('PLATFORM\s+([^\s]+)', line)
+        if re_platform: platform = re_platform.group(1)
+
+        re_toolchain = re.search('TOOLCHAIN\s+([^\s]+)', line)
+        if re_toolchain: toolchain = re_toolchain.group(1)
+
+    if is_board_def:
+        assert platform, f"PLATFORM not found in {cmake_file}"
+
+        if platform not in excluded_platforms:
+            # get the container based on the platform and toolchain
+            container = platform
+            if platform == 'posix':
+                container = 'base-focal'
+                if toolchain:
+                    if toolchain.startswith('aarch64'):
+                        container = 'aarch64'
+                    elif toolchain == 'arm-linux-gnueabihf':
+                        container = 'armhf'
+                    else:
+                        if verbose: print(f'possibly unmatched toolchain: {toolchain}')
+            elif platform == 'nuttx':
+                container = 'nuttx-focal'
+
+            ret = {'target': target_name, 'container': container}
+    return ret
+
+for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
+    if not manufacturer.is_dir():
+        continue
+    if manufacturer.name in excluded_manufacturers:
+        if verbose: print(f'excluding manufacturer {manufacturer.name}')
+        continue
+
+    for board in os.scandir(manufacturer.path):
+        if not board.is_dir():
+            continue
+        for files in os.scandir(board.path):
+            if files.is_file() and files.name.endswith('.cmake'):
+                label = files.name[:-6]
+                target_name = manufacturer.name + '_' + board.name + '_' + label
+                if label in excluded_labels:
+                    if verbose: print(f'excluding label {label} ({target_name})')
+                    continue
+                target = process_target(files.path, target_name)
+                if target is not None:
+                    build_configs.append(target)
+
+
+github_action_config = { 'include': build_configs }
+extra_args = {}
+if args.pretty:
+    extra_args['indent'] = 2
+print(json.dumps(github_action_config, **extra_args))
+

src/lib/version/get_git_tag_or_branch_version.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Script to extract PX4_GIT_TAG_OR_BRANCH_NAME to be used by CI
+
+build_dir="$1"
+if [ ! -d "$build_dir" ]; then
+	echo "usage: $0 <build directory>"
+	exit -1
+fi
+
+version_file="$build_dir/src/lib/version/build_git_version.h"
+
+if [ ! -f "$version_file" ]; then
+	echo "Version file not found. Did you run the target 'ver_gen'?"
+	exit -1
+fi
+
+sed -n 's/.*PX4_GIT_TAG_OR_BRANCH_NAME\s*"\(.*\)".*/version=\1/p' "$version_file"
+

src/lib/version/px_update_git_header.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 
 import argparse
 import os
@@ -37,8 +36,17 @@ header = """
 
 
 # PX4
-git_tag = subprocess.check_output('git describe --exclude ext/* --always --tags --dirty'.split(),
+git_describe_cmd = 'git describe --exclude ext/* --always --tags --dirty'
+git_tag = subprocess.check_output(git_describe_cmd.split(),
                                   stderr=subprocess.STDOUT).decode('utf-8').strip()
+
+try:
+    # get the tag if we're on a tagged commit
+    tag_or_branch = subprocess.check_output((git_describe_cmd+' --exact-match').split(),
+                                            stderr=subprocess.STDOUT).decode('utf-8').strip()
+except:
+    tag_or_branch = None
+
 if validate:
     if verbose:
         print("testing git tag: "+git_tag)
@@ -92,18 +100,23 @@ try:
 except:
     oem_tag = ''
 
-header += """
-#define PX4_GIT_VERSION_STR  "{git_version}"
+if tag_or_branch is None:
+    # replace / so it can be used as directory name
+    tag_or_branch = git_branch_name.replace('/', '-')
+    # either a release or master branch (used for metadata)
+    if not tag_or_branch.startswith('release-'):
+        tag_or_branch = 'master'
+
+header += f"""
+#define PX4_GIT_VERSION_STR "{git_version}"
 #define PX4_GIT_VERSION_BINARY 0x{git_version_short}
-#define PX4_GIT_TAG_STR  "{git_tag}"
-#define PX4_GIT_BRANCH_NAME  "{git_branch_name}"
+#define PX4_GIT_TAG_STR "{git_tag}"
+#define PX4_GIT_BRANCH_NAME "{git_branch_name}"
 
 #define PX4_GIT_OEM_VERSION_STR  "{oem_tag}"
-""".format(git_tag=git_tag,
-           git_version=git_version,
-           git_version_short=git_version_short,
-           git_branch_name=git_branch_name,
-           oem_tag=oem_tag)
+
+#define PX4_GIT_TAG_OR_BRANCH_NAME "{tag_or_branch}" // special variable: git tag, release or master branch
+"""
 
 
 # ECL
@@ -115,11 +128,10 @@ if (os.path.exists('src/lib/ecl/.git')):
                                       cwd='src/lib/ecl', stderr=subprocess.STDOUT).decode('utf-8').strip()
     ecl_git_version_short = ecl_git_version[0:16]
 
-    header += """
+    header += f"""
 #define ECL_LIB_GIT_VERSION_STR  "{ecl_git_version}"
 #define ECL_LIB_GIT_VERSION_BINARY 0x{ecl_git_version_short}
-""".format(ecl_git_version=ecl_git_version,
-           ecl_git_version_short=ecl_git_version_short)
+"""
 
 
 # Mavlink
@@ -128,11 +140,10 @@ if (os.path.exists('mavlink/include/mavlink/v2.0/.git')):
                                       cwd='mavlink/include/mavlink/v2.0', stderr=subprocess.STDOUT).decode('utf-8').strip()
     mavlink_git_version_short = mavlink_git_version[0:16]
 
-    header += """
+    header += f"""
 #define MAVLINK_LIB_GIT_VERSION_STR  "{mavlink_git_version}"
 #define MAVLINK_LIB_GIT_VERSION_BINARY 0x{mavlink_git_version_short}
-""".format(mavlink_git_version=mavlink_git_version,
-           mavlink_git_version_short=mavlink_git_version_short)
+"""
 
 
 # NuttX
@@ -144,13 +155,11 @@ if (os.path.exists('platforms/nuttx/NuttX/nuttx/.git')):
                                       cwd='platforms/nuttx/NuttX/nuttx', stderr=subprocess.STDOUT).decode('utf-8').strip()
     nuttx_git_version_short = nuttx_git_version[0:16]
 
-    header += """
+    header += f"""
 #define NUTTX_GIT_VERSION_STR  "{nuttx_git_version}"
 #define NUTTX_GIT_VERSION_BINARY 0x{nuttx_git_version_short}
 #define NUTTX_GIT_TAG_STR  "{nuttx_git_tag}"
-""".format(nuttx_git_version=nuttx_git_version,
-           nuttx_git_version_short=nuttx_git_version_short,
-           nuttx_git_tag=nuttx_git_tag)
+"""
 
 
 if old_header != header: