github ci: push parameter metadata to s3

This commit is contained in:
Beat Küng 2021-03-22 11:58:22 +01:00 committed by Daniel Agar
parent c3985709e4
commit f528c5d206
6 changed files with 235 additions and 28 deletions

77
.github/workflows/deploy_all.yml vendored Normal file
View File

@ -0,0 +1,77 @@
name: Deploy metadata for all targets
on:
push:
branches:
- 'master'
- 'release/*'
- 'pr-metadata-test'
jobs:
enumerate_targets:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/checkout@v1
with:
token: ${{secrets.ACCESS_TOKEN}}
- id: set-matrix
run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)"
build:
runs-on: ubuntu-latest
needs: enumerate_targets
strategy:
matrix: ${{fromJson(needs.enumerate_targets.outputs.matrix)}}
container: px4io/px4-dev-${{ matrix.container }}:2021-02-04
steps:
- uses: actions/checkout@v1
with:
token: ${{secrets.ACCESS_TOKEN}}
- name: Prepare ccache timestamp
id: ccache_cache_timestamp
shell: cmake -P {0}
run: |
string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
message("::set-output name=timestamp::${current_date}")
- name: ccache cache files
uses: actions/cache@v2
with:
path: ~/.ccache
key: ${{matrix.target}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
restore-keys: ${{matrix.target}}-ccache-
- name: setup ccache
run: |
mkdir -p ~/.ccache
echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
echo "compression = true" >> ~/.ccache/ccache.conf
echo "compression_level = 5" >> ~/.ccache/ccache.conf
echo "max_size = 100M" >> ~/.ccache/ccache.conf
ccache -s
ccache -z
- name: make ${{matrix.target}}
run: make ${{matrix.target}}
- name: ccache post-run
run: ccache -s
- name: parameter metadata
run: |
make ${{matrix.target}} ver_gen
./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV
cd build/${{ matrix.target }}
mkdir _metadata || true
cp parameters.* _metadata
- uses: jakejarvis/s3-sync-action@master
with:
args: --acl public-read
env:
AWS_S3_BUCKET: 'px4-travis'
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: 'us-west-1'
SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/'
DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/'

View File

@ -4,9 +4,8 @@ on:
push:
branches:
- 'master'
pull_request:
branches:
- '*'
- 'release/*'
- 'pr-metadata-test'
jobs:
@ -51,9 +50,18 @@ jobs:
- name: parameter metadata
run: |
make parameters_metadata
cd build/px4_sitl_default/docs
ls -ls *
# TODO: deploy 'parameters.md, parameters.xml to S3 px4-travis:Firmware/master/
./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
- uses: jakejarvis/s3-sync-action@master
with:
args: --acl public-read
env:
AWS_S3_BUCKET: 'px4-travis'
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: 'us-west-1'
SOURCE_DIR: 'build/px4_sitl_default/docs/'
DEST_DIR: 'Firmware/${{ env.version }}/_general/'
uorb_graph:
runs-on: ubuntu-latest

View File

@ -318,7 +318,7 @@ coverity_scan: px4_sitl_default
.PHONY: parameters_metadata airframe_metadata module_documentation px4_metadata doxygen
parameters_metadata:
@$(MAKE) --no-print-directory px4_sitl_default metadata_parameters
@$(MAKE) --no-print-directory px4_sitl_default metadata_parameters ver_gen
airframe_metadata:
@$(MAKE) --no-print-directory px4_sitl_default metadata_airframes

View File

@ -0,0 +1,92 @@
#!/usr/bin/env python3
""" Script to generate a JSON config with all build targets (for CI) """
import argparse
import os
import sys
import json
import re
source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
parser = argparse.ArgumentParser(description='Generate build targets')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
help='Verbose Output')
parser.add_argument('-p', '--pretty', dest='pretty', action='store_true',
help='Pretty output instead of a single line')
args = parser.parse_args()
verbose = args.verbose
build_configs = []
excluded_manufacturers = ['atlflight']
excluded_platforms = ['qurt']
excluded_labels = ['stackcheck', 'nolockstep', 'replay', 'test']
def process_target(cmake_file, target_name):
ret = None
is_board_def = False
platform = None
toolchain = None
for line in open(cmake_file, 'r'):
if 'px4_add_board' in line:
is_board_def = True
if not is_board_def:
continue
re_platform = re.search('PLATFORM\s+([^\s]+)', line)
if re_platform: platform = re_platform.group(1)
re_toolchain = re.search('TOOLCHAIN\s+([^\s]+)', line)
if re_toolchain: toolchain = re_toolchain.group(1)
if is_board_def:
assert platform, f"PLATFORM not found in {cmake_file}"
if platform not in excluded_platforms:
# get the container based on the platform and toolchain
container = platform
if platform == 'posix':
container = 'base-focal'
if toolchain:
if toolchain.startswith('aarch64'):
container = 'aarch64'
elif toolchain == 'arm-linux-gnueabihf':
container = 'armhf'
else:
if verbose: print(f'possibly unmatched toolchain: {toolchain}')
elif platform == 'nuttx':
container = 'nuttx-focal'
ret = {'target': target_name, 'container': container}
return ret
for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
if not manufacturer.is_dir():
continue
if manufacturer.name in excluded_manufacturers:
if verbose: print(f'excluding manufacturer {manufacturer.name}')
continue
for board in os.scandir(manufacturer.path):
if not board.is_dir():
continue
for files in os.scandir(board.path):
if files.is_file() and files.name.endswith('.cmake'):
label = files.name[:-6]
target_name = manufacturer.name + '_' + board.name + '_' + label
if label in excluded_labels:
if verbose: print(f'excluding label {label} ({target_name})')
continue
target = process_target(files.path, target_name)
if target is not None:
build_configs.append(target)
github_action_config = { 'include': build_configs }
extra_args = {}
if args.pretty:
extra_args['indent'] = 2
print(json.dumps(github_action_config, **extra_args))

View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -e
# Script to extract PX4_GIT_TAG_OR_BRANCH_NAME to be used by CI
build_dir="$1"
if [ ! -d "$build_dir" ]; then
echo "usage: $0 <build directory>"
exit -1
fi
version_file="$build_dir/src/lib/version/build_git_version.h"
if [ ! -f "$version_file" ]; then
echo "Version file not found. Did you run the target 'ver_gen'?"
exit -1
fi
sed -n 's/.*PX4_GIT_TAG_OR_BRANCH_NAME\s*"\(.*\)".*/version=\1/p' "$version_file"

View File

@ -1,5 +1,4 @@
#!/usr/bin/env python
from __future__ import print_function
import argparse
import os
@ -37,8 +36,17 @@ header = """
# PX4
git_tag = subprocess.check_output('git describe --exclude ext/* --always --tags --dirty'.split(),
git_describe_cmd = 'git describe --exclude ext/* --always --tags --dirty'
git_tag = subprocess.check_output(git_describe_cmd.split(),
stderr=subprocess.STDOUT).decode('utf-8').strip()
try:
# get the tag if we're on a tagged commit
tag_or_branch = subprocess.check_output((git_describe_cmd+' --exact-match').split(),
stderr=subprocess.STDOUT).decode('utf-8').strip()
except:
tag_or_branch = None
if validate:
if verbose:
print("testing git tag: "+git_tag)
@ -92,18 +100,23 @@ try:
except:
oem_tag = ''
header += """
if tag_or_branch is None:
# replace / so it can be used as directory name
tag_or_branch = git_branch_name.replace('/', '-')
# either a release or master branch (used for metadata)
if not tag_or_branch.startswith('release-'):
tag_or_branch = 'master'
header += f"""
#define PX4_GIT_VERSION_STR "{git_version}"
#define PX4_GIT_VERSION_BINARY 0x{git_version_short}
#define PX4_GIT_TAG_STR "{git_tag}"
#define PX4_GIT_BRANCH_NAME "{git_branch_name}"
#define PX4_GIT_OEM_VERSION_STR "{oem_tag}"
""".format(git_tag=git_tag,
git_version=git_version,
git_version_short=git_version_short,
git_branch_name=git_branch_name,
oem_tag=oem_tag)
#define PX4_GIT_TAG_OR_BRANCH_NAME "{tag_or_branch}" // special variable: git tag, release or master branch
"""
# ECL
@ -115,11 +128,10 @@ if (os.path.exists('src/lib/ecl/.git')):
cwd='src/lib/ecl', stderr=subprocess.STDOUT).decode('utf-8').strip()
ecl_git_version_short = ecl_git_version[0:16]
header += """
header += f"""
#define ECL_LIB_GIT_VERSION_STR "{ecl_git_version}"
#define ECL_LIB_GIT_VERSION_BINARY 0x{ecl_git_version_short}
""".format(ecl_git_version=ecl_git_version,
ecl_git_version_short=ecl_git_version_short)
"""
# Mavlink
@ -128,11 +140,10 @@ if (os.path.exists('mavlink/include/mavlink/v2.0/.git')):
cwd='mavlink/include/mavlink/v2.0', stderr=subprocess.STDOUT).decode('utf-8').strip()
mavlink_git_version_short = mavlink_git_version[0:16]
header += """
header += f"""
#define MAVLINK_LIB_GIT_VERSION_STR "{mavlink_git_version}"
#define MAVLINK_LIB_GIT_VERSION_BINARY 0x{mavlink_git_version_short}
""".format(mavlink_git_version=mavlink_git_version,
mavlink_git_version_short=mavlink_git_version_short)
"""
# NuttX
@ -144,13 +155,11 @@ if (os.path.exists('platforms/nuttx/NuttX/nuttx/.git')):
cwd='platforms/nuttx/NuttX/nuttx', stderr=subprocess.STDOUT).decode('utf-8').strip()
nuttx_git_version_short = nuttx_git_version[0:16]
header += """
header += f"""
#define NUTTX_GIT_VERSION_STR "{nuttx_git_version}"
#define NUTTX_GIT_VERSION_BINARY 0x{nuttx_git_version_short}
#define NUTTX_GIT_TAG_STR "{nuttx_git_tag}"
""".format(nuttx_git_version=nuttx_git_version,
nuttx_git_version_short=nuttx_git_version_short,
nuttx_git_tag=nuttx_git_tag)
"""
if old_header != header: