python scripts code coverage (#10157)

.gitignore

@@ -63,3 +63,5 @@ posix-configs/SITL/init/test/*_generated
 /modules
 
 *.gcov
+.coverage
+.coverage.*

CMakeLists.txt

@@ -261,7 +261,15 @@ if (CATKIN_DEVEL_PREFIX)
 endif()
 
 find_package(PythonInterp REQUIRED)
+
+option(PYTHON_COVERAGE "Python code coverage" OFF)
+if(PYTHON_COVERAGE)
+	message(STATUS "python coverage enabled")
+	set(PYTHON_EXECUTABLE coverage run -p)
+else()
+	# run normally (broken under coveragepy)
 	px4_find_python_module(jinja2 REQUIRED)
+endif()
 
 #=============================================================================
 # check required toolchain variables
@@ -453,6 +461,51 @@ if (BUILD_DOXYGEN)
 	endif()
 endif()
 
+#=============================================================================
+# Metadata - helpers for generating documentation
+#
+
+add_custom_target(metadata_airframes
+	COMMAND ${CMAKE_COMMAND} -E make_directory ${PX4_BINARY_DIR}/docs
+	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/px_process_airframes.py
+		-v -a ${PX4_SOURCE_DIR}//ROMFS/px4fmu_common/init.d
+		--markdown ${PX4_BINARY_DIR}/docs/airframes.md
+	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/px_process_airframes.py
+		-v -a ${PX4_SOURCE_DIR}//ROMFS/px4fmu_common/init.d
+		--xml ${PX4_BINARY_DIR}/docs/airframes.xml
+	COMMENT "Generating full airframe metadata (markdown and xml)"
+	USES_TERMINAL
+)
+
+add_custom_target(metadata_parameters
+	COMMAND ${CMAKE_COMMAND} -E make_directory ${PX4_BINARY_DIR}/docs
+	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py
+		--src-path `find ${PX4_SOURCE_DIR}/src -maxdepth 4 -type d`
+		--inject-xml ${PX4_SOURCE_DIR}/src/lib/parameters/parameters_injected.xml
+		--markdown ${PX4_BINARY_DIR}/docs/parameters.md
+	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py
+		--src-path `find ${PX4_SOURCE_DIR}/src -maxdepth 4 -type d`
+		--inject-xml ${PX4_SOURCE_DIR}/src/lib/parameters/parameters_injected.xml
+		--xml ${PX4_BINARY_DIR}/docs/parameters.xml
+	COMMENT "Generating full parameter metadata (markdown and xml)"
+	USES_TERMINAL
+)
+
+add_custom_target(metadata_module_documentation
+	COMMAND ${CMAKE_COMMAND} -E make_directory ${PX4_BINARY_DIR}/docs
+	COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/px_process_module_doc.py -v --src-path ${PX4_SOURCE_DIR}/src
+		--markdown ${PX4_BINARY_DIR}/docs/modules
+	COMMENT "Generating module documentation"
+	USES_TERMINAL
+)
+
+add_custom_target(all_metadata
+	DEPENDS
+		metadata_airframes
+		metadata_parameters
+		metadata_module_documentation
+)
+
 #=============================================================================
 # packaging
 #

Jenkinsfile

@@ -188,7 +188,7 @@ pipeline {
           steps {
             sh 'export'
             sh 'make distclean'
-            sh 'ulimit -c unlimited; make tests_coverage'
+            sh 'ulimit -c unlimited; make tests_coverage || true' // always pass for now
             withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
               sh 'curl -s https://codecov.io/bash | bash -s - -F unittests'
             }
@@ -203,6 +203,24 @@ pipeline {
           }
         }
 
+        stage('code coverage (python)') {
+          agent {
+            docker {
+              image 'px4io/px4-dev-base:2018-08-04'
+              args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw'
+            }
+          }
+          steps {
+            sh 'export'
+            sh 'make distclean'
+            sh 'make python_coverage'
+            withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) {
+              sh 'curl -s https://codecov.io/bash | bash -s - -F python'
+            }
+            sh 'make distclean'
+          }
+        }
+
       } // parallel
     } // stage Analysis
 
@@ -262,7 +280,9 @@ pipeline {
           steps {
             sh 'make distclean'
             sh 'make airframe_metadata'
-            archiveArtifacts(artifacts: 'airframes.md, airframes.xml', fingerprint: true)
+            dir('build/posix_sitl_default/docs') {
+              archiveArtifacts(artifacts: 'airframes.md, airframes.xml')
+            }
             sh 'make distclean'
           }
         }
@@ -274,7 +294,9 @@ pipeline {
           steps {
             sh 'make distclean'
             sh 'make parameters_metadata'
-            archiveArtifacts(artifacts: 'parameters.md, parameters.xml', fingerprint: true)
+            dir('build/posix_sitl_default/docs') {
+              archiveArtifacts(artifacts: 'parameters.md, parameters.xml')
+            }
             sh 'make distclean'
           }
         }
@@ -286,7 +308,9 @@ pipeline {
           steps {
             sh 'make distclean'
             sh 'make module_documentation'
-            archiveArtifacts(artifacts: 'modules/*.md', fingerprint: true)
+            dir('build/posix_sitl_default/docs') {
+              archiveArtifacts(artifacts: 'modules/*.md')
+            }
             sh 'make distclean'
           }
         }
@@ -302,7 +326,9 @@ pipeline {
             sh 'export'
             sh 'make distclean'
             sh 'make uorb_graphs'
-            archiveArtifacts(artifacts: 'Tools/uorb_graph/graph_sitl.json')
+            dir('Tools/uorb_graph') {
+              archiveArtifacts(artifacts: 'graph_sitl.json')
+            }
             sh 'make distclean'
           }
         }

Makefile

@@ -253,15 +253,13 @@ coverity_scan: posix_sitl_default
 .PHONY: parameters_metadata airframe_metadata module_documentation px4_metadata doxygen
 
 parameters_metadata:
-	@python $(SRC_DIR)/src/lib/parameters/px_process_params.py -s `find $(SRC_DIR)/src -maxdepth 4 -type d` --inject-xml $(SRC_DIR)/src/lib/parameters/parameters_injected.xml --markdown
+	@$(MAKE) --no-print-directory posix_sitl_default metadata_parameters
-	@python $(SRC_DIR)/src/lib/parameters/px_process_params.py -s `find $(SRC_DIR)/src -maxdepth 4 -type d` --inject-xml $(SRC_DIR)/src/lib/parameters/parameters_injected.xml --xml
 
 airframe_metadata:
-	@python $(SRC_DIR)/Tools/px_process_airframes.py -v -a $(SRC_DIR)/ROMFS/px4fmu_common/init.d --markdown
+	@$(MAKE) --no-print-directory posix_sitl_default metadata_airframes
-	@python $(SRC_DIR)/Tools/px_process_airframes.py -v -a $(SRC_DIR)/ROMFS/px4fmu_common/init.d --xml
 
 module_documentation:
-	@python $(SRC_DIR)/Tools/px_process_module_doc.py -v --markdown $(SRC_DIR)/modules --src-path $(SRC_DIR)/src
+	@$(MAKE) --no-print-directory posix_sitl_default metadata_module_documentation
 
 px4_metadata: parameters_metadata airframe_metadata module_documentation
 
@@ -285,7 +283,7 @@ format:
 
 # Testing
 # --------------------------------------------------------------------
-.PHONY: tests tests_coverage tests_mission tests_mission_coverage tests_offboard rostest
+.PHONY: tests tests_coverage tests_mission tests_mission_coverage tests_offboard rostest python_coverage
 
 tests:
 	@$(MAKE) --no-print-directory posix_sitl_default test_results \
@@ -314,6 +312,16 @@ tests_offboard: rostest
 	@$(SRC_DIR)/test/rostest_px4_run.sh mavros_posix_tests_offboard_attctl.test
 	@$(SRC_DIR)/test/rostest_px4_run.sh mavros_posix_tests_offboard_posctl.test
 
+python_coverage:
+	@mkdir -p $(SRC_DIR)/build/python_coverage
+	@cd $(SRC_DIR)/build/python_coverage && cmake $(SRC_DIR) $(CMAKE_ARGS) -G"$(PX4_CMAKE_GENERATOR)" -DCONFIG=posix_sitl_default -DPYTHON_COVERAGE=ON
+	@$(PX4_MAKE) -C $(SRC_DIR)/build/python_coverage
+	@$(PX4_MAKE) -C $(SRC_DIR)/build/python_coverage metadata_airframes
+	@$(PX4_MAKE) -C $(SRC_DIR)/build/python_coverage metadata_parameters
+	#@$(PX4_MAKE) -C $(SRC_DIR)/build/python_coverage module_documentation # TODO: fix within coverage.py
+	@coverage combine `find . -name .coverage\*`
+	@coverage report -m
+
 # static analyzers (scan-build, clang-tidy, cppcheck)
 # --------------------------------------------------------------------
 .PHONY: scan-build posix_sitl_default-clang clang-tidy clang-tidy-fix clang-tidy-quiet cppcheck