sanitizers cleanup (#10551)

- add to AddressSanitizer and UndefinedBehaviorSanitizer to CMAKE_BUILD_TYPE options
 - handle environment variable helpers outside of CMake
 - add -O1 optimization
 - cleanup whitespace

.ci/Jenkinsfile-SITL_tests

@@ -6,27 +6,13 @@ pipeline {
   parameters {
     choice(
       name: 'PX4_CMAKE_BUILD_TYPE',
-      choices: ['RelWithDebInfo', 'Coverage', 'AddressSanitizer'],
+      choices: ['RelWithDebInfo', 'Coverage', 'AddressSanitizer', 'UndefinedBehaviorSanitizer'],
       description: "CMake build type"
       )
   }
 
   stages {
 
-    stage('Setup Environment') {
-      steps {
-        script {
-
-          env.CTEST_OUTPUT_ON_FAILURE=1
-
-          if (env.PX4_CMAKE_BUILD_TYPE == 'AddressSanitizer') {
-            env.PX4_ASAN=1
-            env.ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1"
-          }
-        }
-      }
-    }
-
     stage('Build') {
 
       agent {
@@ -206,8 +192,10 @@ pipeline {
 
   } //stages
   environment {
+    ASAN_OPTIONS = 'detect_stack_use_after_return=1:check_initialization_order=1'
     CCACHE_DIR = '/tmp/ccache'
     CI = true
+    CTEST_OUTPUT_ON_FAILURE = 1
   }
   options {
     buildDiscarder(logRotator(numToKeepStr: '10', artifactDaysToKeepStr: '30'))

CMakeLists.txt

@@ -160,7 +160,7 @@ if (NOT CMAKE_BUILD_TYPE)
 	set(CMAKE_BUILD_TYPE ${PX4_BUILD_TYPE} CACHE STRING "Build type" FORCE)
 endif()
 
-set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug;Release;RelWithDebInfo;MinSizeRel;Coverage")
+set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug;Release;RelWithDebInfo;MinSizeRel;Coverage;AddressSanitizer;UndefinedBehaviorSanitizer")
 
 #=============================================================================
 

Makefile

@@ -114,6 +114,28 @@ endif
 
 ifdef PX4_CMAKE_BUILD_TYPE
 	CMAKE_ARGS += -DCMAKE_BUILD_TYPE=${PX4_CMAKE_BUILD_TYPE}
+else
+
+	# Address Sanitizer
+	ifdef PX4_ASAN
+		CMAKE_ARGS += -DCMAKE_BUILD_TYPE=AddressSanitizer
+	endif
+
+	# Memory Sanitizer
+	ifdef PX4_MSAN
+		CMAKE_ARGS += -DCMAKE_BUILD_TYPE=MemorySanitizer
+	endif
+
+	# Thread Sanitizer
+	ifdef PX4_TSAN
+		CMAKE_ARGS += -DCMAKE_BUILD_TYPE=ThreadSanitizer
+	endif
+
+	# Undefined Behavior Sanitizer
+	ifdef PX4_UBSAN
+		CMAKE_ARGS += -DCMAKE_BUILD_TYPE=UndefinedBehaviorSanitizer
+	endif
+
 endif
 
 # Functions

cmake/common/sanitizers.cmake

@@ -31,33 +31,20 @@
 #
 ############################################################################
 
-option(SANITIZE_ADDRESS "Enable AddressSanitizer" Off)
+if (CMAKE_BUILD_TYPE STREQUAL AddressSanitizer)
-option(SANITIZE_MEMORY "Enable MemorySanitizer" Off)
-option(SANITIZE_THREAD "Enable ThreadSanitizer" Off)
-option(SANITIZE_UNDEFINED "Enable UndefinedBehaviorSanitizer" Off)
-
-if(DEFINED ENV{PX4_ASAN})
-	set(SANITIZE_ADDRESS ON)
-elseif(DEFINED ENV{PX4_MSAN})
-	set(SANITIZE_MEMORY ON)
-elseif(DEFINED ENV{PX4_TSAN})
-	set(SANITIZE_THREAD ON)
-elseif(DEFINED ENV{PX4_UBSAN})
-	set(SANITIZE_UNDEFINED ON)
-endif()
-
-if (SANITIZE_ADDRESS)
 	message(STATUS "AddressSanitizer enabled")
 
 	# environment variables
-        #  ASAN_OPTIONS=detect_stack_use_after_return=1
+	#  ASAN_OPTIONS=check_initialization_order=1,detect_stack_use_after_return=1
-        #  ASAN_OPTIONS=check_initialization_order=1
 	add_compile_options(
+		-O1
 		-g3
-                -fno-omit-frame-pointer
+
 		-fsanitize=address
-		#-fsanitize-address-use-after-scope
+
-		-fno-optimize-sibling-calls
+		-fno-omit-frame-pointer # Leave frame pointers. Allows the fast unwinder to function properly.
+		-fno-common # Do not treat global variable in C as common variables (allows ASan to instrument them)
+		-fno-optimize-sibling-calls # disable inlining and and tail call elimination for perfect stack traces
 	)
 	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address" CACHE INTERNAL "" FORCE)
 	set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address" CACHE INTERNAL "" FORCE)
@@ -68,37 +55,47 @@ if (SANITIZE_ADDRESS)
 		set_tests_properties(${test_name} PROPERTIES FAIL_REGULAR_EXPRESSION "ERROR: LeakSanitizer")
 	endfunction(sanitizer_fail_test_on_error)
 
-elseif(SANITIZE_MEMORY)
+elseif (CMAKE_BUILD_TYPE STREQUAL MemorySanitizer)
 	message(STATUS "MemorySanitizer enabled")
 
 	add_compile_options(
+		-O1
 		-g3
+
 		-fsanitize=memory
+		-fsanitize-memory-track-origins
+
+		-fno-omit-frame-pointer # Leave frame pointers. Allows the fast unwinder to function properly.
+		-fno-common # Do not treat global variable in C as common variables (allows ASan to instrument them)
+		-fno-optimize-sibling-calls # disable inlining and and tail call elimination for perfect stack traces
 	)
+
 	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory" CACHE INTERNAL "" FORCE)
 
 	function(sanitizer_fail_test_on_error test_name)
 		# TODO add right check here
 	endfunction(sanitizer_fail_test_on_error)
 
-elseif(SANITIZE_THREAD)
+elseif (CMAKE_BUILD_TYPE STREQUAL ThreadSanitizer)
 	message(STATUS "ThreadSanitizer enabled")
 
 	add_compile_options(
 		-g3
 		-fsanitize=thread
 	)
+
 	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread" CACHE INTERNAL "" FORCE)
 
 	function(sanitizer_fail_test_on_error test_name)
 		# TODO add right check here
 	endfunction(sanitizer_fail_test_on_error)
 
-elseif(SANITIZE_UNDEFINED)
+elseif (CMAKE_BUILD_TYPE STREQUAL UndefinedBehaviorSanitizer)
 	message(STATUS "UndefinedBehaviorSanitizer enabled")
 
 	add_compile_options(
 		-g3
+
 		#-fsanitize=alignment
 		-fsanitize=bool
 		#-fsanitize=builtin
@@ -126,6 +123,7 @@ elseif(SANITIZE_UNDEFINED)
 
 		-fno-sanitize-recover=bounds,null
 	)
+
 	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined" CACHE INTERNAL "" FORCE)
 
 	function(sanitizer_fail_test_on_error test_name)