diff --git a/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst new file mode 100644 index 00000000000..29a6ff92554 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2021-08-27-16-55-10.bpo-34602.ZjHsYJ.rst @@ -0,0 +1,3 @@ +When building CPython on macOS with ``./configure +--with-undefined-behavior-sanitizer --with-pydebug``, the stack size is now +quadrupled to allow for the entire test suite to pass. diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index ec7d737518b..a45d842ffe7 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -32,18 +32,17 @@ #define THREAD_STACK_SIZE 0 /* use default stack size */ #endif -/* The default stack size for new threads on OSX and BSD is small enough that +/* The default stack size for new threads on BSD is small enough that * we'll get hard crashes instead of 'maximum recursion depth exceeded' * exceptions. * - * The default stack sizes below are the empirically determined minimal stack + * The default stack size below is the empirically determined minimal stack * sizes where a simple recursive function doesn't cause a hard crash. + * + * For macOS the value of THREAD_STACK_SIZE is determined in configure.ac + * as it also depends on the other configure options like chosen sanitizer + * runtimes. */ -#if defined(__APPLE__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 -#undef THREAD_STACK_SIZE -/* Note: This matches the value of -Wl,-stack_size in configure.ac */ -#define THREAD_STACK_SIZE 0x1000000 -#endif #if defined(__FreeBSD__) && defined(THREAD_STACK_SIZE) && THREAD_STACK_SIZE == 0 #undef THREAD_STACK_SIZE #define THREAD_STACK_SIZE 0x400000 diff --git a/configure b/configure index 050f1836011..4f12972540d 100755 --- a/configure +++ b/configure @@ -827,11 +827,11 @@ with_trace_refs with_assertions enable_optimizations with_lto -with_hash_algorithm -with_tzpath with_address_sanitizer with_memory_sanitizer with_undefined_behavior_sanitizer +with_hash_algorithm +with_tzpath with_libs with_system_expat with_system_ffi @@ -1548,12 +1548,6 @@ Optional Packages: --with-lto=[full|thin|no|yes] enable Link-Time-Optimization in any build (default is no) - --with-hash-algorithm=[fnv|siphash24] - select hash algorithm for use in Python/pyhash.c - (default is SipHash24) - --with-tzpath= - Select the default time zone search path for zoneinfo.TZPATH - --with-address-sanitizer enable AddressSanitizer memory error detector, 'asan' (default is no) @@ -1562,6 +1556,12 @@ Optional Packages: --with-undefined-behavior-sanitizer enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no) + --with-hash-algorithm=[fnv|siphash24] + select hash algorithm for use in Python/pyhash.c + (default is SipHash24) + --with-tzpath= + Select the default time zone search path for zoneinfo.TZPATH + --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) @@ -9602,6 +9602,65 @@ $as_echo "no" >&6; } ;; esac +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5 +$as_echo_n "checking for --with-address-sanitizer... " >&6; } + +# Check whether --with-address_sanitizer was given. +if test "${with_address_sanitizer+set}" = set; then : + withval=$with_address_sanitizer; +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +$as_echo "$withval" >&6; } +BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=address $LDFLAGS" +# ASan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5 +$as_echo_n "checking for --with-memory-sanitizer... " >&6; } + +# Check whether --with-memory_sanitizer was given. +if test "${with_memory_sanitizer+set}" = set; then : + withval=$with_memory_sanitizer; +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +$as_echo "$withval" >&6; } +BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" +# MSan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5 +$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; } + +# Check whether --with-undefined_behavior_sanitizer was given. +if test "${with_undefined_behavior_sanitizer+set}" = set; then : + withval=$with_undefined_behavior_sanitizer; +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 +$as_echo "$withval" >&6; } +BASECFLAGS="-fsanitize=undefined $BASECFLAGS" +LDFLAGS="-fsanitize=undefined $LDFLAGS" +with_ubsan="yes" + +else + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +with_ubsan="no" + +fi + + # Set info about shared libraries. @@ -9812,9 +9871,20 @@ then # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - # Note: This matches the value of THREAD_STACK_SIZE in - # thread_pthread.h - LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED" + stack_size="1000000" # 16 MB + if test "$with_ubsan" == "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi + + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + + +cat >>confdefs.h <<_ACEOF +#define THREAD_STACK_SIZE 0x$stack_size +_ACEOF + if test "$enable_framework" then @@ -10410,61 +10480,6 @@ fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-address-sanitizer" >&5 -$as_echo_n "checking for --with-address-sanitizer... " >&6; } - -# Check whether --with-address_sanitizer was given. -if test "${with_address_sanitizer+set}" = set; then : - withval=$with_address_sanitizer; -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 -$as_echo "$withval" >&6; } -BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=address $LDFLAGS" -# ASan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-memory-sanitizer" >&5 -$as_echo_n "checking for --with-memory-sanitizer... " >&6; } - -# Check whether --with-memory_sanitizer was given. -if test "${with_memory_sanitizer+set}" = set; then : - withval=$with_memory_sanitizer; -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 -$as_echo "$withval" >&6; } -BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" -# MSan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-undefined-behavior-sanitizer" >&5 -$as_echo_n "checking for --with-undefined-behavior-sanitizer... " >&6; } - -# Check whether --with-undefined_behavior_sanitizer was given. -if test "${with_undefined_behavior_sanitizer+set}" = set; then : - withval=$with_undefined_behavior_sanitizer; -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $withval" >&5 -$as_echo "$withval" >&6; } -BASECFLAGS="-fsanitize=undefined $BASECFLAGS" -LDFLAGS="-fsanitize=undefined $LDFLAGS" - -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - # Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for t_open in -lnsl" >&5 $as_echo_n "checking for t_open in -lnsl... " >&6; } diff --git a/configure.ac b/configure.ac index aaff79f720e..e01e0c13fba 100644 --- a/configure.ac +++ b/configure.ac @@ -2595,6 +2595,47 @@ case $ac_sys_system/$ac_sys_release in ;; esac +AC_MSG_CHECKING(for --with-address-sanitizer) +AC_ARG_WITH(address_sanitizer, + AS_HELP_STRING([--with-address-sanitizer], + [enable AddressSanitizer memory error detector, 'asan' (default is no)]), +[ +AC_MSG_RESULT($withval) +BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=address $LDFLAGS" +# ASan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" +], +[AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING(for --with-memory-sanitizer) +AC_ARG_WITH(memory_sanitizer, + AS_HELP_STRING([--with-memory-sanitizer], + [enable MemorySanitizer allocation error detector, 'msan' (default is no)]), +[ +AC_MSG_RESULT($withval) +BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" +LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" +# MSan works by controlling memory allocation, our own malloc interferes. +with_pymalloc="no" +], +[AC_MSG_RESULT(no)]) + +AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer) +AC_ARG_WITH(undefined_behavior_sanitizer, + AS_HELP_STRING([--with-undefined-behavior-sanitizer], + [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]), +[ +AC_MSG_RESULT($withval) +BASECFLAGS="-fsanitize=undefined $BASECFLAGS" +LDFLAGS="-fsanitize=undefined $LDFLAGS" +with_ubsan="yes" +], +[ +AC_MSG_RESULT(no) +with_ubsan="no" +]) + # Set info about shared libraries. AC_SUBST(SHLIB_SUFFIX) AC_SUBST(LDSHARED) @@ -2798,9 +2839,18 @@ then # Issue #18075: the default maximum stack size (8MBytes) is too # small for the default recursion limit. Increase the stack size # to ensure that tests don't crash - # Note: This matches the value of THREAD_STACK_SIZE in - # thread_pthread.h - LINKFORSHARED="-Wl,-stack_size,1000000 $LINKFORSHARED" + stack_size="1000000" # 16 MB + if test "$with_ubsan" == "yes" + then + # Undefined behavior sanitizer requires an even deeper stack + stack_size="4000000" # 64 MB + fi + + LINKFORSHARED="-Wl,-stack_size,$stack_size $LINKFORSHARED" + + AC_DEFINE_UNQUOTED(THREAD_STACK_SIZE, + 0x$stack_size, + [Custom thread stack size depending on chosen sanitizer runtimes.]) if test "$enable_framework" then @@ -3044,43 +3094,6 @@ esac AC_MSG_RESULT("$TZPATH")]) AC_SUBST(TZPATH) -AC_MSG_CHECKING(for --with-address-sanitizer) -AC_ARG_WITH(address_sanitizer, - AS_HELP_STRING([--with-address-sanitizer], - [enable AddressSanitizer memory error detector, 'asan' (default is no)]), -[ -AC_MSG_RESULT($withval) -BASECFLAGS="-fsanitize=address -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=address $LDFLAGS" -# ASan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" -], -[AC_MSG_RESULT(no)]) - -AC_MSG_CHECKING(for --with-memory-sanitizer) -AC_ARG_WITH(memory_sanitizer, - AS_HELP_STRING([--with-memory-sanitizer], - [enable MemorySanitizer allocation error detector, 'msan' (default is no)]), -[ -AC_MSG_RESULT($withval) -BASECFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer $BASECFLAGS" -LDFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 $LDFLAGS" -# MSan works by controlling memory allocation, our own malloc interferes. -with_pymalloc="no" -], -[AC_MSG_RESULT(no)]) - -AC_MSG_CHECKING(for --with-undefined-behavior-sanitizer) -AC_ARG_WITH(undefined_behavior_sanitizer, - AS_HELP_STRING([--with-undefined-behavior-sanitizer], - [enable UndefinedBehaviorSanitizer undefined behaviour detector, 'ubsan' (default is no)]), -[ -AC_MSG_RESULT($withval) -BASECFLAGS="-fsanitize=undefined $BASECFLAGS" -LDFLAGS="-fsanitize=undefined $LDFLAGS" -], -[AC_MSG_RESULT(no)]) - # Most SVR4 platforms (e.g. Solaris) need -lsocket and -lnsl. AC_CHECK_LIB(nsl, t_open, [LIBS="-lnsl $LIBS"]) # SVR4 AC_CHECK_LIB(socket, socket, [LIBS="-lsocket $LIBS"], [], $LIBS) # SVR4 sockets diff --git a/pyconfig.h.in b/pyconfig.h.in index 63438d857a0..49407ab62b4 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1509,6 +1509,9 @@ (which you can't on SCO ODT 3.0). */ #undef SYS_SELECT_WITH_SYS_TIME +/* Custom thread stack size depending on chosen sanitizer runtimes. */ +#undef THREAD_STACK_SIZE + /* Library needed by timemodule.c: librt may be needed for clock_gettime() */ #undef TIMEMODULE_LIB