bpo-36465: Make release and debug ABI compatible (GH-12615)

Release build and debug build are now ABI compatible: the Py_DEBUG
define no longer implies Py_TRACE_REFS define which introduces the
only ABI incompatibility.

A new "./configure --with-trace-refs" build option is now required to
get Py_TRACE_REFS define which adds sys.getobjects() function and
PYTHONDUMPREFS environment variable.

Changes:

* Add ./configure --with-trace-refs
* Py_DEBUG no longer implies Py_TRACE_REFS

Doc/using/cmdline.rst

@@ -922,15 +922,18 @@ conflict.
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~
 
-Setting these variables only has an effect in a debug build of Python, that is,
-if Python was configured with the ``--with-pydebug`` build option.
+Setting these variables only has an effect in a debug build of Python.
 
 .. envvar:: PYTHONTHREADDEBUG
 
    If set, Python will print threading debug info.
 
+   Need Python configured with the ``--with-pydebug`` build option.
+
 
 .. envvar:: PYTHONDUMPREFS
 
    If set, Python will dump objects and reference counts still alive after
    shutting down the interpreter.
+
+   Need Python configured with the ``--with-trace-refs`` build option.

Include/object.h

@@ -54,13 +54,8 @@ A standard interface exists for objects that contain an array of items
 whose size is determined when the object is allocated.
 */
 
-/* Py_DEBUG implies Py_TRACE_REFS. */
-#if defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
-#define Py_TRACE_REFS
-#endif
-
-/* Py_TRACE_REFS implies Py_REF_DEBUG. */
-#if defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
+/* Py_DEBUG implies Py_REF_DEBUG. */
+#if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
 #define Py_REF_DEBUG
 #endif
 

Lib/test/support/__init__.py

@@ -1653,7 +1653,7 @@ def python_is_optimized():
 
 _header = 'nP'
 _align = '0n'
-if hasattr(sys, "gettotalrefcount"):
+if hasattr(sys, "getobjects"):
     _header = '2P' + _header
     _align = '0P'
 _vheader = _header + 'n'

Misc/NEWS.d/next/Build/2019-04-09-18-19-43.bpo-36465.-w6vx6.rst

@@ -0,0 +1,5 @@
+Release build and debug build are now ABI compatible: the ``Py_DEBUG`` define
+no longer implies ``Py_TRACE_REFS`` define which introduces the only ABI
+incompatibility. A new ``./configure --with-trace-refs`` build option is now
+required to get ``Py_TRACE_REFS`` define which adds :func:`sys.getobjects`
+function and ``PYTHONDUMPREFS`` environment variable.

configure

@@ -814,6 +814,7 @@ with_suffix
 enable_shared
 enable_profiling
 with_pydebug
+with_trace_refs
 with_assertions
 enable_optimizations
 with_lto
@@ -1500,6 +1501,7 @@ Optional Packages:
                           compiler
   --with-suffix=.exe      set executable suffix
   --with-pydebug          build with Py_DEBUG defined
+  --with-trace-refs       enable tracing references for debugging purpose
   --with-assertions       build with C assertions enabled
   --with-lto              Enable Link Time Optimization in any build. Disabled
                           by default.
@@ -6333,8 +6335,30 @@ $as_echo "no" >&6; }
 fi
 
 
-# Check for --with-assertions. Py_DEBUG implies assertions, but also changes
-# the ABI. This allows enabling assertions without changing the ABI.
+# Check for --with-trace-refs
+# --with-trace-refs
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-trace-refs" >&5
+$as_echo_n "checking for --with-trace-refs... " >&6; }
+
+# Check whether --with-trace-refs was given.
+if test "${with_trace_refs+set}" = set; then :
+  withval=$with_trace_refs;
+else
+  with_trace_refs=no
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_trace_refs" >&5
+$as_echo "$with_trace_refs" >&6; }
+
+if test "$with_trace_refs" = "yes"
+then
+
+$as_echo "#define Py_TRACE_REFS 1" >>confdefs.h
+
+fi
+
+# Check for --with-assertions.
+# This allows enabling assertions without Py_DEBUG.
 assertions='false'
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-assertions" >&5
 $as_echo_n "checking for --with-assertions... " >&6; }

configure.ac

@@ -1228,8 +1228,21 @@ else AC_MSG_RESULT(no); Py_DEBUG='false'
 fi],
 [AC_MSG_RESULT(no)])
 
-# Check for --with-assertions. Py_DEBUG implies assertions, but also changes
-# the ABI. This allows enabling assertions without changing the ABI.
+# Check for --with-trace-refs
+# --with-trace-refs
+AC_MSG_CHECKING(for --with-trace-refs)
+AC_ARG_WITH(trace-refs,
+  AS_HELP_STRING([--with-trace-refs],[enable tracing references for debugging purpose]),,
+  with_trace_refs=no)
+AC_MSG_RESULT($with_trace_refs)
+
+if test "$with_trace_refs" = "yes"
+then
+  AC_DEFINE(Py_TRACE_REFS, 1, [Define if you want to enable tracing references for debugging purpose])
+fi
+
+# Check for --with-assertions.
+# This allows enabling assertions without Py_DEBUG.
 assertions='false'
 AC_MSG_CHECKING(for --with-assertions)
 AC_ARG_WITH(assertions,

pyconfig.h.in

@@ -1374,6 +1374,9 @@
    externally defined: 0 */
 #undef Py_HASH_ALGORITHM
 
+/* Define if you want to enable tracing references for debugging purpose */
+#undef Py_TRACE_REFS
+
 /* assume C89 semantics that RETSIGTYPE is always void */
 #undef RETSIGTYPE