gh-111997: C-API for signalling monitoring events (#116413)

Doc/c-api/index.rst

@@ -25,3 +25,4 @@ document the API functions in detail.
    memory.rst
    objimpl.rst
    apiabiversion.rst
+   monitoring.rst

Doc/c-api/monitoring.rst

@@ -0,0 +1,164 @@
+.. highlight:: c
+
+.. _monitoring:
+
+Monitorong C API
+================
+
+Added in version 3.13.
+
+An extension may need to interact with the event monitoring system. Subscribing
+to events and registering callbacks can be done via the Python API exposed in
+:mod:`sys.monitoring`.
+
+Generating Execution Events
+===========================
+
+The functions below make it possible for an extension to fire monitoring
+events as it emulates the execution of Python code. Each of these functions
+accepts a ``PyMonitoringState`` struct which contains concise information
+about the activation state of events, as well as the event arguments, which
+include a ``PyObject*`` representing the code object, the instruction offset
+and sometimes additional, event-specific arguments (see :mod:`sys.monitoring`
+for details about the signatures of the different event callbacks).
+The ``codelike`` argument should be an instance of :class:`types.CodeType`
+or of a type that emulates it.
+
+The VM disables tracing when firing an event, so there is no need for user
+code to do that.
+
+Monitoring functions should not be called with an exception set,
+except those listed below as working with the current exception.
+
+.. c:type:: PyMonitoringState
+
+  Representation of the state of an event type. It is allocated by the user
+  while its contents are maintained by the monitoring API functions described below.
+
+
+All of the functions below return 0 on success and -1 (with an exception set) on error.
+
+See :mod:`sys.monitoring` for descriptions of the events.
+
+.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``PY_START`` event.
+
+
+.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``PY_RESUME`` event.
+
+
+.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval)
+
+   Fire a ``PY_RETURN`` event.
+
+
+.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval)
+
+   Fire a ``PY_YIELD`` event.
+
+
+.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0)
+
+   Fire a ``CALL`` event.
+
+
+.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno)
+
+   Fire a ``LINE`` event.
+
+
+.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset)
+
+   Fire a ``JUMP`` event.
+
+
+.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset)
+
+   Fire a ``BRANCH`` event.
+
+
+.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval)
+
+   Fire a ``C_RETURN`` event.
+
+
+.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``PY_THROW`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``RAISE`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``C_RAISE`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``RERAISE`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``PY_UNWIND`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+
+   Fire a ``STOP_ITERATION`` event with the current exception (as returned by
+   :c:func:`PyErr_GetRaisedException`).
+
+
+Managing the Monitoring State
+-----------------------------
+
+Monitoring states can be managed with the help of monitoring scopes. A scope
+would typically correspond to a python function.
+
+.. :c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length)
+
+   Enter a monitored scope. ``event_types`` is an array of the event IDs for
+   events that may be fired from the scope. For example, the ID of a ``PY_START``
+   event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal
+   to the base-2 logarithm of ``sys.monitoring.events.PY_START``.
+   ``state_array`` is an array with a monitoring state entry for each event in
+   ``event_types``, it is allocated by the user but populated by
+   ``PyMonitoring_EnterScope`` with information about the activation state of
+   the event. The size of ``event_types`` (and hence also of ``state_array``)
+   is given in ``length``.
+
+   The ``version`` argument is a pointer to a value which should be allocated
+   by the user together with ``state_array`` and initialized to 0,
+   and then set only by ``PyMonitoring_EnterScope`` itelf. It allows this
+   function to determine whether event states have changed since the previous call,
+   and to return quickly if they have not.
+
+   The scopes referred to here are lexical scopes: a function, class or method.
+   ``PyMonitoring_EnterScope`` should be called whenever the lexical scope is
+   entered. Scopes can be reentered, reusing the same *state_array* and *version*,
+   in situations like when emulating a recursive Python function. When a code-like's
+   execution is paused, such as when emulating a generator, the scope needs to
+   be exited and re-entered.
+
+
+.. :c:function:: int PyMonitoring_ExitScope(void)
+
+   Exit the last scope that was entered with ``PyMonitoring_EnterScope``.

Doc/conf.py

@@ -131,6 +131,7 @@ nitpick_ignore = [
     ('c:func', 'vsnprintf'),
     # Standard C types
     ('c:type', 'FILE'),
+    ('c:type', 'int32_t'),
     ('c:type', 'int64_t'),
     ('c:type', 'intmax_t'),
     ('c:type', 'off_t'),

Doc/library/sys.monitoring.rst

@@ -255,7 +255,10 @@ No events are active by default.
 Per code object events
 ''''''''''''''''''''''
 
-Events can also be controlled on a per code object basis.
+Events can also be controlled on a per code object basis. The functions
+defined below which accept a :class:`types.CodeType` should be prepared
+to accept a look-alike object from functions which are not defined
+in Python (see :ref:`monitoring`).
 
 .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int
 

Include/Python.h

@@ -84,6 +84,7 @@
 #include "setobject.h"
 #include "methodobject.h"
 #include "moduleobject.h"
+#include "monitoring.h"
 #include "cpython/funcobject.h"
 #include "cpython/classobject.h"
 #include "fileobject.h"

Include/cpython/monitoring.h

@@ -0,0 +1,250 @@
+#ifndef Py_CPYTHON_MONITORING_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Local events.
+ * These require bytecode instrumentation */
+
+#define PY_MONITORING_EVENT_PY_START 0
+#define PY_MONITORING_EVENT_PY_RESUME 1
+#define PY_MONITORING_EVENT_PY_RETURN 2
+#define PY_MONITORING_EVENT_PY_YIELD 3
+#define PY_MONITORING_EVENT_CALL 4
+#define PY_MONITORING_EVENT_LINE 5
+#define PY_MONITORING_EVENT_INSTRUCTION 6
+#define PY_MONITORING_EVENT_JUMP 7
+#define PY_MONITORING_EVENT_BRANCH 8
+#define PY_MONITORING_EVENT_STOP_ITERATION 9
+
+#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \
+    ((ev) < _PY_MONITORING_LOCAL_EVENTS)
+
+/* Other events, mainly exceptions */
+
+#define PY_MONITORING_EVENT_RAISE 10
+#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11
+#define PY_MONITORING_EVENT_PY_UNWIND 12
+#define PY_MONITORING_EVENT_PY_THROW 13
+#define PY_MONITORING_EVENT_RERAISE 14
+
+
+/* Ancillary events */
+
+#define PY_MONITORING_EVENT_C_RETURN 15
+#define PY_MONITORING_EVENT_C_RAISE 16
+
+
+typedef struct _PyMonitoringState {
+    uint8_t active;
+    uint8_t opaque;
+} PyMonitoringState;
+
+
+PyAPI_FUNC(int)
+PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version,
+                         const uint8_t *event_types, Py_ssize_t length);
+
+PyAPI_FUNC(int)
+PyMonitoring_ExitScope(void);
+
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                                PyObject *retval);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            PyObject* callable, PyObject *arg0);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            int lineno);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            PyObject *target_offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *target_offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+
+#define _PYMONITORING_IF_ACTIVE(STATE, X)  \
+    if ((STATE)->active) { \
+        return (X); \
+    } \
+    else { \
+        return 0; \
+    }
+
+static inline int
+PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyStartEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyResumeEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval));
+}
+
+static inline int
+PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *retval)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval));
+}
+
+static inline int
+PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                           PyObject* callable, PyObject *arg0)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0));
+}
+
+static inline int
+PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                           int lineno)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireLineEvent(state, codelike, offset, lineno));
+}
+
+static inline int
+PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                           PyObject *target_offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset));
+}
+
+static inline int
+PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                             PyObject *target_offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset));
+}
+
+static inline int
+PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *retval)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval));
+}
+
+static inline int
+PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyThrowEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireRaiseEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireReraiseEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireCRaiseEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyUnwindEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireStopIterationEvent(state, codelike, offset));
+}
+
+#undef _PYMONITORING_IF_ACTIVE

Include/internal/pycore_instruments.h

@@ -13,38 +13,6 @@ extern "C" {
 
 #define PY_MONITORING_TOOL_IDS 8
 
-/* Local events.
- * These require bytecode instrumentation */
-
-#define PY_MONITORING_EVENT_PY_START 0
-#define PY_MONITORING_EVENT_PY_RESUME 1
-#define PY_MONITORING_EVENT_PY_RETURN 2
-#define PY_MONITORING_EVENT_PY_YIELD 3
-#define PY_MONITORING_EVENT_CALL 4
-#define PY_MONITORING_EVENT_LINE 5
-#define PY_MONITORING_EVENT_INSTRUCTION 6
-#define PY_MONITORING_EVENT_JUMP 7
-#define PY_MONITORING_EVENT_BRANCH 8
-#define PY_MONITORING_EVENT_STOP_ITERATION 9
-
-#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \
-    ((ev) < _PY_MONITORING_LOCAL_EVENTS)
-
-/* Other events, mainly exceptions */
-
-#define PY_MONITORING_EVENT_RAISE 10
-#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11
-#define PY_MONITORING_EVENT_PY_UNWIND 12
-#define PY_MONITORING_EVENT_PY_THROW 13
-#define PY_MONITORING_EVENT_RERAISE 14
-
-
-/* Ancillary events */
-
-#define PY_MONITORING_EVENT_C_RETURN 15
-#define PY_MONITORING_EVENT_C_RAISE 16
-
-
 typedef uint32_t _PyMonitoringEventSet;
 
 /* Tool IDs */

Include/monitoring.h

@@ -0,0 +1,18 @@
+#ifndef Py_MONITORING_H
+#define Py_MONITORING_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// There is currently no limited API for monitoring
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_MONITORING_H
+#  include "cpython/monitoring.h"
+#  undef Py_CPYTHON_MONITORING_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_MONITORING_H */

Lib/test/test_monitoring.py

@@ -3,16 +3,20 @@
 import collections
 import dis
 import functools
+import math
 import operator
 import sys
 import textwrap
 import types
 import unittest
 import asyncio
-from test import support
+
+import test.support
 from test.support import requires_specialization, script_helper
 from test.support.import_helper import import_module
 
+_testcapi = test.support.import_helper.import_module("_testcapi")
+
 PAIR = (0,1)
 
 def f1():
@@ -1887,5 +1891,180 @@ class TestMonitoringAtShutdown(unittest.TestCase):
         # gh-115832: An object destructor running during the final GC of
         # interpreter shutdown triggered an infinite loop in the
         # instrumentation code.
-        script = support.findfile("_test_monitoring_shutdown.py")
+        script = test.support.findfile("_test_monitoring_shutdown.py")
         script_helper.run_test_script(script)
+
+
+class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase):
+
+    class Scope:
+        def __init__(self, *args):
+            self.args = args
+
+        def __enter__(self):
+            _testcapi.monitoring_enter_scope(*self.args)
+
+        def __exit__(self, *args):
+            _testcapi.monitoring_exit_scope()
+
+    def setUp(self):
+        super(TestCApiEventGeneration, self).setUp()
+
+        capi = _testcapi
+
+        self.codelike = capi.CodeLike(2)
+
+        self.cases = [
+            # (Event, function, *args)
+            ( 1, E.PY_START, capi.fire_event_py_start),
+            ( 1, E.PY_RESUME, capi.fire_event_py_resume),
+            ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10),
+            ( 1, E.PY_RETURN, capi.fire_event_py_return, 20),
+            ( 2, E.CALL, capi.fire_event_call, callable, 40),
+            ( 1, E.JUMP, capi.fire_event_jump, 60),
+            ( 1, E.BRANCH, capi.fire_event_branch, 70),
+            ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)),
+            ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)),
+            ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)),
+            ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)),
+            ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, ValueError(7)),
+        ]
+
+
+    def check_event_count(self, event, func, args, expected):
+        class Counter:
+            def __init__(self):
+                self.count = 0
+            def __call__(self, *args):
+                self.count += 1
+
+        try:
+            counter = Counter()
+            sys.monitoring.register_callback(TEST_TOOL, event, counter)
+            if event == E.C_RETURN or event == E.C_RAISE:
+                sys.monitoring.set_events(TEST_TOOL, E.CALL)
+            else:
+                sys.monitoring.set_events(TEST_TOOL, event)
+            event_value = int(math.log2(event))
+            with self.Scope(self.codelike, event_value):
+                counter.count = 0
+                try:
+                    func(*args)
+                except ValueError as e:
+                    self.assertIsInstance(expected, ValueError)
+                    self.assertEqual(str(e), str(expected))
+                    return
+                else:
+                    self.assertEqual(counter.count, expected)
+
+            prev = sys.monitoring.register_callback(TEST_TOOL, event, None)
+            with self.Scope(self.codelike, event_value):
+                counter.count = 0
+                func(*args)
+                self.assertEqual(counter.count, 0)
+                self.assertEqual(prev, counter)
+        finally:
+            sys.monitoring.set_events(TEST_TOOL, 0)
+
+    def test_fire_event(self):
+        for expected, event, function, *args in self.cases:
+            offset = 0
+            self.codelike = _testcapi.CodeLike(1)
+            with self.subTest(function.__name__):
+                args_ = (self.codelike, offset) + tuple(args)
+                self.check_event_count(event, function, args_, expected)
+
+    def test_missing_exception(self):
+        for _, event, function, *args in self.cases:
+            if not (args and isinstance(args[-1], BaseException)):
+                continue
+            offset = 0
+            self.codelike = _testcapi.CodeLike(1)
+            with self.subTest(function.__name__):
+                args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,)
+                evt = int(math.log2(event))
+                expected = ValueError(f"Firing event {evt} with no exception set")
+                self.check_event_count(event, function, args_, expected)
+
+
+    CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE,
+                       E.EXCEPTION_HANDLED, E.PY_UNWIND }
+
+    def check_disable(self, event, func, args, expected):
+        try:
+            counter = CounterWithDisable()
+            sys.monitoring.register_callback(TEST_TOOL, event, counter)
+            if event == E.C_RETURN or event == E.C_RAISE:
+                sys.monitoring.set_events(TEST_TOOL, E.CALL)
+            else:
+                sys.monitoring.set_events(TEST_TOOL, event)
+            event_value = int(math.log2(event))
+            with self.Scope(self.codelike, event_value):
+                counter.count = 0
+                func(*args)
+                self.assertEqual(counter.count, expected)
+                counter.disable = True
+                if event in self.CANNOT_DISABLE:
+                    # use try-except rather then assertRaises to avoid
+                    # events from framework code
+                    try:
+                        counter.count = 0
+                        func(*args)
+                        self.assertEqual(counter.count, expected)
+                    except ValueError:
+                        pass
+                    else:
+                        self.Error("Expected a ValueError")
+                else:
+                    counter.count = 0
+                    func(*args)
+                    self.assertEqual(counter.count, expected)
+                    counter.count = 0
+                    func(*args)
+                    self.assertEqual(counter.count, expected - 1)
+        finally:
+            sys.monitoring.set_events(TEST_TOOL, 0)
+
+    def test_disable_event(self):
+        for expected, event, function, *args in self.cases:
+            offset = 0
+            self.codelike = _testcapi.CodeLike(2)
+            with self.subTest(function.__name__):
+                args_ = (self.codelike, 0) + tuple(args)
+                self.check_disable(event, function, args_, expected)
+
+    def test_enter_scope_two_events(self):
+        try:
+            yield_counter = CounterWithDisable()
+            unwind_counter = CounterWithDisable()
+            sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter)
+            sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter)
+            sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND)
+
+            yield_value = int(math.log2(E.PY_YIELD))
+            unwind_value = int(math.log2(E.PY_UNWIND))
+            cl = _testcapi.CodeLike(2)
+            common_args = (cl, 0)
+            with self.Scope(cl, yield_value, unwind_value):
+                yield_counter.count = 0
+                unwind_counter.count = 0
+
+                _testcapi.fire_event_py_unwind(*common_args, ValueError(42))
+                assert(yield_counter.count == 0)
+                assert(unwind_counter.count == 1)
+
+                _testcapi.fire_event_py_yield(*common_args, ValueError(42))
+                assert(yield_counter.count == 1)
+                assert(unwind_counter.count == 1)
+
+                yield_counter.disable = True
+                _testcapi.fire_event_py_yield(*common_args, ValueError(42))
+                assert(yield_counter.count == 2)
+                assert(unwind_counter.count == 1)
+
+                _testcapi.fire_event_py_yield(*common_args, ValueError(42))
+                assert(yield_counter.count == 2)
+                assert(unwind_counter.count == 1)
+
+        finally:
+            sys.monitoring.set_events(TEST_TOOL, 0)

Makefile.pre.in

@@ -1022,6 +1022,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/methodobject.h \
 		$(srcdir)/Include/modsupport.h \
 		$(srcdir)/Include/moduleobject.h \
+		$(srcdir)/Include/monitoring.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
 		$(srcdir)/Include/opcode.h \
@@ -1091,6 +1092,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/longobject.h \
 		$(srcdir)/Include/cpython/memoryobject.h \
 		$(srcdir)/Include/cpython/methodobject.h \
+		$(srcdir)/Include/cpython/monitoring.h \
 		$(srcdir)/Include/cpython/object.h \
 		$(srcdir)/Include/cpython/objimpl.h \
 		$(srcdir)/Include/cpython/odictobject.h \

Misc/NEWS.d/next/C API/2024-03-13-17-48-24.gh-issue-111997.8ZbHlA.rst

@@ -0,0 +1 @@
+Add a C-API for firing monitoring events.

Modules/Setup.stdlib.in

@@ -163,7 +163,7 @@
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c
-@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c
+@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c
 @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
 @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c

Modules/_testcapi/monitoring.c

@@ -0,0 +1,507 @@
+#include "parts.h"
+#include "util.h"
+
+#include "monitoring.h"
+
+#define Py_BUILD_CORE
+#include "internal/pycore_instruments.h"
+
+typedef struct {
+    PyObject_HEAD
+    PyMonitoringState *monitoring_states;
+    uint64_t version;
+    int num_events;
+    /* Other fields */
+} PyCodeLikeObject;
+
+
+static PyObject *
+CodeLike_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    int num_events;
+    if (!PyArg_ParseTuple(args, "i", &num_events)) {
+        return NULL;
+    }
+    PyMonitoringState *states = (PyMonitoringState *)PyMem_Calloc(
+            num_events, sizeof(PyMonitoringState));
+    if (states == NULL) {
+        return NULL;
+    }
+    PyCodeLikeObject *self = (PyCodeLikeObject *) type->tp_alloc(type, 0);
+    if (self != NULL) {
+        self->version = 0;
+        self->monitoring_states = states;
+        self->num_events = num_events;
+    }
+    else {
+        PyMem_Free(states);
+    }
+    return (PyObject *) self;
+}
+
+static void
+CodeLike_dealloc(PyCodeLikeObject *self)
+{
+    if (self->monitoring_states) {
+        PyMem_Free(self->monitoring_states);
+    }
+    Py_TYPE(self)->tp_free((PyObject *) self);
+}
+
+static PyObject *
+CodeLike_str(PyCodeLikeObject *self)
+{
+    PyObject *res = NULL;
+    PyObject *sep = NULL;
+    PyObject *parts = NULL;
+    if (self->monitoring_states) {
+        parts = PyList_New(0);
+        if (parts == NULL) {
+            goto end;
+        }
+
+        PyObject *heading = PyUnicode_FromString("PyCodeLikeObject");
+        if (heading == NULL) {
+            goto end;
+        }
+        int err = PyList_Append(parts, heading);
+        Py_DECREF(heading);
+        if (err < 0) {
+            goto end;
+        }
+
+        for (int i = 0; i < self->num_events; i++) {
+            PyObject *part = PyUnicode_FromFormat(" %d", self->monitoring_states[i].active);
+            if (part == NULL) {
+                goto end;
+            }
+            int err = PyList_Append(parts, part);
+            Py_XDECREF(part);
+            if (err < 0) {
+                goto end;
+            }
+        }
+        sep = PyUnicode_FromString(": ");
+        if (sep == NULL) {
+            goto end;
+        }
+        res = PyUnicode_Join(sep, parts);
+    }
+end:
+    Py_XDECREF(sep);
+    Py_XDECREF(parts);
+    return res;
+}
+
+static PyTypeObject PyCodeLike_Type = {
+    .ob_base = PyVarObject_HEAD_INIT(NULL, 0)
+    .tp_name = "monitoring.CodeLike",
+    .tp_doc = PyDoc_STR("CodeLike objects"),
+    .tp_basicsize = sizeof(PyCodeLikeObject),
+    .tp_itemsize = 0,
+    .tp_flags = Py_TPFLAGS_DEFAULT,
+    .tp_new = CodeLike_new,
+    .tp_dealloc = (destructor) CodeLike_dealloc,
+    .tp_str = (reprfunc) CodeLike_str,
+};
+
+#define RAISE_UNLESS_CODELIKE(v)  if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \
+        PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \
+        return NULL; \
+    }
+
+/*******************************************************************/
+
+static PyMonitoringState *
+setup_fire(PyObject *codelike, int offset, PyObject *exc)
+{
+    RAISE_UNLESS_CODELIKE(codelike);
+    PyCodeLikeObject *cl = ((PyCodeLikeObject *)codelike);
+    assert(offset >= 0 && offset < cl->num_events);
+    PyMonitoringState *state = &cl->monitoring_states[offset];
+
+    if (exc != NULL) {
+        PyErr_SetRaisedException(Py_NewRef(exc));
+    }
+    return state;
+}
+
+static int
+teardown_fire(int res, PyMonitoringState *state, PyObject *exception)
+{
+    if (res == -1) {
+        return -1;
+    }
+    if (exception) {
+        assert(PyErr_Occurred());
+        assert(((PyObject*)Py_TYPE(exception)) == PyErr_Occurred());
+    }
+
+    else {
+        assert(!PyErr_Occurred());
+    }
+    PyErr_Clear();
+    return state->active;
+}
+
+static PyObject *
+fire_event_py_start(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FirePyStartEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_py_resume(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FirePyResumeEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_py_return(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *retval;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_c_return(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *retval;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireCReturnEvent(state, codelike, offset, retval);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_py_yield(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *retval;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_call(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *callable, *arg0;
+    if (!PyArg_ParseTuple(args, "OiOO", &codelike, &offset, &callable, &arg0)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_line(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset, lineno;
+    if (!PyArg_ParseTuple(args, "Oii", &codelike, &offset, &lineno)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireLineEvent(state, codelike, offset, lineno);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_jump(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *target_offset;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_branch(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *target_offset;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) {
+        return NULL;
+    }
+    PyObject *exception = NULL;
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_py_throw(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FirePyThrowEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_raise(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireRaiseEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_c_raise(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireCRaiseEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_reraise(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireReraiseEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_exception_handled(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireExceptionHandledEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_py_unwind(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FirePyUnwindEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+static PyObject *
+fire_event_stop_iteration(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int offset;
+    PyObject *exception;
+    if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) {
+        return NULL;
+    }
+    NULLABLE(exception);
+    PyMonitoringState *state = setup_fire(codelike, offset, exception);
+    if (state == NULL) {
+        return NULL;
+    }
+    int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset);
+    RETURN_INT(teardown_fire(res, state, exception));
+}
+
+/*******************************************************************/
+
+static PyObject *
+enter_scope(PyObject *self, PyObject *args)
+{
+    PyObject *codelike;
+    int event1, event2=0;
+    Py_ssize_t num_events = PyTuple_Size(args) - 1;
+    if (num_events == 1) {
+        if (!PyArg_ParseTuple(args, "Oi", &codelike, &event1)) {
+            return NULL;
+        }
+    }
+    else {
+        assert(num_events == 2);
+        if (!PyArg_ParseTuple(args, "Oii", &codelike, &event1, &event2)) {
+            return NULL;
+        }
+    }
+    RAISE_UNLESS_CODELIKE(codelike);
+    PyCodeLikeObject *cl = (PyCodeLikeObject *) codelike;
+
+    uint8_t events[] = { event1, event2 };
+
+    PyMonitoring_EnterScope(cl->monitoring_states,
+                            &cl->version,
+                            events,
+                            num_events);
+
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+exit_scope(PyObject *self, PyObject *args)
+{
+    PyMonitoring_ExitScope();
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef TestMethods[] = {
+    {"fire_event_py_start", fire_event_py_start, METH_VARARGS},
+    {"fire_event_py_resume", fire_event_py_resume, METH_VARARGS},
+    {"fire_event_py_return", fire_event_py_return, METH_VARARGS},
+    {"fire_event_c_return", fire_event_c_return, METH_VARARGS},
+    {"fire_event_py_yield", fire_event_py_yield, METH_VARARGS},
+    {"fire_event_call", fire_event_call, METH_VARARGS},
+    {"fire_event_line", fire_event_line, METH_VARARGS},
+    {"fire_event_jump", fire_event_jump, METH_VARARGS},
+    {"fire_event_branch", fire_event_branch, METH_VARARGS},
+    {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS},
+    {"fire_event_raise", fire_event_raise, METH_VARARGS},
+    {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS},
+    {"fire_event_reraise", fire_event_reraise, METH_VARARGS},
+    {"fire_event_exception_handled", fire_event_exception_handled, METH_VARARGS},
+    {"fire_event_py_unwind", fire_event_py_unwind, METH_VARARGS},
+    {"fire_event_stop_iteration", fire_event_stop_iteration, METH_VARARGS},
+    {"monitoring_enter_scope", enter_scope, METH_VARARGS},
+    {"monitoring_exit_scope", exit_scope, METH_VARARGS},
+    {NULL},
+};
+
+int
+_PyTestCapi_Init_Monitoring(PyObject *m)
+{
+    if (PyType_Ready(&PyCodeLike_Type) < 0) {
+        return -1;
+    }
+    if (PyModule_AddObjectRef(m, "CodeLike", (PyObject *) &PyCodeLike_Type) < 0) {
+        Py_DECREF(m);
+        return -1;
+    }
+    if (PyModule_AddFunctions(m, TestMethods) < 0) {
+        return -1;
+    }
+    return 0;
+}

Modules/_testcapi/parts.h

@@ -58,6 +58,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module);
 int _PyTestCapi_Init_GC(PyObject *module);
 int _PyTestCapi_Init_Hash(PyObject *module);
 int _PyTestCapi_Init_Time(PyObject *module);
+int _PyTestCapi_Init_Monitoring(PyObject *module);
 int _PyTestCapi_Init_Object(PyObject *module);
 
 #endif // Py_TESTCAPI_PARTS_H

Modules/_testcapimodule.c

@@ -4135,6 +4135,9 @@ PyInit__testcapi(void)
     if (_PyTestCapi_Init_Time(m) < 0) {
         return NULL;
     }
+    if (_PyTestCapi_Init_Monitoring(m) < 0) {
+        return NULL;
+    }
     if (_PyTestCapi_Init_Object(m) < 0) {
         return NULL;
     }

PCbuild/_testcapi.vcxproj

@@ -125,6 +125,7 @@
     <ClCompile Include="..\Modules\_testcapi\immortal.c" />
     <ClCompile Include="..\Modules\_testcapi\gc.c" />
     <ClCompile Include="..\Modules\_testcapi\run.c" />
+    <ClCompile Include="..\Modules\_testcapi\monitoring.c" />
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc" />

PCbuild/_testcapi.vcxproj.filters

@@ -108,6 +108,9 @@
     <ClCompile Include="..\Modules\_testcapi\run.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Modules\_testcapi\monitoring.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="..\PC\python_nt.rc">

PCbuild/_testinternalcapi.vcxproj.filters

@@ -27,4 +27,4 @@
       <Filter>Resource Files</Filter>
     </ResourceCompile>
   </ItemGroup>
-</Project>
+</Project>

Python/instrumentation.c

@@ -2424,3 +2424,304 @@ error:
     Py_DECREF(mod);
     return NULL;
 }
+
+
+static int
+capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                          PyObject **args, Py_ssize_t nargs, int event)
+{
+    PyThreadState *tstate = _PyThreadState_GET();
+    PyInterpreterState *interp = tstate->interp;
+
+    uint8_t tools = state->active;
+    assert(args[1] == NULL);
+    args[1] = codelike;
+    if (offset < 0) {
+        PyErr_SetString(PyExc_ValueError, "offset must be non-negative");
+        return -1;
+    }
+    PyObject *offset_obj = PyLong_FromLong(offset);
+    if (offset_obj == NULL) {
+        return -1;
+    }
+    assert(args[2] == NULL);
+    args[2] = offset_obj;
+    Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET;
+    PyObject **callargs = &args[1];
+    int err = 0;
+
+    while (tools) {
+        int tool = most_significant_bit(tools);
+        assert(tool >= 0 && tool < 8);
+        assert(tools & (1 << tool));
+        tools ^= (1 << tool);
+        int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event);
+        if (res == 0) {
+            /* Nothing to do */
+        }
+        else if (res < 0) {
+            /* error */
+            err = -1;
+            break;
+        }
+        else {
+            /* DISABLE */
+            if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) {
+                PyErr_Format(PyExc_ValueError,
+                             "Cannot disable %s events. Callback removed.",
+                             event_names[event]);
+                /* Clear tool to prevent infinite loop */
+                Py_CLEAR(interp->monitoring_callables[tool][event]);
+                err = -1;
+                break;
+            }
+            else {
+                state->active &= ~(1 << tool);
+            }
+        }
+    }
+    return err;
+}
+
+int
+PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version,
+                         const uint8_t *event_types, Py_ssize_t length)
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    if (global_version(interp) == *version) {
+        return 0;
+    }
+
+    _Py_GlobalMonitors *m = &interp->monitors;
+    for (Py_ssize_t i = 0; i < length; i++) {
+        int event = event_types[i];
+        state_array[i].active = m->tools[event];
+    }
+    *version = global_version(interp);
+    return 0;
+}
+
+int
+PyMonitoring_ExitScope(void)
+{
+    return 0;
+}
+
+int
+_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    assert(state->active);
+    PyObject *args[3] = { NULL, NULL, NULL };
+    return capi_call_instrumentation(state, codelike, offset, args, 2,
+                                     PY_MONITORING_EVENT_PY_START);
+}
+
+int
+_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    assert(state->active);
+    PyObject *args[3] = { NULL, NULL, NULL };
+    return capi_call_instrumentation(state, codelike, offset, args, 2,
+                                     PY_MONITORING_EVENT_PY_RESUME);
+}
+
+
+
+int
+_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                                PyObject* retval)
+{
+    assert(state->active);
+    PyObject *args[4] = { NULL, NULL, NULL, retval };
+    return capi_call_instrumentation(state, codelike, offset, args, 3,
+                                     PY_MONITORING_EVENT_PY_RETURN);
+}
+
+int
+_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject* retval)
+{
+    assert(state->active);
+    PyObject *args[4] = { NULL, NULL, NULL, retval };
+    return capi_call_instrumentation(state, codelike, offset, args, 3,
+                                     PY_MONITORING_EVENT_PY_YIELD);
+}
+
+int
+_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            PyObject* callable, PyObject *arg0)
+{
+    assert(state->active);
+    PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 };
+    return capi_call_instrumentation(state, codelike, offset, args, 4,
+                                     PY_MONITORING_EVENT_CALL);
+}
+
+int
+_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            int lineno)
+{
+    assert(state->active);
+    PyObject *lno = PyLong_FromLong(lineno);
+    if (lno == NULL) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, lno };
+    int res= capi_call_instrumentation(state, codelike, offset, args, 3,
+                                       PY_MONITORING_EVENT_LINE);
+    Py_DECREF(lno);
+    return res;
+}
+
+int
+_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            PyObject *target_offset)
+{
+    assert(state->active);
+    PyObject *args[4] = { NULL, NULL, NULL, target_offset };
+    return capi_call_instrumentation(state, codelike, offset, args, 3,
+                                     PY_MONITORING_EVENT_JUMP);
+}
+
+int
+_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *target_offset)
+{
+    assert(state->active);
+    PyObject *args[4] = { NULL, NULL, NULL, target_offset };
+    return capi_call_instrumentation(state, codelike, offset, args, 3,
+                                     PY_MONITORING_EVENT_BRANCH);
+}
+
+int
+_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval)
+{
+    assert(state->active);
+    PyObject *args[4] = { NULL, NULL, NULL, retval };
+    return capi_call_instrumentation(state, codelike, offset, args, 3,
+                                     PY_MONITORING_EVENT_C_RETURN);
+}
+
+static inline int
+exception_event_setup(PyObject **exc, int event) {
+    *exc = PyErr_GetRaisedException();
+    if (*exc == NULL) {
+        PyErr_Format(PyExc_ValueError,
+                     "Firing event %d with no exception set",
+                     event);
+        return -1;
+    }
+    return 0;
+}
+
+
+static inline int
+exception_event_teardown(int err, PyObject *exc) {
+    if (err == 0) {
+        PyErr_SetRaisedException(exc);
+    }
+    else {
+        assert(PyErr_Occurred());
+        Py_DECREF(exc);
+    }
+    return err;
+}
+
+int
+_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_PY_THROW;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_RAISE;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_C_RAISE;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_RERAISE;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_PY_UNWIND;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}
+
+int
+_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    int event = PY_MONITORING_EVENT_STOP_ITERATION;
+    assert(state->active);
+    PyObject *exc;
+    if (exception_event_setup(&exc, event) < 0) {
+        return -1;
+    }
+    PyObject *args[4] = { NULL, NULL, NULL, exc };
+    int err = capi_call_instrumentation(state, codelike, offset, args, 3, event);
+    return exception_event_teardown(err, exc);
+}

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -397,6 +397,7 @@ Modules/xxmodule.c	-	Str_Type	-
 Modules/xxmodule.c	-	Xxo_Type	-
 Modules/xxsubtype.c	-	spamdict_type	-
 Modules/xxsubtype.c	-	spamlist_type	-
+Modules/_testcapi/monitoring.c	-	PyCodeLike_Type	-
 
 ##-----------------------
 ## non-static types - initialized once