bpo-36842: Implement PEP 578 (GH-12613)

Adds sys.audit, sys.addaudithook, io.open_code, and associated C APIs.

Doc/c-api/code.rst

@@ -40,6 +40,7 @@ bound into a function.
    :c:func:`PyCode_New` directly can bind you to a precise Python
    version since the definition of the bytecode changes often.
 
+   .. audit-event:: code.__new__ "code filename name argcount kwonlyargcount nlocals stacksize flags"
 
 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 

Doc/c-api/file.rst

@@ -60,6 +60,32 @@ the :mod:`io` APIs instead.
    raised if the end of the file is reached immediately.
 
 
+.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler)
+
+   Overrides the normal behavior of :func:`io.open_code` to pass its parameter
+   through the provided handler.
+
+   The handler is a function of type :c:type:`PyObject *(\*)(PyObject *path,
+   void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`.
+
+   The *userData* pointer is passed into the hook function. Since hook
+   functions may be called from different runtimes, this pointer should not
+   refer directly to Python state.
+
+   As this hook is intentionally used during import, avoid importing new modules
+   during its execution unless they are known to be frozen or available in
+   ``sys.modules``.
+
+   Once a hook has been set, it cannot be removed or replaced, and later calls to
+   :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns
+   -1 and sets an exception if the interpreter has been initialized.
+
+   This function is safe to call before :c:func:`Py_Initialize`.
+
+   .. versionadded:: 3.8
+
+
+
 .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags)
 
    .. index:: single: Py_PRINT_RAW

Doc/c-api/sys.rst

@@ -289,6 +289,56 @@ accessible to C code.  They all work with the current interpreter thread's
    .. versionadded:: 3.2
 
 
+.. c:function:: int PySys_Audit(const char *event, const char *format, ...)
+
+   .. index:: single: audit events
+
+   Raises an auditing event with any active hooks. Returns zero for success
+   and non-zero with an exception set on failure.
+
+   If any hooks have been added, *format* and other arguments will be used
+   to construct a tuple to pass. Apart from ``N``, the same format characters
+   as used in :c:func:`Py_BuildValue` are available. If the built value is not
+   a tuple, it will be added into a single-element tuple. (The ``N`` format
+   option consumes a reference, but since there is no way to know whether
+   arguments to this function will be consumed, using it may cause reference
+   leaks.)
+
+   :func:`sys.audit` performs the same function from Python code.
+
+   .. versionadded:: 3.8
+
+
+.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
+
+   .. index:: single: audit events
+
+   Adds to the collection of active auditing hooks. Returns zero for success
+   and non-zero on failure. If the runtime has been initialized, also sets an
+   error on failure. Hooks added through this API are called for all
+   interpreters created by the runtime.
+
+   This function is safe to call before :c:func:`Py_Initialize`. When called
+   after runtime initialization, existing audit hooks are notified and may
+   silently abort the operation by raising an error subclassed from
+   :class:`Exception` (other errors will not be silenced).
+
+   The hook function is of type :c:type:`int (*)(const char *event, PyObject
+   *args, void *userData)`, where *args* is guaranteed to be a
+   :c:type:`PyTupleObject`. The hook function is always called with the GIL
+   held by the Python interpreter that raised the event.
+
+   The *userData* pointer is passed into the hook function. Since hook
+   functions may be called from different runtimes, this pointer should not
+   refer directly to Python state.
+
+   See :pep:`578` for a detailed decription of auditing. Functions in the
+   runtime and standard library that raise events include the details in each
+   function's documentation.
+
+   .. versionadded:: 3.8
+
+
 .. _processcontrol:
 
 Process Control

Doc/howto/instrumentation.rst

@@ -332,6 +332,15 @@ Available static markers
    .. versionadded:: 3.7
 
 
+.. c:function:: audit(str event, void *tuple)
+
+   Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called.
+   ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject`
+   pointer to a tuple object.
+
+   .. versionadded:: 3.8
+
+
 SystemTap Tapsets
 -----------------
 

Doc/library/array.rst

@@ -83,6 +83,7 @@ The module defines the following type:
    to add initial items to the array.  Otherwise, the iterable initializer is
    passed to the :meth:`extend` method.
 
+   .. audit-event:: array.__new__ "typecode initializer"
 
 .. data:: typecodes
 

Doc/library/ctypes.rst

@@ -1509,6 +1509,17 @@ object is available:
    :c:type:`int`, which is of course not always the truth, so you have to assign
    the correct :attr:`restype` attribute to use these functions.
 
+.. audit-event:: ctypes.dlopen name
+
+   Loading a library through any of these objects raises an
+   :ref:`auditing event <auditing>` ``ctypes.dlopen`` with string argument
+   ``name``, the name used to load the library.
+
+.. audit-event:: ctypes.dlsym "library name"
+
+   Accessing a function on a loaded library raises an auditing event
+   ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name``
+   (the symbol's name as a string or integer).
 
 .. _ctypes-foreign-functions:
 
@@ -2032,6 +2043,12 @@ Data types
       This method returns a ctypes type instance using the memory specified by
       *address* which must be an integer.
 
+      .. audit-event:: ctypes.cdata address
+
+         This method, and others that indirectly call this method, raises an
+         :func:`auditing event <sys.audit>` ``ctypes.cdata`` with argument
+         ``address``.
+
    .. method:: from_param(obj)
 
       This method adapts *obj* to a ctypes type.  It is called with the actual

Doc/library/functions.rst

@@ -275,6 +275,12 @@ are always available.  They are listed here in alphabetical order.
    If you want to parse Python code into its AST representation, see
    :func:`ast.parse`.
 
+   .. audit-event:: compile "source filename"
+
+      Raises an :func:`auditing event <sys.audit>` ``compile`` with arguments
+      ``source`` and ``filename``. This event may also be raised by implicit
+      compilation.
+
    .. note::
 
       When compiling a string with multi-line code in ``'single'`` or
@@ -473,6 +479,11 @@ are always available.  They are listed here in alphabetical order.
    See :func:`ast.literal_eval` for a function that can safely evaluate strings
    with expressions containing only literals.
 
+   .. audit-event:: exec code_object
+
+      Raises an :func:`auditing event <sys.audit>` ``exec`` with the code object as
+      the argument. Code compilation events may also be raised.
+
 .. index:: builtin: exec
 
 .. function:: exec(object[, globals[, locals]])
@@ -502,6 +513,11 @@ are always available.  They are listed here in alphabetical order.
    builtins are available to the executed code by inserting your own
    ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`.
 
+   .. audit-event:: exec code_object
+
+      Raises an :func:`auditing event <sys.audit>` ``exec`` with the code object as
+      the argument. Code compilation events may also be raised.
+
    .. note::
 
       The built-in functions :func:`globals` and :func:`locals` return the current
@@ -747,6 +763,16 @@ are always available.  They are listed here in alphabetical order.
    If the :mod:`readline` module was loaded, then :func:`input` will use it
    to provide elaborate line editing and history features.
 
+   .. audit-event:: builtins.input prompt
+
+      Raises an :func:`auditing event <sys.audit>` ``builtins.input`` with
+      argument ``prompt`` before reading input
+
+   .. audit-event:: builtins.input/result result
+
+      Raises an auditing event ``builtins.input/result`` with the result after
+      successfully reading input.
+
 
 .. class:: int([x])
            int(x, base=10)
@@ -1176,6 +1202,11 @@ are always available.  They are listed here in alphabetical order.
    (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
    and :mod:`shutil`.
 
+   .. audit-event:: open "file mode flags"
+
+   The ``mode`` and ``flags`` arguments may have been modified or inferred from
+   the original call.
+
    .. versionchanged::
       3.3
 

Doc/library/io.rst

@@ -120,6 +120,27 @@ High-level Module Interface
 
    This is an alias for the builtin :func:`open` function.
 
+   .. audit-event:: open "path mode flags"
+
+      This function raises an :func:`auditing event <sys.audit>` ``open`` with
+      arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags``
+      arguments may have been modified or inferred from the original call.
+
+
+.. function:: open_code(path)
+
+   Opens the provided file with mode ``'rb'``. This function should be used
+   when the intent is to treat the contents as executable code.
+
+   ``path`` should be an absolute path.
+
+   The behavior of this function may be overridden by an earlier call to the
+   :c:func:`PyFile_SetOpenCodeHook`, however, it should always be considered
+   interchangeable with ``open(path, 'rb')``. Overriding the behavior is
+   intended for additional validation or preprocessing of the file.
+
+   .. versionadded:: 3.8
+
 
 .. exception:: BlockingIOError
 

Doc/library/mmap.rst

@@ -67,6 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
    will be relative to the offset from the beginning of the file. *offset*
    defaults to 0.  *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`.
 
+   .. audit-event:: mmap.__new__ "fileno length access offset"
 
 .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset])
    :noindex:
@@ -155,6 +156,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
 
           mm.close()
 
+   .. audit-event:: mmap.__new__ "fileno length access offset"
 
    Memory-mapped file objects support the following methods:
 

Doc/library/os.rst

@@ -651,7 +651,7 @@ process and user.
 File Object Creation
 --------------------
 
-This function creates new :term:`file objects <file object>`.  (See also
+These functions create new :term:`file objects <file object>`.  (See also
 :func:`~os.open` for opening file descriptors.)
 
 
@@ -829,11 +829,14 @@ as internal buffering of data.
    most *length* bytes in size.  As of Python 3.3, this is equivalent to
    ``os.truncate(fd, length)``.
 
+   .. audit-event:: os.truncate "fd length"
+
    .. availability:: Unix, Windows.
 
    .. versionchanged:: 3.5
       Added support for Windows
 
+
 .. function:: get_blocking(fd)
 
    Get the blocking mode of the file descriptor: ``False`` if the
@@ -845,6 +848,7 @@ as internal buffering of data.
 
    .. versionadded:: 3.5
 
+
 .. function:: isatty(fd)
 
    Return ``True`` if the file descriptor *fd* is open and connected to a
@@ -912,6 +916,8 @@ as internal buffering of data.
    This function can support :ref:`paths relative to directory descriptors
    <dir_fd>` with the *dir_fd* parameter.
 
+   .. audit-event:: open "path mode flags"
+
    .. versionchanged:: 3.4
       The new file descriptor is now non-inheritable.
 
@@ -2756,6 +2762,8 @@ features:
 
    This function can support :ref:`specifying a file descriptor <path_fd>`.
 
+   .. audit-event:: os.truncate "path length"
+
    .. availability:: Unix, Windows.
 
    .. versionadded:: 3.3
@@ -3715,6 +3723,8 @@ written in Python, such as a mail server's external command delivery program.
    to using this function.  See the :ref:`subprocess-replacements` section in
    the :mod:`subprocess` documentation for some helpful recipes.
 
+   .. audit-event:: os.system command
+
    .. availability:: Unix, Windows.
 
 

Doc/library/pickle.rst

@@ -427,6 +427,7 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and
       how they can be loaded, potentially reducing security risks. Refer to
       :ref:`pickle-restrict` for details.
 
+      .. audit-event:: pickle.find_class "module name"
 
 .. _pickle-picklable:
 

Doc/library/socket.rst

@@ -526,6 +526,8 @@ The following functions all create :ref:`socket objects <socket-objects>`.
 
    The newly created socket is :ref:`non-inheritable <fd_inheritance>`.
 
+   .. audit-event:: socket.__new__ "self family type protocol"
+
    .. versionchanged:: 3.3
       The AF_CAN family was added.
       The AF_RDS family was added.
@@ -718,6 +720,8 @@ The :mod:`socket` module also offers various network-related services:
    :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect`
    method.
 
+   .. audit-event:: socket.getaddrinfo "host port family type protocol"
+
    The following example fetches address information for a hypothetical TCP
    connection to ``example.org`` on port 80 (results may differ on your
    system if IPv6 isn't enabled)::
@@ -753,6 +757,8 @@ The :mod:`socket` module also offers various network-related services:
    interface. :func:`gethostbyname` does not support IPv6 name resolution, and
    :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support.
 
+   .. audit-event:: socket.gethostbyname hostname
+
 
 .. function:: gethostbyname_ex(hostname)
 
@@ -765,12 +771,16 @@ The :mod:`socket` module also offers various network-related services:
    resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual
    stack support.
 
+   .. audit-event:: socket.gethostbyname hostname
+
 
 .. function:: gethostname()
 
    Return a string containing the hostname of the machine where  the Python
    interpreter is currently executing.
 
+   .. audit-event:: socket.gethostname
+
    Note: :func:`gethostname` doesn't always return the fully qualified domain
    name; use :func:`getfqdn` for that.
 
@@ -785,6 +795,8 @@ The :mod:`socket` module also offers various network-related services:
    domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports
    both IPv4 and IPv6.
 
+   .. audit-event:: socket.gethostbyaddr ip_address
+
 
 .. function:: getnameinfo(sockaddr, flags)
 
@@ -798,6 +810,8 @@ The :mod:`socket` module also offers various network-related services:
 
    For more information about *flags* you can consult :manpage:`getnameinfo(3)`.
 
+   .. audit-event:: socket.getnameinfo sockaddr
+
 .. function:: getprotobyname(protocolname)
 
    Translate an Internet protocol name (for example, ``'icmp'``) to a constant
@@ -813,6 +827,8 @@ The :mod:`socket` module also offers various network-related services:
    service.  The optional protocol name, if given, should be ``'tcp'`` or
    ``'udp'``, otherwise any protocol will match.
 
+   .. audit-event:: socket.getservbyname "servicename protocolname"
+
 
 .. function:: getservbyport(port[, protocolname])
 
@@ -820,6 +836,8 @@ The :mod:`socket` module also offers various network-related services:
    service.  The optional protocol name, if given, should be ``'tcp'`` or
    ``'udp'``, otherwise any protocol will match.
 
+   .. audit-event:: socket.getservbyport "port protocolname"
+
 
 .. function:: ntohl(x)
 
@@ -1003,6 +1021,8 @@ The :mod:`socket` module also offers various network-related services:
    Set the machine's hostname to *name*.  This will raise an
    :exc:`OSError` if you don't have enough rights.
 
+   .. audit-event:: socket.sethostname name
+
    .. availability:: Unix.
 
    .. versionadded:: 3.3
@@ -1078,6 +1098,7 @@ to sockets.
    Bind the socket to *address*.  The socket must not already be bound. (The format
    of *address* depends on the address family --- see above.)
 
+   .. audit-event:: socket.bind "self address"
 
 .. method:: socket.close()
 
@@ -1115,6 +1136,8 @@ to sockets.
    :exc:`InterruptedError` exception if the connection is interrupted by a
    signal (or the exception raised by the signal handler).
 
+   .. audit-event:: socket.connect "self address"
+
    .. versionchanged:: 3.5
       The method now waits until the connection completes instead of raising an
       :exc:`InterruptedError` exception if the connection is interrupted by a
@@ -1131,6 +1154,7 @@ to sockets.
    :c:data:`errno` variable.  This is useful to support, for example, asynchronous
    connects.
 
+   .. audit-event:: socket.connect "self address"
 
 .. method:: socket.detach()
 
@@ -1472,6 +1496,8 @@ to sockets.
    bytes sent. (The format of *address* depends on the address family --- see
    above.)
 
+   .. audit-event:: socket.sendto "self address"
+
    .. versionchanged:: 3.5
       If the system call is interrupted and the signal handler does not raise
       an exception, the method now retries the system call instead of raising
@@ -1511,6 +1537,8 @@ to sockets.
 
    .. availability:: most Unix platforms, possibly others.
 
+   .. audit-event:: socket.sendmsg "self address"
+
    .. versionadded:: 3.3
 
    .. versionchanged:: 3.5

Doc/library/sys.rst

@@ -19,6 +19,30 @@ always available.
    .. versionadded:: 3.2
 
 
+.. function:: addaudithook(hook)
+
+   Adds the callable *hook* to the collection of active auditing hooks for the
+   current interpreter.
+
+   When an auditing event is raised through the :func:`sys.audit` function, each
+   hook will be called in the order it was added with the event name and the
+   tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are
+   called first, followed by hooks added in the current interpreter.
+
+   Calling this function will trigger an event for all existing hooks, and if
+   any raise an exception derived from :class:`Exception`, the add will be
+   silently ignored. As a result, callers cannot assume that their hook has been
+   added unless they control all existing hooks.
+
+   .. versionadded:: 3.8
+
+   .. impl-detail::
+
+      When tracing is enabled, Python hooks are only traced if the callable has
+      a ``__cantrace__`` member that is set to a true value. Otherwise, trace
+      functions will not see the hook.
+
+
 .. data:: argv
 
    The list of command line arguments passed to a Python script. ``argv[0]`` is the
@@ -37,6 +61,30 @@ always available.
       ``[os.fsencode(arg) for arg in sys.argv]``.
 
 
+.. _auditing:
+
+.. function:: audit(event, *args)
+
+   .. index:: single: auditing
+
+   Raises an auditing event with any active hooks. The event name is a string
+   identifying the event and its associated schema, which is the number and
+   types of arguments. The schema for a given event is considered public and
+   stable API and should not be modified between releases.
+
+   This function will raise the first exception raised by any hook. In general,
+   these errors should not be handled and should terminate the process as
+   quickly as possible.
+
+   Hooks are added using the :func:`sys.addaudithook` or
+   :c:func:`PySys_AddAuditHook` functions.
+
+   The native equivalent of this function is :c:func:`PySys_Audit`. Using the
+   native function is preferred when possible.
+
+   .. versionadded:: 3.8
+
+
 .. data:: base_exec_prefix
 
    Set during Python startup, before ``site.py`` is run, to the same value as
@@ -114,6 +162,8 @@ always available.
 
    This function should be used for internal and specialized purposes only.
 
+   .. audit-event:: sys._current_frames
+
 
 .. function:: breakpointhook()
 
@@ -617,6 +667,8 @@ always available.
    that is deeper than the call stack, :exc:`ValueError` is raised.  The default
    for *depth* is zero, returning the frame at the top of the call stack.
 
+   .. audit-event:: sys._getframe
+
    .. impl-detail::
 
       This function should be used for internal and specialized purposes only.
@@ -1146,6 +1198,8 @@ always available.
    ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends
    on the event type.
 
+   .. audit-event:: sys.setprofile
+
    The events have the following meaning:
 
    ``'call'``
@@ -1266,6 +1320,8 @@ always available.
 
    For more information on code and frame objects, refer to :ref:`types`.
 
+   .. audit-event:: sys.settrace
+
    .. impl-detail::
 
       The :func:`settrace` function is intended only for implementing debuggers,
@@ -1286,6 +1342,13 @@ always available.
    first time. The *finalizer* will be called when an asynchronous generator
    is about to be garbage collected.
 
+   .. audit-event:: sys.set_asyncgen_hooks_firstiter
+
+   .. audit-event:: sys.set_asyncgen_hooks_finalizer
+
+   Two auditing events are raised because the underlying API consists of two
+   calls, each of which must raise its own event.
+
    .. versionadded:: 3.6
       See :pep:`525` for more details, and for a reference example of a
       *finalizer* method see the implementation of

Doc/library/urllib.request.rst

@@ -95,6 +95,12 @@ The :mod:`urllib.request` module defines the following functions:
    parameter to ``urllib.urlopen``, can be obtained by using
    :class:`ProxyHandler` objects.
 
+   .. audit-event:: urllib.request "fullurl data headers method"
+
+      The default opener raises an :func:`auditing event <sys.audit>`
+      ``urllib.request`` with arguments ``fullurl``, ``data``, ``headers``,
+      ``method`` taken from the request object.
+
    .. versionchanged:: 3.2
       *cafile* and *capath* were added.
 
@@ -118,6 +124,7 @@ The :mod:`urllib.request` module defines the following functions:
        :func:`ssl.create_default_context` select the system's trusted CA
        certificates for you.
 
+
 .. function:: install_opener(opener)
 
    Install an :class:`OpenerDirector` instance as the default global opener.

Doc/tools/extensions/pyspecific.py

@@ -151,6 +151,45 @@ class Availability(Directive):
         return [pnode]
 
 
+# Support for documenting audit event
+
+class AuditEvent(Directive):
+
+    has_content = True
+    required_arguments = 1
+    optional_arguments = 1
+    final_argument_whitespace = True
+
+    _label = [
+        "Raises an :ref:`auditing event <auditing>` {name} with no arguments.",
+        "Raises an :ref:`auditing event <auditing>` {name} with argument {args}.",
+        "Raises an :ref:`auditing event <auditing>` {name} with arguments {args}.",
+    ]
+
+    def run(self):
+        if len(self.arguments) >= 2 and self.arguments[1]:
+            args = [
+                "``{}``".format(a.strip())
+                for a in self.arguments[1].strip("'\"").split()
+                if a.strip()
+            ]
+        else:
+            args = []
+
+        label = translators['sphinx'].gettext(self._label[min(2, len(args))])
+        text = label.format(name="``{}``".format(self.arguments[0]),
+                            args=", ".join(args))
+
+        pnode = nodes.paragraph(text, classes=["audit-hook"])
+        if self.content:
+            self.state.nested_parse(self.content, self.content_offset, pnode)
+        else:
+            n, m = self.state.inline_text(text, self.lineno)
+            pnode.extend(n + m)
+
+        return [pnode]
+
+
 # Support for documenting decorators
 
 class PyDecoratorMixin(object):
@@ -424,6 +463,7 @@ def setup(app):
     app.add_role('source', source_role)
     app.add_directive('impl-detail', ImplementationDetail)
     app.add_directive('availability', Availability)
+    app.add_directive('audit-event', AuditEvent)
     app.add_directive('deprecated-removed', DeprecatedRemoved)
     app.add_builder(PydocTopicsBuilder)
     app.add_builder(suspicious.CheckSuspiciousMarkupBuilder)

Include/cpython/fileobject.h

@@ -0,0 +1,32 @@
+#ifndef Py_CPYTHON_FILEOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000
+PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors;
+#endif
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
+PyAPI_DATA(int) Py_UTF8Mode;
+#endif
+
+/* The std printer acts as a preliminary sys.stderr until the new io
+   infrastructure is in place. */
+PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int);
+PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
+
+typedef PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *);
+
+PyAPI_FUNC(PyObject *) PyFile_OpenCode(const char *utf8path);
+PyAPI_FUNC(PyObject *) PyFile_OpenCodeObject(PyObject *path);
+PyAPI_FUNC(int) PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData);
+
+#ifdef __cplusplus
+}
+#endif

Include/cpython/sysmodule.h

@@ -0,0 +1,21 @@
+#ifndef Py_CPYTHON_SYSMODULE_H
+#  error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key);
+PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *);
+
+PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *);
+
+typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *);
+
+PyAPI_FUNC(int) PySys_Audit(const char*, const char *, ...);
+PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*);
+
+#ifdef __cplusplus
+}
+#endif

Include/fileobject.h

@@ -15,32 +15,13 @@ PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int);
 PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int);
 PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *);
 PyAPI_FUNC(int) PyObject_AsFileDescriptor(PyObject *);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
-#endif
 
 /* The default encoding used by the platform file system APIs
    If non-NULL, this is different than the default encoding for strings
 */
 PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding;
-#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000
-PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors;
-#endif
 PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding;
 
-#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
-PyAPI_DATA(int) Py_UTF8Mode;
-#endif
-
-/* Internal API
-
-   The std printer acts as a preliminary sys.stderr until the new io
-   infrastructure is in place. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int);
-PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
-#endif /* Py_LIMITED_API */
-
 /* A routine to check if a file descriptor can be select()-ed. */
 #ifdef _MSC_VER
     /* On Windows, any socket fd can be select()-ed, no matter how high */
@@ -49,6 +30,12 @@ PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
     #define _PyIsSelectable_fd(FD) ((unsigned int)(FD) < (unsigned int)FD_SETSIZE)
 #endif
 
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_FILEOBJECT_H
+#  include  "cpython/fileobject.h"
+#  undef Py_CPYTHON_FILEOBJECT_H
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pystate.h

@@ -9,8 +9,10 @@ extern "C" {
 #endif
 
 #include "cpython/coreconfig.h"
+#include "fileobject.h"
 #include "pystate.h"
 #include "pythread.h"
+#include "sysmodule.h"
 
 #include "pycore_gil.h"   /* _gil_runtime_state  */
 #include "pycore_pathconfig.h"
@@ -131,6 +133,8 @@ struct _is {
     uint64_t tstate_next_unique_id;
 
     struct _warnings_runtime_state warnings;
+
+    PyObject *audit_hooks;
 };
 
 PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T);
@@ -154,6 +158,13 @@ struct _xidregitem {
     struct _xidregitem *next;
 };
 
+/* runtime audit hook state */
+
+typedef struct _Py_AuditHookEntry {
+    struct _Py_AuditHookEntry *next;
+    Py_AuditHookFunction hookCFunction;
+    void *userData;
+} _Py_AuditHookEntry;
 
 /* GIL state */
 
@@ -224,6 +235,11 @@ typedef struct pyruntimestate {
     struct _gilstate_runtime_state gilstate;
 
     _PyPreConfig preconfig;
+
+    Py_OpenCodeHookFunction open_code_hook;
+    void *open_code_userdata;
+    _Py_AuditHookEntry *audit_hook_head;
+
     // XXX Consolidate globals found via the check-c-globals script.
 } _PyRuntimeState;
 

Include/pydtrace.d

@@ -12,6 +12,7 @@ provider python {
     probe gc__done(long);
     probe import__find__load__start(const char *);
     probe import__find__load__done(const char *, int);
+    probe audit(const char *, void *);
 };
 
 #pragma D attributes Evolving/Evolving/Common provider python provider

Include/pydtrace.h

@@ -36,6 +36,7 @@ static inline void PyDTrace_INSTANCE_DELETE_START(int arg0) {}
 static inline void PyDTrace_INSTANCE_DELETE_DONE(int arg0) {}
 static inline void PyDTrace_IMPORT_FIND_LOAD_START(const char *arg0) {}
 static inline void PyDTrace_IMPORT_FIND_LOAD_DONE(const char *arg0, int arg1) {}
+static inline void PyDTrace_AUDIT(const char *arg0, void *arg1) {}
 
 static inline int PyDTrace_LINE_ENABLED(void) { return 0; }
 static inline int PyDTrace_FUNCTION_ENTRY_ENABLED(void) { return 0; }
@@ -48,6 +49,7 @@ static inline int PyDTrace_INSTANCE_DELETE_START_ENABLED(void) { return 0; }
 static inline int PyDTrace_INSTANCE_DELETE_DONE_ENABLED(void) { return 0; }
 static inline int PyDTrace_IMPORT_FIND_LOAD_START_ENABLED(void) { return 0; }
 static inline int PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED(void) { return 0; }
+static inline int PyDTrace_AUDIT_ENABLED(void) { return 0; }
 
 #endif /* !WITH_DTRACE */
 

Include/sysmodule.h

@@ -9,10 +9,6 @@ extern "C" {
 
 PyAPI_FUNC(PyObject *) PySys_GetObject(const char *);
 PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key);
-PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *);
-#endif
 
 PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **);
 PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int);
@@ -34,7 +30,9 @@ PyAPI_FUNC(void) PySys_AddXOption(const wchar_t *);
 PyAPI_FUNC(PyObject *) PySys_GetXOptions(void);
 
 #ifndef Py_LIMITED_API
-PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *);
+#  define Py_CPYTHON_SYSMODULE_H
+#  include  "cpython/sysmodule.h"
+#  undef Py_CPYTHON_SYSMODULE_H
 #endif
 
 #ifdef __cplusplus

Lib/_pyio.py

@@ -254,6 +254,29 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
         result.close()
         raise
 
+# Define a default pure-Python implementation for open_code()
+# that does not allow hooks. Warn on first use. Defined for tests.
+def _open_code_with_warning(path):
+    """Opens the provided file with mode ``'rb'``. This function
+    should be used when the intent is to treat the contents as
+    executable code.
+
+    ``path`` should be an absolute path.
+
+    When supported by the runtime, this function can be hooked
+    in order to allow embedders more control over code files.
+    This functionality is not supported on the current runtime.
+    """
+    import warnings
+    warnings.warn("_pyio.open_code() may not be using hooks",
+                  RuntimeWarning, 2)
+    return open(path, "rb")
+
+try:
+    open_code = io.open_code
+except AttributeError:
+    open_code = _open_code_with_warning
+
 
 class DocDescriptor:
     """Helper for builtins.open.__doc__

Lib/importlib/_bootstrap_external.py

@@ -963,8 +963,12 @@ class FileLoader:
 
     def get_data(self, path):
         """Return the data from path as raw bytes."""
-        with _io.FileIO(path, 'r') as file:
+        if isinstance(self, (SourceLoader, ExtensionFileLoader)):
-            return file.read()
+            with _io.open_code(str(path)) as file:
+                return file.read()
+        else:
+            with _io.FileIO(path, 'r') as file:
+                return file.read()
 
     # ResourceReader ABC API.
 

Lib/io.py

@@ -41,8 +41,8 @@ __author__ = ("Guido van Rossum <guido@python.org>, "
               "Amaury Forgeot d'Arc <amauryfa@gmail.com>, "
               "Benjamin Peterson <benjamin@python.org>")
 
-__all__ = ["BlockingIOError", "open", "IOBase", "RawIOBase", "FileIO",
+__all__ = ["BlockingIOError", "open", "open_code", "IOBase", "RawIOBase",
-           "BytesIO", "StringIO", "BufferedIOBase",
+           "FileIO", "BytesIO", "StringIO", "BufferedIOBase",
            "BufferedReader", "BufferedWriter", "BufferedRWPair",
            "BufferedRandom", "TextIOBase", "TextIOWrapper",
            "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"]
@@ -52,7 +52,7 @@ import _io
 import abc
 
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
-                 open, FileIO, BytesIO, StringIO, BufferedReader,
+                 open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
                  IncrementalNewlineDecoder, TextIOWrapper)
 

Lib/pickle.py

@@ -1436,6 +1436,7 @@ class _Unpickler:
 
     def find_class(self, module, name):
         # Subclasses may override this.
+        sys.audit('pickle.find_class', module, name)
         if self.proto < 3 and self.fix_imports:
             if (module, name) in _compat_pickle.NAME_MAPPING:
                 module, name = _compat_pickle.NAME_MAPPING[(module, name)]

Lib/test/libregrtest/setup.py

@@ -107,6 +107,12 @@ def setup_tests(ns):
 
     support.use_resources = ns.use_resources
 
+    if hasattr(sys, 'addaudithook'):
+        # Add an auditing hook for all tests to ensure PySys_Audit is tested
+        def _test_audit_hook(name, args):
+            pass
+        sys.addaudithook(_test_audit_hook)
+
 
 def replace_stdout():
     """Set stdout encoder error handler to backslashreplace (as stderr error

Lib/test/test_audit.py

@@ -0,0 +1,260 @@
+"""Tests for sys.audit and sys.addaudithook
+"""
+
+import os
+import subprocess
+import sys
+import unittest
+from test import support
+
+if not hasattr(sys, "addaudithook") or not hasattr(sys, "audit"):
+    raise unittest.SkipTest("test only relevant when sys.audit is available")
+
+
+class TestHook:
+    """Used in standard hook tests to collect any logged events.
+
+    Should be used in a with block to ensure that it has no impact
+    after the test completes. Audit hooks cannot be removed, so the
+    best we can do for the test run is disable it by calling close().
+    """
+
+    def __init__(self, raise_on_events=None, exc_type=RuntimeError):
+        self.raise_on_events = raise_on_events or ()
+        self.exc_type = exc_type
+        self.seen = []
+        self.closed = False
+
+    def __enter__(self, *a):
+        sys.addaudithook(self)
+        return self
+
+    def __exit__(self, *a):
+        self.close()
+
+    def close(self):
+        self.closed = True
+
+    @property
+    def seen_events(self):
+        return [i[0] for i in self.seen]
+
+    def __call__(self, event, args):
+        if self.closed:
+            return
+        self.seen.append((event, args))
+        if event in self.raise_on_events:
+            raise self.exc_type("saw event " + event)
+
+
+class TestFinalizeHook:
+    """Used in the test_finalize_hooks function to ensure that hooks
+    are correctly cleaned up, that they are notified about the cleanup,
+    and are unable to prevent it.
+    """
+
+    def __init__(self):
+        print("Created", id(self), file=sys.stderr, flush=True)
+
+    def __call__(self, event, args):
+        # Avoid recursion when we call id() below
+        if event == "builtins.id":
+            return
+
+        print(event, id(self), file=sys.stderr, flush=True)
+
+        if event == "cpython._PySys_ClearAuditHooks":
+            raise RuntimeError("Should be ignored")
+        elif event == "cpython.PyInterpreterState_Clear":
+            raise RuntimeError("Should be ignored")
+
+
+def run_finalize_test():
+    """Called by test_finalize_hooks in a subprocess."""
+    sys.addaudithook(TestFinalizeHook())
+
+
+class AuditTest(unittest.TestCase):
+    def test_basic(self):
+        with TestHook() as hook:
+            sys.audit("test_event", 1, 2, 3)
+            self.assertEqual(hook.seen[0][0], "test_event")
+            self.assertEqual(hook.seen[0][1], (1, 2, 3))
+
+    def test_block_add_hook(self):
+        # Raising an exception should prevent a new hook from being added,
+        # but will not propagate out.
+        with TestHook(raise_on_events="sys.addaudithook") as hook1:
+            with TestHook() as hook2:
+                sys.audit("test_event")
+                self.assertIn("test_event", hook1.seen_events)
+                self.assertNotIn("test_event", hook2.seen_events)
+
+    def test_block_add_hook_baseexception(self):
+        # Raising BaseException will propagate out when adding a hook
+        with self.assertRaises(BaseException):
+            with TestHook(
+                raise_on_events="sys.addaudithook", exc_type=BaseException
+            ) as hook1:
+                # Adding this next hook should raise BaseException
+                with TestHook() as hook2:
+                    pass
+
+    def test_finalize_hooks(self):
+        events = []
+        with subprocess.Popen(
+            [
+                sys.executable,
+                "-c",
+                "import test.test_audit; test.test_audit.run_finalize_test()",
+            ],
+            encoding="utf-8",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        ) as p:
+            p.wait()
+            for line in p.stderr:
+                events.append(line.strip().partition(" "))
+        firstId = events[0][2]
+        self.assertSequenceEqual(
+            [
+                ("Created", " ", firstId),
+                ("cpython._PySys_ClearAuditHooks", " ", firstId),
+            ],
+            events,
+        )
+
+    def test_pickle(self):
+        pickle = support.import_module("pickle")
+
+        class PicklePrint:
+            def __reduce_ex__(self, p):
+                return str, ("Pwned!",)
+
+        payload_1 = pickle.dumps(PicklePrint())
+        payload_2 = pickle.dumps(("a", "b", "c", 1, 2, 3))
+
+        # Before we add the hook, ensure our malicious pickle loads
+        self.assertEqual("Pwned!", pickle.loads(payload_1))
+
+        with TestHook(raise_on_events="pickle.find_class") as hook:
+            with self.assertRaises(RuntimeError):
+                # With the hook enabled, loading globals is not allowed
+                pickle.loads(payload_1)
+            # pickles with no globals are okay
+            pickle.loads(payload_2)
+
+    def test_monkeypatch(self):
+        class A:
+            pass
+
+        class B:
+            pass
+
+        class C(A):
+            pass
+
+        a = A()
+
+        with TestHook() as hook:
+            # Catch name changes
+            C.__name__ = "X"
+            # Catch type changes
+            C.__bases__ = (B,)
+            # Ensure bypassing __setattr__ is still caught
+            type.__dict__["__bases__"].__set__(C, (B,))
+            # Catch attribute replacement
+            C.__init__ = B.__init__
+            # Catch attribute addition
+            C.new_attr = 123
+            # Catch class changes
+            a.__class__ = B
+
+        actual = [(a[0], a[1]) for e, a in hook.seen if e == "object.__setattr__"]
+        self.assertSequenceEqual(
+            [(C, "__name__"), (C, "__bases__"), (C, "__bases__"), (a, "__class__")],
+            actual,
+        )
+
+    def test_open(self):
+        # SSLContext.load_dh_params uses _Py_fopen_obj rather than normal open()
+        try:
+            import ssl
+
+            load_dh_params = ssl.create_default_context().load_dh_params
+        except ImportError:
+            load_dh_params = None
+
+        # Try a range of "open" functions.
+        # All of them should fail
+        with TestHook(raise_on_events={"open"}) as hook:
+            for fn, *args in [
+                (open, support.TESTFN, "r"),
+                (open, sys.executable, "rb"),
+                (open, 3, "wb"),
+                (open, support.TESTFN, "w", -1, None, None, None, False, lambda *a: 1),
+                (load_dh_params, support.TESTFN),
+            ]:
+                if not fn:
+                    continue
+                self.assertRaises(RuntimeError, fn, *args)
+
+        actual_mode = [(a[0], a[1]) for e, a in hook.seen if e == "open" and a[1]]
+        actual_flag = [(a[0], a[2]) for e, a in hook.seen if e == "open" and not a[1]]
+        self.assertSequenceEqual(
+            [
+                i
+                for i in [
+                    (support.TESTFN, "r"),
+                    (sys.executable, "r"),
+                    (3, "w"),
+                    (support.TESTFN, "w"),
+                    (support.TESTFN, "rb") if load_dh_params else None,
+                ]
+                if i is not None
+            ],
+            actual_mode,
+        )
+        self.assertSequenceEqual([], actual_flag)
+
+    def test_cantrace(self):
+        traced = []
+
+        def trace(frame, event, *args):
+            if frame.f_code == TestHook.__call__.__code__:
+                traced.append(event)
+
+        old = sys.settrace(trace)
+        try:
+            with TestHook() as hook:
+                # No traced call
+                eval("1")
+
+                # No traced call
+                hook.__cantrace__ = False
+                eval("2")
+
+                # One traced call
+                hook.__cantrace__ = True
+                eval("3")
+
+                # Two traced calls (writing to private member, eval)
+                hook.__cantrace__ = 1
+                eval("4")
+
+                # One traced call (writing to private member)
+                hook.__cantrace__ = 0
+        finally:
+            sys.settrace(old)
+
+        self.assertSequenceEqual(["call"] * 4, traced)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 2 and sys.argv[1] == "spython_test":
+        # Doesn't matter what we add - it will be blocked
+        sys.addaudithook(None)
+
+        sys.exit(0)
+
+    unittest.main()

Lib/test/test_embed.py

@@ -927,5 +927,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                           api=API_PYTHON)
 
 
+class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
+    def test_open_code_hook(self):
+        self.run_embedded_interpreter("test_open_code_hook")
+
+    def test_audit(self):
+        self.run_embedded_interpreter("test_audit")
+
+    def test_audit_subinterpreter(self):
+        self.run_embedded_interpreter("test_audit_subinterpreter")
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_fileio.py

@@ -565,6 +565,7 @@ class OtherFileTests:
         self.assertRaises(MyException, MyFileIO, fd)
         os.close(fd)  # should not raise OSError(EBADF)
 
+
 class COtherFileTests(OtherFileTests, unittest.TestCase):
     FileIO = _io.FileIO
     modulename = '_io'
@@ -576,10 +577,32 @@ class COtherFileTests(OtherFileTests, unittest.TestCase):
         self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MAX + 1)
         self.assertRaises(TypeError, self.FileIO, _testcapi.INT_MIN - 1)
 
+    def test_open_code(self):
+        # Check that the default behaviour of open_code matches
+        # open("rb")
+        with self.FileIO(__file__, "rb") as f:
+            expected = f.read()
+        with _io.open_code(__file__) as f:
+            actual = f.read()
+        self.assertEqual(expected, actual)
+
+
 class PyOtherFileTests(OtherFileTests, unittest.TestCase):
     FileIO = _pyio.FileIO
     modulename = '_pyio'
 
+    def test_open_code(self):
+        # Check that the default behaviour of open_code matches
+        # open("rb")
+        with self.FileIO(__file__, "rb") as f:
+            expected = f.read()
+        with check_warnings(quiet=True) as w:
+            # Always test _open_code_with_warning
+            with _pyio._open_code_with_warning(__file__) as f:
+                actual = f.read()
+            self.assertEqual(expected, actual)
+            self.assertNotEqual(w.warnings, [])
+
 
 def test_main():
     # Historically, these tests have been sloppy about removing TESTFN.

Lib/test/test_io.py

@@ -3861,7 +3861,7 @@ class MiscIOTest(unittest.TestCase):
         for name in self.io.__all__:
             obj = getattr(self.io, name, None)
             self.assertIsNotNone(obj, name)
-            if name == "open":
+            if name in ("open", "open_code"):
                 continue
             elif "error" in name.lower() or name == "UnsupportedOperation":
                 self.assertTrue(issubclass(obj, Exception), name)

Lib/urllib/request.py

@@ -521,6 +521,7 @@ class OpenerDirector:
             meth = getattr(processor, meth_name)
             req = meth(req)
 
+        sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
         response = self._open(req, data)
 
         # post-process response

Lib/zipimport.py

@@ -351,7 +351,7 @@ def _get_module_info(self, fullname):
 # data_size and file_offset are 0.
 def _read_directory(archive):
     try:
-        fp = _io.open(archive, 'rb')
+        fp = _io.open_code(archive)
     except OSError:
         raise ZipImportError(f"can't open Zip file: {archive!r}", path=archive)
 
@@ -533,7 +533,7 @@ def _get_data(archive, toc_entry):
     if data_size < 0:
         raise ZipImportError('negative data size')
 
-    with _io.open(archive, 'rb') as fp:
+    with _io.open_code(archive) as fp:
         # Check to make sure the local file header is correct
         try:
             fp.seek(file_offset)

Makefile.pre.in

@@ -1052,6 +1052,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/abstract.h \
 		$(srcdir)/Include/cpython/coreconfig.h \
 		$(srcdir)/Include/cpython/dictobject.h \
+		$(srcdir)/Include/cpython/fileobject.h \
 		$(srcdir)/Include/cpython/interpreteridobject.h \
 		$(srcdir)/Include/cpython/object.h \
 		$(srcdir)/Include/cpython/objimpl.h \
@@ -1059,6 +1060,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/pylifecycle.h \
 		$(srcdir)/Include/cpython/pymem.h \
 		$(srcdir)/Include/cpython/pystate.h \
+		$(srcdir)/Include/cpython/sysmodule.h \
 		$(srcdir)/Include/cpython/traceback.h \
 		$(srcdir)/Include/cpython/tupleobject.h \
 		$(srcdir)/Include/cpython/unicodeobject.h \

Misc/NEWS.d/next/Core and Builtins/2019-05-07-16-50-12.bpo-36842.NYww_N.rst

@@ -0,0 +1 @@
+Implement PEP 578, adding sys.audit, io.open_code and related APIs.

Modules/_ctypes/_ctypes.c

@@ -2920,6 +2920,10 @@ PyCData_AtAddress(PyObject *type, void *buf)
     CDataObject *pd;
     StgDictObject *dict;
 
+    if (PySys_Audit("ctypes.cdata", "n", (Py_ssize_t)buf) < 0) {
+        return NULL;
+    }
+
     assert(PyType_Check(type));
     dict = PyType_stgdict(type);
     if (!dict) {
@@ -3455,6 +3459,18 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
+#ifdef MS_WIN32
+    if (PySys_Audit("ctypes.dlsym",
+                    ((uintptr_t)name & ~0xFFFF) ? "Os" : "On",
+                    dll, name) < 0) {
+        return NULL;
+    }
+#else
+    if (PySys_Audit("ctypes.dlsym", "Os", dll, name) < 0) {
+        return NULL;
+    }
+#endif
+
     obj = PyObject_GetAttrString(dll, "_handle");
     if (!obj) {
         Py_DECREF(ftuple);

Modules/_ctypes/callproc.c

@@ -1277,6 +1277,10 @@ static PyObject *load_library(PyObject *self, PyObject *args)
     if (!name)
         return NULL;
 
+    if (PySys_Audit("ctypes.dlopen", "O", nameobj) < 0) {
+        return NULL;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     /* bpo-36085: Limit DLL search directories to avoid pre-loading
      * attacks and enable use of the AddDllDirectory function.
@@ -1382,6 +1386,9 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args)
         name_str = NULL;
         name2 = NULL;
     }
+    if (PySys_Audit("ctypes.dlopen", "s", name_str) < 0) {
+        return NULL;
+    }
     handle = ctypes_dlopen(name_str, mode);
     Py_XDECREF(name2);
     if (!handle) {

Modules/_io/_iomodule.c

@@ -503,6 +503,25 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode,
     Py_XDECREF(modeobj);
     return NULL;
 }
+
+/*[clinic input]
+_io.open_code
+
+    path : unicode
+
+Opens the provided file with the intent to import the contents.
+
+This may perform extra validation beyond open(), but is otherwise interchangeable
+with calling open(path, 'rb').
+
+[clinic start generated code]*/
+
+static PyObject *
+_io_open_code_impl(PyObject *module, PyObject *path)
+/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=f5c18e23f4b2ed9f]*/
+{
+    return PyFile_OpenCodeObject(path);
+}
 
 /*
  * Private helpers for the io module.
@@ -630,6 +649,7 @@ iomodule_free(PyObject *mod) {
 
 static PyMethodDef module_methods[] = {
     _IO_OPEN_METHODDEF
+    _IO_OPEN_CODE_METHODDEF
     {NULL, NULL}
 };
 

Modules/_io/clinic/_iomodule.c.h

@@ -281,4 +281,46 @@ skip_optional_pos:
 exit:
     return return_value;
 }
-/*[clinic end generated code: output=19fc9b69a5166f63 input=a9049054013a1b77]*/
+
+PyDoc_STRVAR(_io_open_code__doc__,
+"open_code($module, /, path)\n"
+"--\n"
+"\n"
+"Opens the provided file with the intent to import the contents.\n"
+"\n"
+"This may perform extra validation beyond open(), but is otherwise interchangeable\n"
+"with calling open(path, \'rb\').");
+
+#define _IO_OPEN_CODE_METHODDEF    \
+    {"open_code", (PyCFunction)(void(*)(void))_io_open_code, METH_FASTCALL|METH_KEYWORDS, _io_open_code__doc__},
+
+static PyObject *
+_io_open_code_impl(PyObject *module, PyObject *path);
+
+static PyObject *
+_io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"path", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "open_code", 0};
+    PyObject *argsbuf[1];
+    PyObject *path;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    if (!PyUnicode_Check(args[0])) {
+        _PyArg_BadArgument("open_code", 1, "str", args[0]);
+        goto exit;
+    }
+    if (PyUnicode_READY(args[0]) == -1) {
+        goto exit;
+    }
+    path = args[0];
+    return_value = _io_open_code_impl(module, path);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=d479285078750d68 input=a9049054013a1b77]*/

Modules/_io/fileio.c

@@ -358,6 +358,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
     flags |= O_CLOEXEC;
 #endif
 
+    if (PySys_Audit("open", "Osi", nameobj, mode, flags) < 0) {
+        goto error;
+    }
+
     if (fd >= 0) {
         self->fd = fd;
         self->closefd = closefd;

Modules/_pickle.c

@@ -6659,6 +6659,11 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,
     PyObject *global;
     PyObject *module;
 
+    if (PySys_Audit("pickle.find_class", "OO",
+                    module_name, global_name) < 0) {
+        return NULL;
+    }
+
     /* Try to map the old names used in Python 2.x to the new ones used in
        Python 3.x.  We do this only with old pickle protocols and when the
        user has not disabled the feature. */

Modules/_winapi.c

@@ -461,6 +461,12 @@ _winapi_CreateFile_impl(PyObject *module, LPCTSTR file_name,
 {
     HANDLE handle;
 
+    if (PySys_Audit("_winapi.CreateFile", "uIIII",
+                    file_name, desired_access, share_mode,
+                    creation_disposition, flags_and_attributes) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     handle = CreateFile(file_name, desired_access,
                         share_mode, security_attributes,
@@ -542,6 +548,10 @@ _winapi_CreateJunction_impl(PyObject *module, LPWSTR src_path,
     if (wcsncmp(src_path, L"\\??\\", prefix_len) == 0)
         return PyErr_SetFromWindowsErr(ERROR_INVALID_PARAMETER);
 
+    if (PySys_Audit("_winapi.CreateJunction", "uu", src_path, dst_path) < 0) {
+        return NULL;
+    }
+
     /* Adjust privileges to allow rewriting directory entry as a
        junction point. */
     if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &token))
@@ -670,6 +680,11 @@ _winapi_CreateNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD open_mode,
 {
     HANDLE handle;
 
+    if (PySys_Audit("_winapi.CreateNamedPipe", "uII",
+                    name, open_mode, pipe_mode) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     handle = CreateNamedPipe(name, open_mode, pipe_mode,
                              max_instances, out_buffer_size,
@@ -704,6 +719,10 @@ _winapi_CreatePipe_impl(PyObject *module, PyObject *pipe_attrs, DWORD size)
     HANDLE write_pipe;
     BOOL result;
 
+    if (PySys_Audit("_winapi.CreatePipe", NULL) < 0) {
+        return NULL;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     result = CreatePipe(&read_pipe, &write_pipe, NULL, size);
     Py_END_ALLOW_THREADS
@@ -1055,6 +1074,11 @@ _winapi_CreateProcess_impl(PyObject *module,
     wchar_t *command_line_copy = NULL;
     AttributeList attribute_list = {0};
 
+    if (PySys_Audit("_winapi.CreateProcess", "uuu", application_name,
+                    command_line, current_directory) < 0) {
+        return NULL;
+    }
+
     ZeroMemory(&si, sizeof(si));
     si.StartupInfo.cb = sizeof(si);
 
@@ -1270,8 +1294,10 @@ _winapi_GetModuleFileName_impl(PyObject *module, HMODULE module_handle)
     BOOL result;
     WCHAR filename[MAX_PATH];
 
+    Py_BEGIN_ALLOW_THREADS
     result = GetModuleFileNameW(module_handle, filename, MAX_PATH);
     filename[MAX_PATH-1] = '\0';
+    Py_END_ALLOW_THREADS
 
     if (! result)
         return PyErr_SetFromWindowsErr(GetLastError());
@@ -1402,9 +1428,16 @@ _winapi_OpenProcess_impl(PyObject *module, DWORD desired_access,
 {
     HANDLE handle;
 
+    if (PySys_Audit("_winapi.OpenProcess", "II",
+                    process_id, desired_access) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
     handle = OpenProcess(desired_access, inherit_handle, process_id);
+    Py_END_ALLOW_THREADS
     if (handle == NULL) {
-        PyErr_SetFromWindowsErr(0);
+        PyErr_SetFromWindowsErr(GetLastError());
         handle = INVALID_HANDLE_VALUE;
     }
 
@@ -1539,6 +1572,7 @@ _winapi_SetNamedPipeHandleState_impl(PyObject *module, HANDLE named_pipe,
     PyObject *oArgs[3] = {mode, max_collection_count, collect_data_timeout};
     DWORD dwArgs[3], *pArgs[3] = {NULL, NULL, NULL};
     int i;
+    BOOL b;
 
     for (i = 0 ; i < 3 ; i++) {
         if (oArgs[i] != Py_None) {
@@ -1549,7 +1583,11 @@ _winapi_SetNamedPipeHandleState_impl(PyObject *module, HANDLE named_pipe,
         }
     }
 
-    if (!SetNamedPipeHandleState(named_pipe, pArgs[0], pArgs[1], pArgs[2]))
+    Py_BEGIN_ALLOW_THREADS
+    b = SetNamedPipeHandleState(named_pipe, pArgs[0], pArgs[1], pArgs[2]);
+    Py_END_ALLOW_THREADS
+
+    if (!b)
         return PyErr_SetFromWindowsErr(0);
 
     Py_RETURN_NONE;
@@ -1573,6 +1611,11 @@ _winapi_TerminateProcess_impl(PyObject *module, HANDLE handle,
 {
     BOOL result;
 
+    if (PySys_Audit("_winapi.TerminateProcess", "nI",
+                    (Py_ssize_t)handle, exit_code) < 0) {
+        return NULL;
+    }
+
     result = TerminateProcess(handle, exit_code);
 
     if (! result)

Modules/arraymodule.c

@@ -2635,6 +2635,11 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (!PyArg_ParseTuple(args, "C|O:array", &c, &initial))
         return NULL;
 
+    if (PySys_Audit("array.__new__", "CO",
+                    c, initial ? initial : Py_None) < 0) {
+        return NULL;
+    }
+
     if (initial && c != 'u') {
         if (PyUnicode_Check(initial)) {
             PyErr_Format(PyExc_TypeError, "cannot use a str to initialize "

Modules/mmapmodule.c

@@ -1110,6 +1110,11 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
                             "mmap invalid access parameter.");
     }
 
+    if (PySys_Audit("mmap.__new__", "ini" _Py_PARSE_OFF_T,
+                    fileno, map_size, access, offset) < 0) {
+        return NULL;
+    }
+
 #ifdef __APPLE__
     /* Issue #11277: fsync(2) is not enough on OS X - a special, OS X specific
        fcntl(2) is necessary to force DISKSYNC and get around mmap(2) bug */
@@ -1240,6 +1245,11 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
         return NULL;
     }
 
+    if (PySys_Audit("mmap.__new__", "iniL",
+                    fileno, map_size, access, offset) < 0) {
+        return NULL;
+    }
+
     switch((access_mode)access) {
     case ACCESS_READ:
         flProtect = PAGE_READONLY;

Modules/posixmodule.c

@@ -4264,6 +4264,11 @@ os_system_impl(PyObject *module, const Py_UNICODE *command)
 /*[clinic end generated code: output=5b7c3599c068ca42 input=303f5ce97df606b0]*/
 {
     long result;
+
+    if (PySys_Audit("system", "(u)", command) < 0) {
+        return -1;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     _Py_BEGIN_SUPPRESS_IPH
     result = _wsystem(command);
@@ -4286,6 +4291,11 @@ os_system_impl(PyObject *module, PyObject *command)
 {
     long result;
     const char *bytes = PyBytes_AsString(command);
+
+    if (PySys_Audit("system", "(O)", command) < 0) {
+        return -1;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     result = system(bytes);
     Py_END_ALLOW_THREADS
@@ -8279,6 +8289,10 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd)
     flags |= O_CLOEXEC;
 #endif
 
+    if (PySys_Audit("open", "OOi", path->object, Py_None, flags) < 0) {
+        return -1;
+    }
+
     _Py_BEGIN_SUPPRESS_IPH
     do {
         Py_BEGIN_ALLOW_THREADS
@@ -9598,6 +9612,10 @@ os_ftruncate_impl(PyObject *module, int fd, Py_off_t length)
     int result;
     int async_err = 0;
 
+    if (PySys_Audit("os.truncate", "in", fd, length) < 0) {
+        return NULL;
+    }
+
     do {
         Py_BEGIN_ALLOW_THREADS
         _Py_BEGIN_SUPPRESS_IPH
@@ -9641,6 +9659,10 @@ os_truncate_impl(PyObject *module, path_t *path, Py_off_t length)
     if (path->fd != -1)
         return os_ftruncate_impl(module, path->fd, length);
 
+    if (PySys_Audit("os.truncate", "On", path->object, length) < 0) {
+        return NULL;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     _Py_BEGIN_SUPPRESS_IPH
 #ifdef MS_WINDOWS

Modules/socketmodule.c

@@ -3053,6 +3053,11 @@ sock_bind(PySocketSockObject *s, PyObject *addro)
     if (!getsockaddrarg(s, addro, SAS2SA(&addrbuf), &addrlen, "bind")) {
         return NULL;
     }
+
+    if (PySys_Audit("socket.bind", "OO", s, addro) < 0) {
+        return NULL;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     res = bind(s->sock_fd, SAS2SA(&addrbuf), addrlen);
     Py_END_ALLOW_THREADS
@@ -3219,6 +3224,10 @@ sock_connect(PySocketSockObject *s, PyObject *addro)
         return NULL;
     }
 
+    if (PySys_Audit("socket.connect", "OO", s, addro) < 0) {
+        return NULL;
+    }
+
     res = internal_connect(s, SAS2SA(&addrbuf), addrlen, 1);
     if (res < 0)
         return NULL;
@@ -3246,6 +3255,10 @@ sock_connect_ex(PySocketSockObject *s, PyObject *addro)
         return NULL;
     }
 
+    if (PySys_Audit("socket.connect", "OO", s, addro) < 0) {
+        return NULL;
+    }
+
     res = internal_connect(s, SAS2SA(&addrbuf), addrlen, 0);
     if (res < 0)
         return NULL;
@@ -4248,6 +4261,10 @@ sock_sendto(PySocketSockObject *s, PyObject *args)
         return NULL;
     }
 
+    if (PySys_Audit("socket.sendto", "OO", s, addro) < 0) {
+        return NULL;
+    }
+
     ctx.buf = pbuf.buf;
     ctx.len = pbuf.len;
     ctx.flags = flags;
@@ -4379,8 +4396,15 @@ sock_sendmsg(PySocketSockObject *s, PyObject *args)
         {
             goto finally;
         }
+        if (PySys_Audit("socket.sendmsg", "OO", s, addr_arg) < 0) {
+            return NULL;
+        }
         msg.msg_name = &addrbuf;
         msg.msg_namelen = addrlen;
+    } else {
+        if (PySys_Audit("socket.sendmsg", "OO", s, Py_None) < 0) {
+            return NULL;
+        }
     }
 
     /* Fill in an iovec for each message part, and save the Py_buffer
@@ -5030,6 +5054,17 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds)
                                      &family, &type, &proto, &fdobj))
         return -1;
 
+#ifdef MS_WINDOWS
+    /* In this case, we don't use the family, type and proto args */
+    if (fdobj != NULL && fdobj != Py_None)
+#endif
+    {
+        if (PySys_Audit("socket.__new__", "Oiii",
+                        s, family, type, proto) < 0) {
+            return -1;
+        }
+    }
+
     if (fdobj != NULL && fdobj != Py_None) {
 #ifdef MS_WINDOWS
         /* recreate a socket that was duplicated */
@@ -5042,6 +5077,12 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds)
                 return -1;
             }
             memcpy(&info, PyBytes_AS_STRING(fdobj), sizeof(info));
+
+            if (PySys_Audit("socket()", "iii", info.iAddressFamily,
+                            info.iSocketType, info.iProtocol) < 0) {
+                return -1;
+            }
+
             Py_BEGIN_ALLOW_THREADS
             fd = WSASocketW(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO,
                      FROM_PROTOCOL_INFO, &info, 0, WSA_FLAG_OVERLAPPED);
@@ -5284,6 +5325,10 @@ static PyTypeObject sock_type = {
 static PyObject *
 socket_gethostname(PyObject *self, PyObject *unused)
 {
+    if (PySys_Audit("socket.gethostname", NULL) < 0) {
+        return NULL;
+    }
+
 #ifdef MS_WINDOWS
     /* Don't use winsock's gethostname, as this returns the ANSI
        version of the hostname, whereas we need a Unicode string.
@@ -5362,6 +5407,11 @@ extern int sethostname(const char *, size_t);
             return NULL;
         flag = 1;
     }
+
+    if (PySys_Audit("socket.sethostname", "(O)", hnobj) < 0) {
+        return NULL;
+    }
+
     res = PyObject_GetBuffer(hnobj, &buf, PyBUF_SIMPLE);
     if (!res) {
         res = sethostname(buf.buf, buf.len);
@@ -5387,6 +5437,9 @@ socket_gethostbyname(PyObject *self, PyObject *args)
 
     if (!PyArg_ParseTuple(args, "et:gethostbyname", "idna", &name))
         return NULL;
+    if (PySys_Audit("socket.gethostbyname", "O", args) < 0) {
+        goto finally;
+    }
     if (setipaddr(name, (struct sockaddr *)&addrbuf,  sizeof(addrbuf), AF_INET) < 0)
         goto finally;
     ret = make_ipv4_addr(&addrbuf);
@@ -5571,6 +5624,9 @@ socket_gethostbyname_ex(PyObject *self, PyObject *args)
 
     if (!PyArg_ParseTuple(args, "et:gethostbyname_ex", "idna", &name))
         return NULL;
+    if (PySys_Audit("socket.gethostbyname", "O", args) < 0) {
+        goto finally;
+    }
     if (setipaddr(name, SAS2SA(&addr), sizeof(addr), AF_INET) < 0)
         goto finally;
     Py_BEGIN_ALLOW_THREADS
@@ -5649,6 +5705,9 @@ socket_gethostbyaddr(PyObject *self, PyObject *args)
 
     if (!PyArg_ParseTuple(args, "et:gethostbyaddr", "idna", &ip_num))
         return NULL;
+    if (PySys_Audit("socket.gethostbyaddr", "O", args) < 0) {
+        goto finally;
+    }
     af = AF_UNSPEC;
     if (setipaddr(ip_num, sa, sizeof(addr), af) < 0)
         goto finally;
@@ -5720,6 +5779,11 @@ socket_getservbyname(PyObject *self, PyObject *args)
     struct servent *sp;
     if (!PyArg_ParseTuple(args, "s|s:getservbyname", &name, &proto))
         return NULL;
+
+    if (PySys_Audit("socket.getservbyname", "ss", name, proto) < 0) {
+        return NULL;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     sp = getservbyname(name, proto);
     Py_END_ALLOW_THREADS
@@ -5757,6 +5821,11 @@ socket_getservbyport(PyObject *self, PyObject *args)
             "getservbyport: port must be 0-65535.");
         return NULL;
     }
+
+    if (PySys_Audit("socket.getservbyport", "is", port, proto) < 0) {
+        return NULL;
+    }
+
     Py_BEGIN_ALLOW_THREADS
     sp = getservbyport(htons((short)port), proto);
     Py_END_ALLOW_THREADS
@@ -6392,6 +6461,12 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs)
         pptr = "00";
     }
 #endif
+
+    if (PySys_Audit("socket.getaddrinfo", "OOiii",
+                    hobj, pobj, family, socktype, protocol) < 0) {
+        return NULL;
+    }
+
     memset(&hints, 0, sizeof(hints));
     hints.ai_family = family;
     hints.ai_socktype = socktype;
@@ -6483,6 +6558,11 @@ socket_getnameinfo(PyObject *self, PyObject *args)
                         "getnameinfo(): flowinfo must be 0-1048575.");
         return NULL;
     }
+
+    if (PySys_Audit("socket.getnameinfo", "(O)", sa) < 0) {
+        return NULL;
+    }
+
     PyOS_snprintf(pbuf, sizeof(pbuf), "%d", port);
     memset(&hints, 0, sizeof(hints));
     hints.ai_family = AF_UNSPEC;

Objects/codeobject.c

@@ -380,6 +380,12 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kw)
                           &PyTuple_Type, &cellvars))
         return NULL;
 
+    if (PySys_Audit("code.__new__", "OOOiiiii",
+                    code, filename, name, argcount, kwonlyargcount,
+                    nlocals, stacksize, flags) < 0) {
+        goto cleanup;
+    }
+
     if (argcount < 0) {
         PyErr_SetString(
             PyExc_ValueError,

Objects/descrobject.c

@@ -144,6 +144,14 @@ member_get(PyMemberDescrObject *descr, PyObject *obj, PyObject *type)
 
     if (descr_check((PyDescrObject *)descr, obj, &res))
         return res;
+
+    if (descr->d_member->flags & READ_RESTRICTED) {
+        if (PySys_Audit("object.__getattr__", "Os",
+            obj ? obj : Py_None, descr->d_member->name) < 0) {
+            return NULL;
+        }
+    }
+
     return PyMember_GetOne((char *)obj, descr->d_member);
 }
 

Objects/fileobject.c

@@ -2,6 +2,7 @@
 
 #define PY_SSIZE_T_CLEAN
 #include "Python.h"
+#include "pycore_pystate.h"
 
 #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER)
 /* clang MemorySanitizer doesn't yet understand getc_unlocked. */
@@ -33,7 +34,8 @@ PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const c
     PyObject *io, *stream;
     _Py_IDENTIFIER(open);
 
-    io = PyImport_ImportModule("io");
+    /* import _io in case we are being used to open io.py */
+    io = PyImport_ImportModule("_io");
     if (io == NULL)
         return NULL;
     stream = _PyObject_CallMethodId(io, &PyId_open, "isisssi", fd, mode,
@@ -514,6 +516,72 @@ PyTypeObject PyStdPrinter_Type = {
 };
 
 
+/* ************************** open_code hook ***************************
+ * The open_code hook allows embedders to override the method used to
+ * open files that are going to be used by the runtime to execute code
+ */
+
+int
+PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData) {
+    if (Py_IsInitialized() &&
+        PySys_Audit("setopencodehook", NULL) < 0) {
+        return -1;
+    }
+
+    if (_PyRuntime.open_code_hook) {
+        if (Py_IsInitialized()) {
+            PyErr_SetString(PyExc_SystemError,
+                "failed to change existing open_code hook");
+        }
+        return -1;
+    }
+
+    _PyRuntime.open_code_hook = hook;
+    _PyRuntime.open_code_userdata = userData;
+    return 0;
+}
+
+PyObject *
+PyFile_OpenCodeObject(PyObject *path)
+{
+    PyObject *iomod, *f = NULL;
+    _Py_IDENTIFIER(open);
+
+    if (!PyUnicode_Check(path)) {
+        PyErr_Format(PyExc_TypeError, "'path' must be 'str', not '%.200s'",
+                     Py_TYPE(path)->tp_name);
+        return NULL;
+    }
+
+    Py_OpenCodeHookFunction hook = _PyRuntime.open_code_hook;
+    if (hook) {
+        f = hook(path, _PyRuntime.open_code_userdata);
+    } else {
+        iomod = PyImport_ImportModule("_io");
+        if (iomod) {
+            f = _PyObject_CallMethodId(iomod, &PyId_open, "Os",
+                                       path, "rb");
+            Py_DECREF(iomod);
+        }
+    }
+
+    return f;
+}
+
+PyObject *
+PyFile_OpenCode(const char *utf8path)
+{
+    PyObject *pathobj = PyUnicode_FromString(utf8path);
+    PyObject *f;
+    if (!pathobj) {
+        return NULL;
+    }
+    f = PyFile_OpenCodeObject(pathobj);
+    Py_DECREF(pathobj);
+    return f;
+}
+
+
 #ifdef __cplusplus
 }
 #endif

Objects/funcobject.c

@@ -250,6 +250,10 @@ static PyMemberDef func_memberlist[] = {
 static PyObject *
 func_get_code(PyFunctionObject *op, void *Py_UNUSED(ignored))
 {
+    if (PySys_Audit("object.__getattr__", "Os", op, "__code__") < 0) {
+        return NULL;
+    }
+
     Py_INCREF(op->func_code);
     return op->func_code;
 }
@@ -266,6 +270,12 @@ func_set_code(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored))
                         "__code__ must be set to a code object");
         return -1;
     }
+
+    if (PySys_Audit("object.__setattr__", "OsO",
+                    op, "__code__", value) < 0) {
+        return -1;
+    }
+
     nfree = PyCode_GetNumFree((PyCodeObject *)value);
     nclosure = (op->func_closure == NULL ? 0 :
             PyTuple_GET_SIZE(op->func_closure));
@@ -329,6 +339,9 @@ func_set_qualname(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored
 static PyObject *
 func_get_defaults(PyFunctionObject *op, void *Py_UNUSED(ignored))
 {
+    if (PySys_Audit("object.__getattr__", "Os", op, "__defaults__") < 0) {
+        return NULL;
+    }
     if (op->func_defaults == NULL) {
         Py_RETURN_NONE;
     }
@@ -348,6 +361,16 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored
                         "__defaults__ must be set to a tuple object");
         return -1;
     }
+    if (value) {
+        if (PySys_Audit("object.__setattr__", "OsO",
+                        op, "__defaults__", value) < 0) {
+            return -1;
+        }
+    } else if (PySys_Audit("object.__delattr__", "Os",
+                           op, "__defaults__") < 0) {
+        return -1;
+    }
+
     Py_XINCREF(value);
     Py_XSETREF(op->func_defaults, value);
     return 0;
@@ -356,6 +379,10 @@ func_set_defaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignored
 static PyObject *
 func_get_kwdefaults(PyFunctionObject *op, void *Py_UNUSED(ignored))
 {
+    if (PySys_Audit("object.__getattr__", "Os",
+                    op, "__kwdefaults__") < 0) {
+        return NULL;
+    }
     if (op->func_kwdefaults == NULL) {
         Py_RETURN_NONE;
     }
@@ -375,6 +402,16 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value, void *Py_UNUSED(ignor
             "__kwdefaults__ must be set to a dict object");
         return -1;
     }
+    if (value) {
+        if (PySys_Audit("object.__setattr__", "OsO",
+                        op, "__kwdefaults__", value) < 0) {
+            return -1;
+        }
+    } else if (PySys_Audit("object.__delattr__", "Os",
+                           op, "__kwdefaults__") < 0) {
+        return -1;
+    }
+
     Py_XINCREF(value);
     Py_XSETREF(op->func_kwdefaults, value);
     return 0;
@@ -507,6 +544,9 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals,
             }
         }
     }
+    if (PySys_Audit("function.__new__", "O", code) < 0) {
+        return NULL;
+    }
 
     newfunc = (PyFunctionObject *)PyFunction_New((PyObject *)code,
                                                  globals);

Objects/object.c

@@ -1366,6 +1366,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name,
         }
     }
 
+    /* XXX [Steve Dower] These are really noisy - worth it? */
+    /*if (PyType_Check(obj) || PyModule_Check(obj)) {
+        if (value && PySys_Audit("object.__setattr__", "OOO", obj, name, value) < 0)
+            return -1;
+        if (!value && PySys_Audit("object.__delattr__", "OO", obj, name) < 0)
+            return -1;
+    }*/
+
     if (dict == NULL) {
         dictptr = _PyObject_GetDictPtr(obj);
         if (dictptr == NULL) {

Objects/typeobject.c

@@ -392,6 +392,12 @@ check_set_special_type_attr(PyTypeObject *type, PyObject *value, const char *nam
                      "can't delete %s.%s", type->tp_name, name);
         return 0;
     }
+
+    if (PySys_Audit("object.__setattr__", "OsO",
+                    type, name, value) < 0) {
+        return 0;
+    }
+
     return 1;
 }
 
@@ -3956,6 +3962,11 @@ object_set_class(PyObject *self, PyObject *value, void *closure)
           Py_TYPE(value)->tp_name);
         return -1;
     }
+    if (PySys_Audit("object.__setattr__", "OsO",
+                    self, "__class__", value) < 0) {
+        return -1;
+    }
+
     newto = (PyTypeObject *)value;
     /* In versions of CPython prior to 3.5, the code in
        compatible_for_assignment was not set up to correctly check for memory

PCbuild/pythoncore.vcxproj

@@ -129,12 +129,14 @@
     <ClInclude Include="..\Include\cpython\abstract.h" />
     <ClInclude Include="..\Include\cpython\coreconfig.h" />
     <ClInclude Include="..\Include\cpython\dictobject.h" />
+    <ClInclude Include="..\Include\cpython\fileobject.h" />
     <ClInclude Include="..\Include\cpython\object.h" />
     <ClInclude Include="..\Include\cpython\objimpl.h" />
     <ClInclude Include="..\Include\cpython\pyerrors.h" />
     <ClInclude Include="..\Include\cpython\pylifecycle.h" />
     <ClInclude Include="..\Include\cpython\pymem.h" />
     <ClInclude Include="..\Include\cpython\pystate.h" />
+    <ClInclude Include="..\Include\cpython\sysmodule.h" />
     <ClInclude Include="..\Include\cpython\traceback.h" />
     <ClInclude Include="..\Include\cpython\tupleobject.h" />
     <ClInclude Include="..\Include\cpython\unicodeobject.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -90,6 +90,9 @@
     <ClInclude Include="..\Include\cpython\dictobject.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\cpython\fileobject.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\cpython\object.h">
       <Filter>Include</Filter>
     </ClInclude>
@@ -108,6 +111,9 @@
     <ClInclude Include="..\Include\cpython\pystate.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\cpython\sysmodule.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\cpython\traceback.h">
       <Filter>Include</Filter>
     </ClInclude>

Parser/asdl_c.py

@@ -1201,6 +1201,10 @@ mod_ty PyAST_obj2mod_ex(PyObject* ast, PyArena* arena, int mode, int feature_ver
     char *req_name[] = {"Module", "Expression", "Interactive"};
     int isinstance;
 
+    if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {
+        return NULL;
+    }
+
     req_type[0] = (PyObject*)Module_type;
     req_type[1] = (PyObject*)Expression_type;
     req_type[2] = (PyObject*)Interactive_type;

Parser/parsetok.c

@@ -94,6 +94,11 @@ PyParser_ParseStringObject(const char *s, PyObject *filename,
     if (initerr(err_ret, filename) < 0)
         return NULL;
 
+    if (PySys_Audit("compile", "yO", s, err_ret->filename) < 0) {
+        err_ret->error = E_ERROR;
+        return NULL;
+    }
+
     if (*flags & PyPARSE_IGNORE_COOKIE)
         tok = PyTokenizer_FromUTF8(s, exec_input);
     else
@@ -165,6 +170,10 @@ PyParser_ParseFileObject(FILE *fp, PyObject *filename,
     if (initerr(err_ret, filename) < 0)
         return NULL;
 
+    if (PySys_Audit("compile", "OO", Py_None, err_ret->filename) < 0) {
+        return NULL;
+    }
+
     if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
         err_ret->error = E_NOMEM;
         return NULL;

Programs/_testembed.c

@@ -1091,6 +1091,164 @@ static int test_init_dev_mode(void)
     return 0;
 }
 
+static PyObject *_open_code_hook(PyObject *path, void *data)
+{
+    if (PyUnicode_CompareWithASCIIString(path, "$$test-filename") == 0) {
+        return PyLong_FromVoidPtr(data);
+    }
+    PyObject *io = PyImport_ImportModule("_io");
+    if (!io) {
+        return NULL;
+    }
+    return PyObject_CallMethod(io, "open", "Os", path, "rb");
+}
+
+static int test_open_code_hook(void)
+{
+    int result = 0;
+
+    /* Provide a hook */
+    result = PyFile_SetOpenCodeHook(_open_code_hook, &result);
+    if (result) {
+        printf("Failed to set hook\n");
+        return 1;
+    }
+    /* A second hook should fail */
+    result = PyFile_SetOpenCodeHook(_open_code_hook, &result);
+    if (!result) {
+        printf("Should have failed to set second hook\n");
+        return 2;
+    }
+
+    Py_IgnoreEnvironmentFlag = 0;
+    _testembed_Py_Initialize();
+    result = 0;
+
+    PyObject *r = PyFile_OpenCode("$$test-filename");
+    if (!r) {
+        PyErr_Print();
+        result = 3;
+    } else {
+        void *cmp = PyLong_AsVoidPtr(r);
+        Py_DECREF(r);
+        if (cmp != &result) {
+            printf("Did not get expected result from hook\n");
+            result = 4;
+        }
+    }
+
+    if (!result) {
+        PyObject *io = PyImport_ImportModule("_io");
+        PyObject *r = io
+            ? PyObject_CallMethod(io, "open_code", "s", "$$test-filename")
+            : NULL;
+        if (!r) {
+            PyErr_Print();
+            result = 5;
+        } else {
+            void *cmp = PyLong_AsVoidPtr(r);
+            Py_DECREF(r);
+            if (cmp != &result) {
+                printf("Did not get expected result from hook\n");
+                result = 6;
+            }
+        }
+        Py_XDECREF(io);
+    }
+
+    Py_Finalize();
+    return result;
+}
+
+static int _audit_hook(const char *event, PyObject *args, void *userdata)
+{
+    if (strcmp(event, "_testembed.raise") == 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Intentional error");
+        return -1;
+    } else if (strcmp(event, "_testembed.set") == 0) {
+        if (!PyArg_ParseTuple(args, "n", userdata)) {
+            return -1;
+        }
+        return 0;
+    }
+    return 0;
+}
+
+static int _test_audit(Py_ssize_t setValue)
+{
+    Py_ssize_t sawSet = 0;
+
+    Py_IgnoreEnvironmentFlag = 0;
+    PySys_AddAuditHook(_audit_hook, &sawSet);
+    _testembed_Py_Initialize();
+
+    if (PySys_Audit("_testembed.raise", NULL) == 0) {
+        printf("No error raised");
+        return 1;
+    }
+    if (PySys_Audit("_testembed.nop", NULL) != 0) {
+        printf("Nop event failed");
+        /* Exception from above may still remain */
+        PyErr_Clear();
+        return 2;
+    }
+    if (!PyErr_Occurred()) {
+        printf("Exception not preserved");
+        return 3;
+    }
+    PyErr_Clear();
+
+    if (PySys_Audit("_testembed.set", "n", setValue) != 0) {
+        PyErr_Print();
+        printf("Set event failed");
+        return 4;
+    }
+
+    if (sawSet != 42) {
+        printf("Failed to see *userData change\n");
+        return 5;
+    }
+    return 0;
+}
+
+static int test_audit(void)
+{
+    int result = _test_audit(42);
+    Py_Finalize();
+    return result;
+}
+
+static volatile int _audit_subinterpreter_interpreter_count = 0;
+
+static int _audit_subinterpreter_hook(const char *event, PyObject *args, void *userdata)
+{
+    printf("%s\n", event);
+    if (strcmp(event, "cpython.PyInterpreterState_New") == 0) {
+        _audit_subinterpreter_interpreter_count += 1;
+    }
+    return 0;
+}
+
+static int test_audit_subinterpreter(void)
+{
+    PyThreadState *ts;
+
+    Py_IgnoreEnvironmentFlag = 0;
+    PySys_AddAuditHook(_audit_subinterpreter_hook, NULL);
+    _testembed_Py_Initialize();
+
+    ts = Py_NewInterpreter();
+    ts = Py_NewInterpreter();
+    ts = Py_NewInterpreter();
+
+    Py_Finalize();
+
+    switch (_audit_subinterpreter_interpreter_count) {
+        case 3: return 0;
+        case 0: return -1;
+        default: return _audit_subinterpreter_interpreter_count;
+    }
+}
 
 static int test_init_read_set(void)
 {
@@ -1299,6 +1457,9 @@ static struct TestCase TestCases[] = {
     {"test_init_run_main", test_init_run_main},
     {"test_init_main", test_init_main},
     {"test_run_main", test_run_main},
+    {"test_open_code_hook", test_open_code_hook},
+    {"test_audit", test_audit},
+    {"test_audit_subinterpreter", test_audit_subinterpreter},
     {NULL, NULL}
 };
 

Python/Python-ast.c

@@ -9024,6 +9024,10 @@ mod_ty PyAST_obj2mod_ex(PyObject* ast, PyArena* arena, int mode, int feature_ver
     char *req_name[] = {"Module", "Expression", "Interactive"};
     int isinstance;
 
+    if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {
+        return NULL;
+    }
+
     req_type[0] = (PyObject*)Module_type;
     req_type[1] = (PyObject*)Expression_type;
     req_type[2] = (PyObject*)Interactive_type;

Python/bltinmodule.c

@@ -977,9 +977,13 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals,
     }
 
     if (PyCode_Check(source)) {
+        if (PySys_Audit("exec", "O", source) < 0) {
+            return NULL;
+        }
+
         if (PyCode_GetNumFree((PyCodeObject *)source) > 0) {
             PyErr_SetString(PyExc_TypeError,
-        "code object passed to eval() may not contain free variables");
+                "code object passed to eval() may not contain free variables");
             return NULL;
         }
         return PyEval_EvalCode(source, globals, locals);
@@ -1061,6 +1065,10 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals,
     }
 
     if (PyCode_Check(source)) {
+        if (PySys_Audit("exec", "O", source) < 0) {
+            return NULL;
+        }
+
         if (PyCode_GetNumFree((PyCodeObject *)source) > 0) {
             PyErr_SetString(PyExc_TypeError,
                 "code object passed to exec() may not "
@@ -1207,7 +1215,14 @@ static PyObject *
 builtin_id(PyModuleDef *self, PyObject *v)
 /*[clinic end generated code: output=0aa640785f697f65 input=5a534136419631f4]*/
 {
-    return PyLong_FromVoidPtr(v);
+    PyObject *id = PyLong_FromVoidPtr(v);
+
+    if (id && PySys_Audit("builtins.id", "O", id) < 0) {
+        Py_DECREF(id);
+        return NULL;
+    }
+
+    return id;
 }
 
 
@@ -1986,6 +2001,10 @@ builtin_input_impl(PyObject *module, PyObject *prompt)
         return NULL;
     }
 
+    if (PySys_Audit("builtins.input", "O", prompt ? prompt : Py_None) < 0) {
+        return NULL;
+    }
+
     /* First of all, flush stderr */
     tmp = _PyObject_CallMethodId(ferr, &PyId_flush, NULL);
     if (tmp == NULL)
@@ -2116,6 +2135,13 @@ builtin_input_impl(PyObject *module, PyObject *prompt)
         Py_DECREF(stdin_errors);
         Py_XDECREF(po);
         PyMem_FREE(s);
+
+        if (result != NULL) {
+            if (PySys_Audit("builtins.input/result", "O", result) < 0) {
+                return NULL;
+            }
+        }
+
         return result;
 
     _readline_errors:

Python/ceval.c

@@ -4555,6 +4555,10 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
 void
 PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
 {
+    if (PySys_Audit("sys.setprofile", NULL) < 0) {
+        return;
+    }
+
     PyThreadState *tstate = _PyThreadState_GET();
     PyObject *temp = tstate->c_profileobj;
     Py_XINCREF(arg);
@@ -4572,6 +4576,10 @@ PyEval_SetProfile(Py_tracefunc func, PyObject *arg)
 void
 PyEval_SetTrace(Py_tracefunc func, PyObject *arg)
 {
+    if (PySys_Audit("sys.settrace", NULL) < 0) {
+        return;
+    }
+
     _PyRuntimeState *runtime = &_PyRuntime;
     PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
     PyObject *temp = tstate->c_traceobj;
@@ -4608,6 +4616,11 @@ void
 _PyEval_SetCoroutineWrapper(PyObject *wrapper)
 {
     PyThreadState *tstate = _PyThreadState_GET();
+
+    if (PySys_Audit("sys.set_coroutine_wrapper", NULL) < 0) {
+        return;
+    }
+
     Py_XINCREF(wrapper);
     Py_XSETREF(tstate->coroutine_wrapper, wrapper);
 }
@@ -4623,6 +4636,11 @@ void
 _PyEval_SetAsyncGenFirstiter(PyObject *firstiter)
 {
     PyThreadState *tstate = _PyThreadState_GET();
+
+    if (PySys_Audit("sys.set_asyncgen_hook_firstiter", NULL) < 0) {
+        return;
+    }
+
     Py_XINCREF(firstiter);
     Py_XSETREF(tstate->async_gen_firstiter, firstiter);
 }
@@ -4638,6 +4656,11 @@ void
 _PyEval_SetAsyncGenFinalizer(PyObject *finalizer)
 {
     PyThreadState *tstate = _PyThreadState_GET();
+
+    if (PySys_Audit("sys.set_asyncgen_hook_finalizer", NULL) < 0) {
+        return;
+    }
+
     Py_XINCREF(finalizer);
     Py_XSETREF(tstate->async_gen_finalizer, finalizer);
 }

Python/clinic/sysmodule.c.h

@@ -2,6 +2,38 @@
 preserve
 [clinic start generated code]*/
 
+PyDoc_STRVAR(sys_addaudithook__doc__,
+"addaudithook($module, /, hook)\n"
+"--\n"
+"\n"
+"Adds a new audit hook callback.");
+
+#define SYS_ADDAUDITHOOK_METHODDEF    \
+    {"addaudithook", (PyCFunction)(void(*)(void))sys_addaudithook, METH_FASTCALL|METH_KEYWORDS, sys_addaudithook__doc__},
+
+static PyObject *
+sys_addaudithook_impl(PyObject *module, PyObject *hook);
+
+static PyObject *
+sys_addaudithook(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    static const char * const _keywords[] = {"hook", NULL};
+    static _PyArg_Parser _parser = {NULL, _keywords, "addaudithook", 0};
+    PyObject *argsbuf[1];
+    PyObject *hook;
+
+    args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf);
+    if (!args) {
+        goto exit;
+    }
+    hook = args[0];
+    return_value = sys_addaudithook_impl(module, hook);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(sys_displayhook__doc__,
 "displayhook($module, object, /)\n"
 "--\n"
@@ -1076,4 +1108,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=603e4d5a453dc769 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=3c32bc91ec659509 input=a9049054013a1b77]*/

Python/fileutils.c

@@ -1262,6 +1262,10 @@ _Py_open_impl(const char *pathname, int flags, int gil_held)
 #endif
 
     if (gil_held) {
+        if (PySys_Audit("open", "sOi", pathname, Py_None, flags) < 0) {
+            return -1;
+        }
+
         do {
             Py_BEGIN_ALLOW_THREADS
             fd = open(pathname, flags);
@@ -1331,6 +1335,9 @@ FILE *
 _Py_wfopen(const wchar_t *path, const wchar_t *mode)
 {
     FILE *f;
+    if (PySys_Audit("open", "uui", path, mode, 0) < 0) {
+        return NULL;
+    }
 #ifndef MS_WINDOWS
     char *cpath;
     char cmode[10];
@@ -1366,6 +1373,10 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
 FILE*
 _Py_fopen(const char *pathname, const char *mode)
 {
+    if (PySys_Audit("open", "ssi", pathname, mode, 0) < 0) {
+        return NULL;
+    }
+
     FILE *f = fopen(pathname, mode);
     if (f == NULL)
         return NULL;
@@ -1401,6 +1412,9 @@ _Py_fopen_obj(PyObject *path, const char *mode)
 
     assert(PyGILState_Check());
 
+    if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
+        return NULL;
+    }
     if (!PyUnicode_Check(path)) {
         PyErr_Format(PyExc_TypeError,
                      "str file path expected under Windows, got %R",
@@ -1434,6 +1448,10 @@ _Py_fopen_obj(PyObject *path, const char *mode)
         return NULL;
     path_bytes = PyBytes_AS_STRING(bytes);
 
+    if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
+        return NULL;
+    }
+
     do {
         Py_BEGIN_ALLOW_THREADS
         f = fopen(path_bytes, mode);

Python/import.c

@@ -1661,6 +1661,17 @@ import_find_and_load(PyObject *abs_name)
 
     _PyTime_t t1 = 0, accumulated_copy = accumulated;
 
+    PyObject *sys_path = PySys_GetObject("path");
+    PyObject *sys_meta_path = PySys_GetObject("meta_path");
+    PyObject *sys_path_hooks = PySys_GetObject("path_hooks");
+    if (PySys_Audit("import", "OOOOO",
+                    abs_name, Py_None, sys_path ? sys_path : Py_None,
+                    sys_meta_path ? sys_meta_path : Py_None,
+                    sys_path_hooks ? sys_path_hooks : Py_None) < 0) {
+        return NULL;
+    }
+
+
     /* XOptions is initialized after first some imports.
      * So we can't have negative cache before completed initialization.
      * Anyway, importlib._find_and_load is much slower than

Python/importdl.c

@@ -119,6 +119,11 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
     if (path == NULL)
         goto error;
 
+    if (PySys_Audit("import", "OOOOO", name_unicode, path,
+                    Py_None, Py_None, Py_None) < 0) {
+        return NULL;
+    }
+
 #ifdef MS_WINDOWS
     exportfunc = _PyImport_FindSharedFuncptrWindows(hook_prefix, name_buf,
                                                     path, fp);

Python/pylifecycle.c

@@ -1250,6 +1250,13 @@ Py_FinalizeEx(void)
         /* nothing */;
 #endif
 
+    /* Clear all loghooks */
+    /* We want minimal exposure of this function, so define the extern
+     * here. The linker should discover the correct function without
+     * exporting a symbol. */
+    extern void _PySys_ClearAuditHooks(void);
+    _PySys_ClearAuditHooks();
+
     /* Destroy all modules */
     PyImport_Cleanup();
 

Python/pystate.c

@@ -45,8 +45,19 @@ static void _PyThreadState_Delete(_PyRuntimeState *runtime, PyThreadState *tstat
 static _PyInitError
 _PyRuntimeState_Init_impl(_PyRuntimeState *runtime)
 {
+    /* We preserve the hook across init, because there is
+       currently no public API to set it between runtime
+       initialization and interpreter initialization. */
+    void *open_code_hook = runtime->open_code_hook;
+    void *open_code_userdata = runtime->open_code_userdata;
+    _Py_AuditHookEntry *audit_hook_head = runtime->audit_hook_head;
+
     memset(runtime, 0, sizeof(*runtime));
 
+    runtime->open_code_hook = open_code_hook;
+    runtime->open_code_userdata = open_code_userdata;
+    runtime->audit_hook_head = audit_hook_head;
+
     _PyGC_Initialize(&runtime->gc);
     _PyEval_Initialize(&runtime->ceval);
     _PyPreConfig_InitPythonConfig(&runtime->preconfig);
@@ -181,6 +192,10 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime)
 PyInterpreterState *
 PyInterpreterState_New(void)
 {
+    if (PySys_Audit("cpython.PyInterpreterState_New", NULL) < 0) {
+        return NULL;
+    }
+
     PyInterpreterState *interp = PyMem_RawMalloc(sizeof(PyInterpreterState));
     if (interp == NULL) {
         return NULL;
@@ -233,6 +248,8 @@ PyInterpreterState_New(void)
 
     interp->tstate_next_unique_id = 0;
 
+    interp->audit_hooks = NULL;
+
     return interp;
 }
 
@@ -240,11 +257,18 @@ PyInterpreterState_New(void)
 static void
 _PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp)
 {
+    if (PySys_Audit("cpython.PyInterpreterState_Clear", NULL) < 0) {
+        PyErr_Clear();
+    }
+
     HEAD_LOCK(runtime);
     for (PyThreadState *p = interp->tstate_head; p != NULL; p = p->next) {
         PyThreadState_Clear(p);
     }
     HEAD_UNLOCK(runtime);
+
+    Py_CLEAR(interp->audit_hooks);
+
     _PyCoreConfig_Clear(&interp->core_config);
     Py_CLEAR(interp->codec_search_path);
     Py_CLEAR(interp->codec_search_cache);
@@ -1057,6 +1081,10 @@ _PyThread_CurrentFrames(void)
     PyObject *result;
     PyInterpreterState *i;
 
+    if (PySys_Audit("sys._current_frames", NULL) < 0) {
+        return NULL;
+    }
+
     result = PyDict_New();
     if (result == NULL)
         return NULL;

Python/pythonrun.c

@@ -1091,6 +1091,12 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals,
     co = PyAST_CompileObject(mod, filename, flags, -1, arena);
     if (co == NULL)
         return NULL;
+
+    if (PySys_Audit("exec", "O", co) < 0) {
+        Py_DECREF(co);
+        return NULL;
+    }
+
     v = run_eval_code_obj(co, globals, locals);
     Py_DECREF(co);
     return v;

Python/sysmodule.c

@@ -22,7 +22,9 @@ Data members:
 #include "pycore_pymem.h"
 #include "pycore_pathconfig.h"
 #include "pycore_pystate.h"
+#include "pycore_tupleobject.h"
 #include "pythread.h"
+#include "pydtrace.h"
 
 #include "osdefs.h"
 #include <locale.h>
@@ -111,6 +113,308 @@ PySys_SetObject(const char *name, PyObject *v)
     }
 }
 
+static int
+should_audit(void)
+{
+    PyThreadState *ts = _PyThreadState_GET();
+    if (!ts) {
+        return 0;
+    }
+    PyInterpreterState *is = ts ? ts->interp : NULL;
+    return _PyRuntime.audit_hook_head
+        || (is && is->audit_hooks)
+        || PyDTrace_AUDIT_ENABLED();
+}
+
+int
+PySys_Audit(const char *event, const char *argFormat, ...)
+{
+    PyObject *eventName = NULL;
+    PyObject *eventArgs = NULL;
+    PyObject *hooks = NULL;
+    PyObject *hook = NULL;
+    int res = -1;
+
+    /* N format is inappropriate, because you do not know
+       whether the reference is consumed by the call.
+       Assert rather than exception for perf reasons */
+    assert(!argFormat || !strchr(argFormat, 'N'));
+
+    /* Early exit when no hooks are registered */
+    if (!should_audit()) {
+        return 0;
+    }
+
+    _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head;
+    PyThreadState *ts = _PyThreadState_GET();
+    PyInterpreterState *is = ts ? ts->interp : NULL;
+    int dtrace = PyDTrace_AUDIT_ENABLED();
+
+    PyObject *exc_type, *exc_value, *exc_tb;
+    if (ts) {
+        PyErr_Fetch(&exc_type, &exc_value, &exc_tb);
+    }
+
+    /* Initialize event args now */
+    if (argFormat && argFormat[0]) {
+        va_list args;
+        va_start(args, argFormat);
+        eventArgs = Py_VaBuildValue(argFormat, args);
+        if (eventArgs && !PyTuple_Check(eventArgs)) {
+            PyObject *argTuple = PyTuple_Pack(1, eventArgs);
+            Py_DECREF(eventArgs);
+            eventArgs = argTuple;
+        }
+    } else {
+        eventArgs = PyTuple_New(0);
+    }
+    if (!eventArgs) {
+        goto exit;
+    }
+
+    /* Call global hooks */
+    for (; e; e = e->next) {
+        if (e->hookCFunction(event, eventArgs, e->userData) < 0) {
+            goto exit;
+        }
+    }
+
+    /* Dtrace USDT point */
+    if (dtrace) {
+        PyDTrace_AUDIT(event, (void *)eventArgs);
+    }
+
+    /* Call interpreter hooks */
+    if (is && is->audit_hooks) {
+        eventName = PyUnicode_FromString(event);
+        if (!eventName) {
+            goto exit;
+        }
+
+        hooks = PyObject_GetIter(is->audit_hooks);
+        if (!hooks) {
+            goto exit;
+        }
+
+        /* Disallow tracing in hooks unless explicitly enabled */
+        ts->tracing++;
+        ts->use_tracing = 0;
+        while ((hook = PyIter_Next(hooks)) != NULL) {
+            PyObject *o;
+            int canTrace = -1;
+            o = PyObject_GetAttrString(hook, "__cantrace__");
+            if (o) {
+                canTrace = PyObject_IsTrue(o);
+                Py_DECREF(o);
+            } else if (PyErr_Occurred() &&
+                       PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                PyErr_Clear();
+                canTrace = 0;
+            }
+            if (canTrace < 0) {
+                break;
+            }
+            if (canTrace) {
+                ts->use_tracing = (ts->c_tracefunc || ts->c_profilefunc);
+                ts->tracing--;
+            }
+            o = PyObject_CallFunctionObjArgs(hook, eventName,
+                                             eventArgs, NULL);
+            if (canTrace) {
+                ts->tracing++;
+                ts->use_tracing = 0;
+            }
+            if (!o) {
+                break;
+            }
+            Py_DECREF(o);
+            Py_CLEAR(hook);
+        }
+        ts->use_tracing = (ts->c_tracefunc || ts->c_profilefunc);
+        ts->tracing--;
+        if (PyErr_Occurred()) {
+            goto exit;
+        }
+    }
+
+    res = 0;
+
+exit:
+    Py_XDECREF(hook);
+    Py_XDECREF(hooks);
+    Py_XDECREF(eventName);
+    Py_XDECREF(eventArgs);
+
+    if (ts) {
+        if (!res) {
+            PyErr_Restore(exc_type, exc_value, exc_tb);
+        } else {
+            assert(PyErr_Occurred());
+            Py_XDECREF(exc_type);
+            Py_XDECREF(exc_value);
+            Py_XDECREF(exc_tb);
+        }
+    }
+
+    return res;
+}
+
+/* We expose this function primarily for our own cleanup during
+ * finalization. In general, it should not need to be called,
+ * and as such it is not defined in any header files.
+ */
+void _PySys_ClearAuditHooks(void) {
+    /* Must be finalizing to clear hooks */
+    _PyRuntimeState *runtime = &_PyRuntime;
+    PyThreadState *ts = _PyRuntimeState_GetThreadState(runtime);
+    assert(!ts || _Py_CURRENTLY_FINALIZING(runtime, ts));
+    if (!ts || !_Py_CURRENTLY_FINALIZING(runtime, ts))
+        return;
+
+    if (Py_VerboseFlag) {
+        PySys_WriteStderr("# clear sys.audit hooks\n");
+    }
+
+    /* Hooks can abort later hooks for this event, but cannot
+       abort the clear operation itself. */
+    PySys_Audit("cpython._PySys_ClearAuditHooks", NULL);
+    PyErr_Clear();
+
+    _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head, *n;
+    _PyRuntime.audit_hook_head = NULL;
+    while (e) {
+        n = e->next;
+        PyMem_RawFree(e);
+        e = n;
+    }
+}
+
+int
+PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
+{
+    /* Invoke existing audit hooks to allow them an opportunity to abort. */
+    /* Cannot invoke hooks until we are initialized */
+    if (Py_IsInitialized()) {
+        if (PySys_Audit("sys.addaudithook", NULL) < 0) {
+            if (PyErr_ExceptionMatches(PyExc_Exception)) {
+                /* We do not report errors derived from Exception */
+                PyErr_Clear();
+                return 0;
+            }
+            return -1;
+        }
+    }
+
+    _Py_AuditHookEntry *e = _PyRuntime.audit_hook_head;
+    if (!e) {
+        e = (_Py_AuditHookEntry*)PyMem_RawMalloc(sizeof(_Py_AuditHookEntry));
+        _PyRuntime.audit_hook_head = e;
+    } else {
+        while (e->next)
+            e = e->next;
+        e = e->next = (_Py_AuditHookEntry*)PyMem_RawMalloc(
+            sizeof(_Py_AuditHookEntry));
+    }
+
+    if (!e) {
+        if (Py_IsInitialized())
+            PyErr_NoMemory();
+        return -1;
+    }
+
+    e->next = NULL;
+    e->hookCFunction = (Py_AuditHookFunction)hook;
+    e->userData = userData;
+
+    return 0;
+}
+
+/*[clinic input]
+sys.addaudithook
+
+    hook: object
+
+Adds a new audit hook callback.
+[clinic start generated code]*/
+
+static PyObject *
+sys_addaudithook_impl(PyObject *module, PyObject *hook)
+/*[clinic end generated code: output=4f9c17aaeb02f44e input=0f3e191217a45e34]*/
+{
+    /* Invoke existing audit hooks to allow them an opportunity to abort. */
+    if (PySys_Audit("sys.addaudithook", NULL) < 0) {
+        if (PyErr_ExceptionMatches(PyExc_Exception)) {
+            /* We do not report errors derived from Exception */
+            PyErr_Clear();
+            Py_RETURN_NONE;
+        }
+        return NULL;
+    }
+
+    PyInterpreterState *is = _PyInterpreterState_Get();
+
+    if (is->audit_hooks == NULL) {
+        is->audit_hooks = PyList_New(0);
+        if (is->audit_hooks == NULL) {
+            return NULL;
+        }
+    }
+
+    if (PyList_Append(is->audit_hooks, hook) < 0) {
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(audit_doc,
+"audit(event, *args)\n\
+\n\
+Passes the event to any audit hooks that are attached.");
+
+static PyObject *
+sys_audit(PyObject *self, PyObject *const *args, Py_ssize_t argc)
+{
+    if (argc == 0) {
+        PyErr_SetString(PyExc_TypeError, "audit() missing 1 required positional argument: 'event'");
+        return NULL;
+    }
+
+    if (!should_audit()) {
+        Py_RETURN_NONE;
+    }
+
+    PyObject *auditEvent = args[0];
+    if (!auditEvent) {
+        PyErr_SetString(PyExc_TypeError, "expected str for argument 'event'");
+        return NULL;
+    }
+    if (!PyUnicode_Check(auditEvent)) {
+        PyErr_Format(PyExc_TypeError, "expected str for argument 'event', not %.200s",
+            Py_TYPE(auditEvent)->tp_name);
+        return NULL;
+    }
+    const char *event = PyUnicode_AsUTF8(auditEvent);
+    if (!event) {
+        return NULL;
+    }
+
+    PyObject *auditArgs = _PyTuple_FromArray(args + 1, argc - 1);
+    if (!auditArgs) {
+        return NULL;
+    }
+
+    int res = PySys_Audit(event, "O", auditArgs);
+    Py_DECREF(auditArgs);
+
+    if (res < 0) {
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
+
 static PyObject *
 sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords)
 {
@@ -1469,6 +1773,10 @@ sys__getframe_impl(PyObject *module, int depth)
 {
     PyFrameObject *f = _PyThreadState_GET()->frame;
 
+    if (PySys_Audit("sys._getframe", "O", f) < 0) {
+        return NULL;
+    }
+
     while (depth > 0 && f != NULL) {
         f = f->f_back;
         --depth;
@@ -1642,8 +1950,11 @@ sys_getandroidapilevel_impl(PyObject *module)
 #endif   /* ANDROID_API_LEVEL */
 
 
+
 static PyMethodDef sys_methods[] = {
     /* Might as well keep this in alphabetic order */
+    SYS_ADDAUDITHOOK_METHODDEF
+    {"audit",           (PyCFunction)(void(*)(void))sys_audit, METH_FASTCALL, audit_doc },
     {"breakpointhook",  (PyCFunction)(void(*)(void))sys_breakpointhook,
      METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc},
     SYS_CALLSTATS_METHODDEF