From 4fee28aa42dbac7f08a6d2829546b7d8e848034d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 27 Jun 2019 11:07:16 -0700 Subject: [PATCH] bpo-37390: Add audit event table to documentations (GH-14406) Also updates some (unreleased) event names to be consistent with the others. (cherry picked from commit 44f91c388a6f4da9ed3300df32ca290b8aa104ea) Co-authored-by: Steve Dower --- Doc/c-api/code.rst | 2 +- Doc/c-api/sys.rst | 4 - Doc/library/array.rst | 2 +- Doc/library/audit_events.rst | 21 +++++ Doc/library/ctypes.rst | 6 +- Doc/library/debug.rst | 5 +- Doc/library/ensurepip.rst | 2 +- Doc/library/ftplib.rst | 6 +- Doc/library/functions.rst | 14 +-- Doc/library/glob.rst | 4 +- Doc/library/imaplib.rst | 4 +- Doc/library/io.rst | 2 +- Doc/library/mmap.rst | 4 +- Doc/library/nntplib.rst | 20 ++-- Doc/library/os.rst | 12 +-- Doc/library/pdb.rst | 2 +- Doc/library/pickle.rst | 2 +- Doc/library/poplib.rst | 20 ++-- Doc/library/shutil.rst | 6 +- Doc/library/smtplib.rst | 10 +- Doc/library/socket.rst | 30 +++--- Doc/library/sqlite3.rst | 2 +- Doc/library/subprocess.rst | 2 +- Doc/library/sys.rst | 15 +-- Doc/library/telnetlib.rst | 4 +- Doc/library/tempfile.rst | 10 +- Doc/library/urllib.request.rst | 2 +- Doc/library/webbrowser.rst | 2 +- Doc/tools/extensions/pyspecific.py | 142 ++++++++++++++++++++++++++--- Lib/ftplib.py | 4 +- Lib/imaplib.py | 4 +- Lib/nntplib.py | 6 +- Lib/poplib.py | 4 +- Lib/smtplib.py | 4 +- 34 files changed, 266 insertions(+), 113 deletions(-) create mode 100644 Doc/library/audit_events.rst diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 48428109e6f..7353df56e7d 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -45,7 +45,7 @@ bound into a function. The first parameter (*argcount*) now represents the total number of positional arguments, including positional-only. - .. audit-event:: code.__new__ "code filename name argcount posonlyargcount kwonlyargcount nlocals stacksize flags" + .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags c.PyCode_New .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 7d870a8d4e4..838a97cacfd 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -291,8 +291,6 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: int PySys_Audit(const char *event, const char *format, ...) - .. index:: single: audit events - Raises an auditing event with any active hooks. Returns zero for success and non-zero with an exception set on failure. @@ -311,8 +309,6 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) - .. index:: single: audit events - Adds to the collection of active auditing hooks. Returns zero for success and non-zero on failure. If the runtime has been initialized, also sets an error on failure. Hooks added through this API are called for all diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 1f95dd61b9f..59b94f10b43 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -83,7 +83,7 @@ The module defines the following type: to add initial items to the array. Otherwise, the iterable initializer is passed to the :meth:`extend` method. - .. audit-event:: array.__new__ "typecode initializer" + .. audit-event:: array.__new__ typecode,initializer array.array .. data:: typecodes diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst new file mode 100644 index 00000000000..c23b9c61832 --- /dev/null +++ b/Doc/library/audit_events.rst @@ -0,0 +1,21 @@ +.. _audit-events: + +.. index:: single: audit events + +Audit events table +================== + +This table contains all events raised by :func:`sys.audit` or +:c:func:`PySys_Audit` calls throughout the CPython runtime and the +standard library. + +See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for +information on handling these events. + +.. impl-detail:: + + This table is generated from the CPython documentation, and may not + represent events raised by other implementations. See your runtime + specific documentation for actual events raised. + +.. audit-event-table:: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c1218ad32ec..2c57cc7ec42 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1509,13 +1509,13 @@ object is available: :c:type:`int`, which is of course not always the truth, so you have to assign the correct :attr:`restype` attribute to use these functions. -.. audit-event:: ctypes.dlopen name +.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader Loading a library through any of these objects raises an :ref:`auditing event ` ``ctypes.dlopen`` with string argument ``name``, the name used to load the library. -.. audit-event:: ctypes.dlsym "library name" +.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader Accessing a function on a loaded library raises an auditing event ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` @@ -2043,7 +2043,7 @@ Data types This method returns a ctypes type instance using the memory specified by *address* which must be an integer. - .. audit-event:: ctypes.cdata address + .. audit-event:: ctypes.cdata address ctypes._CData.from_address This method, and others that indirectly call this method, raises an :ref:`auditing event ` ``ctypes.cdata`` with argument diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst index 88a2fa62a56..60223657a44 100644 --- a/Doc/library/debug.rst +++ b/Doc/library/debug.rst @@ -5,10 +5,13 @@ Debugging and Profiling These libraries help you with Python development: the debugger enables you to step through code, analyze stack frames and set breakpoints etc., and the profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. +allowing you to identify bottlenecks in your programs. Auditing events +provide visibility into runtime behaviors that would otherwise require +intrusive debugging or patching. .. toctree:: + audit_events.rst bdb.rst faulthandler.rst pdb.rst diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 3b6b01fb452..a2bb045e57e 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -119,7 +119,7 @@ Module API *verbosity* controls the level of output to :data:`sys.stdout` from the bootstrapping operation. - .. audit-event:: ensurepip.bootstrap root + .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap .. note:: diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 36abfbde3e3..e006d019b83 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -190,7 +190,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as its source address before connecting. - .. audit-event:: ftplib.FTP.connect "self host port" + .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect .. versionchanged:: 3.3 *source_address* parameter was added. @@ -225,7 +225,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. Send a simple command string to the server and return the response string. - .. audit-event:: ftplib.FTP.sendcmd "self cmd" + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd .. method:: FTP.voidcmd(cmd) @@ -234,7 +234,7 @@ followed by ``lines`` for the text version or ``binary`` for the binary version. nothing if a response code corresponding to success (codes in the range 200--299) is received. Raise :exc:`error_reply` otherwise. - .. audit-event:: ftplib.FTP.sendcmd "self cmd" + .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd .. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 637c82b6708..e146f5a95ac 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -128,7 +128,7 @@ are always available. They are listed here in alphabetical order. :func:`breakpoint` will automatically call that, allowing you to drop into the debugger of choice. - .. audit-event:: builtins.breakpoint "sys.breakpointhook" + .. audit-event:: builtins.breakpoint breakpointhook breakpoint .. versionadded:: 3.7 @@ -277,7 +277,7 @@ are always available. They are listed here in alphabetical order. If you want to parse Python code into its AST representation, see :func:`ast.parse`. - .. audit-event:: compile "source filename" + .. audit-event:: compile source,filename compile Raises an :ref:`auditing event ` ``compile`` with arguments ``source`` and ``filename``. This event may also be raised by implicit @@ -490,7 +490,7 @@ are always available. They are listed here in alphabetical order. See :func:`ast.literal_eval` for a function that can safely evaluate strings with expressions containing only literals. - .. audit-event:: exec code_object + .. audit-event:: exec code_object eval Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. @@ -525,7 +525,7 @@ are always available. They are listed here in alphabetical order. builtins are available to the executed code by inserting your own ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. - .. audit-event:: exec code_object + .. audit-event:: exec code_object exec Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. @@ -779,12 +779,12 @@ are always available. They are listed here in alphabetical order. If the :mod:`readline` module was loaded, then :func:`input` will use it to provide elaborate line editing and history features. - .. audit-event:: builtins.input prompt + .. audit-event:: builtins.input prompt input Raises an :ref:`auditing event ` ``builtins.input`` with argument ``prompt`` before reading input - .. audit-event:: builtins.input/result result + .. audit-event:: builtins.input/result result input Raises an auditing event ``builtins.input/result`` with the result after successfully reading input. @@ -1222,7 +1222,7 @@ are always available. They are listed here in alphabetical order. (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, and :mod:`shutil`. - .. audit-event:: open "file mode flags" + .. audit-event:: open file,mode,flags open The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 1f29fc562dd..96584437cef 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -52,7 +52,7 @@ For example, ``'[?]'`` matches the character ``'?'``. more directories and subdirectories. If the pattern is followed by an ``os.sep``, only directories and subdirectories match. - .. audit-event:: glob.glob "pathname recursive" + .. audit-event:: glob.glob pathname,recursive glob.glob .. note:: Using the "``**``" pattern in large directory trees may consume @@ -67,7 +67,7 @@ For example, ``'[?]'`` matches the character ``'?'``. Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. - .. audit-event:: glob.glob "pathname recursive" + .. audit-event:: glob.glob pathname,recursive glob.iglob .. function:: escape(pathname) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index c08c0a5591b..df63d820cfe 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -361,7 +361,7 @@ An :class:`IMAP4` instance has the following methods: :meth:`IMAP4.send`, and :meth:`IMAP4.shutdown` methods. You may override this method. - .. audit-event:: imaplib.IMAP4.open "self host port" + .. audit-event:: imaplib.open self,host,port imaplib.IMAP4.open .. method:: IMAP4.partial(message_num, message_part, start, length) @@ -432,7 +432,7 @@ An :class:`IMAP4` instance has the following methods: Sends ``data`` to the remote server. You may override this method. - .. audit-event:: imaplib.IMAP4.send "self data" + .. audit-event:: imaplib.send self,data imaplib.IMAP4.send .. method:: IMAP4.setacl(mailbox, who, what) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 28c5da96d90..fce9a747fd3 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -120,7 +120,7 @@ High-level Module Interface This is an alias for the builtin :func:`open` function. - .. audit-event:: open "path mode flags" + .. audit-event:: open path,mode,flags io.open This function raises an :ref:`auditing event ` ``open`` with arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags`` diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index c7a13abad88..12b14d69332 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -67,7 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length will be relative to the offset from the beginning of the file. *offset* defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. - .. audit-event:: mmap.__new__ "fileno length access offset" + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap .. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) :noindex: @@ -156,7 +156,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length mm.close() - .. audit-event:: mmap.__new__ "fileno length access offset" + .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap Memory-mapped file objects support the following methods: diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst index 297bbedc867..46f1c078355 100644 --- a/Doc/library/nntplib.rst +++ b/Doc/library/nntplib.rst @@ -79,11 +79,13 @@ The module itself defines the following classes: ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') >>> - .. audit-event:: nntplib.NNTP "self host port" + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP - All commands will raise an :ref:`auditing event ` - ``nntplib.NNTP.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. + .. audit-event:: nntplib.putline self,line nntplib.NNTP + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. versionchanged:: 3.2 *usenetrc* is now ``False`` by default. @@ -105,11 +107,13 @@ The module itself defines the following classes: STARTTLS as described below. However, some servers only support the former. - .. audit-event:: nntplib.NNTP "self host port" + .. audit-event:: nntplib.connect self,host,port nntplib.NNTP_SSL - All commands will raise an :ref:`auditing event ` - ``nntplib.NNTP.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. + .. audit-event:: nntplib.putline self,line nntplib.NNTP_SSL + + All commands will raise an :ref:`auditing event ` + ``nntplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. versionadded:: 3.2 diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 45ce643d88e..ee3c35c2a99 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -851,7 +851,7 @@ as internal buffering of data. most *length* bytes in size. As of Python 3.3, this is equivalent to ``os.truncate(fd, length)``. - .. audit-event:: os.truncate "fd length" + .. audit-event:: os.truncate fd,length os.ftruncate .. availability:: Unix, Windows. @@ -938,7 +938,7 @@ as internal buffering of data. This function can support :ref:`paths relative to directory descriptors ` with the *dir_fd* parameter. - .. audit-event:: open "path mode flags" + .. audit-event:: open path,mode,flags os.open .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. @@ -1806,7 +1806,7 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. - .. audit-event:: os.listdir path + .. audit-event:: os.listdir path os.listdir .. note:: To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. @@ -2185,7 +2185,7 @@ features: This function can also support :ref:`specifying a file descriptor `; the file descriptor must refer to a directory. - .. audit-event:: os.scandir path + .. audit-event:: os.scandir path os.scandir The :func:`scandir` iterator supports the :term:`context manager` protocol and has the following method: @@ -2793,7 +2793,7 @@ features: This function can support :ref:`specifying a file descriptor `. - .. audit-event:: os.truncate "path length" + .. audit-event:: os.truncate path,length os.truncate .. availability:: Unix, Windows. @@ -3799,7 +3799,7 @@ written in Python, such as a mail server's external command delivery program. to using this function. See the :ref:`subprocess-replacements` section in the :mod:`subprocess` documentation for some helpful recipes. - .. audit-event:: os.system command + .. audit-event:: os.system command os.system .. availability:: Unix, Windows. diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7ebad79130a..f26b6a8b553 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -181,7 +181,7 @@ access further features, you have to do this yourself: import pdb; pdb.Pdb(skip=['django.*']).set_trace() - .. audit-event:: pdb.Pdb + .. audit-event:: pdb.Pdb "" pdb.Pdb .. versionadded:: 3.1 The *skip* argument. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 6aa30492c70..e6025aeaf47 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -437,7 +437,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, how they can be loaded, potentially reducing security risks. Refer to :ref:`pickle-restrict` for details. - .. audit-event:: pickle.find_class "module name" + .. audit-event:: pickle.find_class module,name pickle.Unpickler.find_class .. class:: PickleBuffer(buffer) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index d227b53ff90..28b42fa60c1 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -39,11 +39,13 @@ The :mod:`poplib` module provides two classes: connection attempt (if not specified, the global default timeout setting will be used). - .. audit-event:: poplib.POP3 "self host port" + .. audit-event:: poplib.connect self,host,port poplib.POP3 - All commands will raise an :ref:`auditing event ` - ``poplib.POP3.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. + .. audit-event:: poplib.putline self,line poplib.POP3 + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) @@ -60,11 +62,13 @@ The :mod:`poplib` module provides two classes: point to PEM-formatted private key and certificate chain files, respectively, for the SSL connection. - .. audit-event:: poplib.POP3 "self host port" + .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL - All commands will raise an :ref:`auditing event ` - ``poplib.POP3.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. + .. audit-event:: poplib.putline self,line popplib.POP3_SSL + + All commands will raise an :ref:`auditing event ` + ``poplib.putline`` with arguments ``self`` and ``line``, + where ``line`` is the bytes about to be sent to the remote host. .. versionchanged:: 3.2 *context* parameter added. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 3cca9c84a27..eaeee8df81b 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -249,7 +249,7 @@ Directory and files operations   as arguments. By default, :func:`~shutil.copy2` is used, but any function   that supports the same signature (like :func:`~shutil.copy`) can be used. - .. audit-event:: shutil.copytree "src dst" + .. audit-event:: shutil.copytree src,dst shutil.copytree .. versionchanged:: 3.3 Copy metadata when *symlinks* is false. @@ -298,7 +298,7 @@ Directory and files operations *excinfo*, will be the exception information returned by :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. - .. audit-event:: shutil.rmtree path + .. audit-event:: shutil.rmtree path shutil.rmtree .. versionchanged:: 3.3 Added a symlink attack resistant version that is used automatically @@ -562,7 +562,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. The *verbose* argument is unused and deprecated. - .. audit-event:: shutil.make_archive "base_name format root_dir base_dir" + .. audit-event:: shutil.make_archive base_name,format,root_dir,base_dir shutil.make_archive .. versionchanged:: 3.8 The modern pax (POSIX.1-2001) format is now used instead of diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 8771f109380..6176c35a0e4 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -55,9 +55,11 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). (250, b'Ok') >>> - All commands will raise an :ref:`auditing event ` - ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, - where ``data`` is the bytes about to be sent to the remote host. + .. audit-event:: smtplib.send self,data smtplib.SMTP + + All commands will raise an :ref:`auditing event ` + ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, + where ``data`` is the bytes about to be sent to the remote host. .. versionchanged:: 3.3 Support for the :keyword:`with` statement was added. @@ -246,7 +248,7 @@ An :class:`SMTP` instance has the following methods: 2-tuple of the response code and message sent by the server in its connection response. - .. audit-event:: smtplib.SMTP.connect "self host port" + .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect .. method:: SMTP.helo(name='') diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index e0dbbb4c0d3..f1010fb35a4 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -526,7 +526,7 @@ The following functions all create :ref:`socket objects `. The newly created socket is :ref:`non-inheritable `. - .. audit-event:: socket.__new__ "self family type protocol" + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket .. versionchanged:: 3.3 The AF_CAN family was added. @@ -720,7 +720,7 @@ The :mod:`socket` module also offers various network-related services: :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` method. - .. audit-event:: socket.getaddrinfo "host port family type protocol" + .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo The following example fetches address information for a hypothetical TCP connection to ``example.org`` on port 80 (results may differ on your @@ -757,7 +757,7 @@ The :mod:`socket` module also offers various network-related services: interface. :func:`gethostbyname` does not support IPv6 name resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. - .. audit-event:: socket.gethostbyname hostname + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname .. function:: gethostbyname_ex(hostname) @@ -771,7 +771,7 @@ The :mod:`socket` module also offers various network-related services: resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. - .. audit-event:: socket.gethostbyname hostname + .. audit-event:: socket.gethostbyname hostname socket.gethostbyname_ex .. function:: gethostname() @@ -779,7 +779,7 @@ The :mod:`socket` module also offers various network-related services: Return a string containing the hostname of the machine where the Python interpreter is currently executing. - .. audit-event:: socket.gethostname + .. audit-event:: socket.gethostname "" socket.gethostname Note: :func:`gethostname` doesn't always return the fully qualified domain name; use :func:`getfqdn` for that. @@ -795,7 +795,7 @@ The :mod:`socket` module also offers various network-related services: domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports both IPv4 and IPv6. - .. audit-event:: socket.gethostbyaddr ip_address + .. audit-event:: socket.gethostbyaddr ip_address socket.gethostbyaddr .. function:: getnameinfo(sockaddr, flags) @@ -810,7 +810,7 @@ The :mod:`socket` module also offers various network-related services: For more information about *flags* you can consult :manpage:`getnameinfo(3)`. - .. audit-event:: socket.getnameinfo sockaddr + .. audit-event:: socket.getnameinfo sockaddr socket.getnameinfo .. function:: getprotobyname(protocolname) @@ -827,7 +827,7 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. - .. audit-event:: socket.getservbyname "servicename protocolname" + .. audit-event:: socket.getservbyname servicename,protocolname socket.getservbyname .. function:: getservbyport(port[, protocolname]) @@ -836,7 +836,7 @@ The :mod:`socket` module also offers various network-related services: service. The optional protocol name, if given, should be ``'tcp'`` or ``'udp'``, otherwise any protocol will match. - .. audit-event:: socket.getservbyport "port protocolname" + .. audit-event:: socket.getservbyport port,protocolname socket.getservbyport .. function:: ntohl(x) @@ -1021,7 +1021,7 @@ The :mod:`socket` module also offers various network-related services: Set the machine's hostname to *name*. This will raise an :exc:`OSError` if you don't have enough rights. - .. audit-event:: socket.sethostname name + .. audit-event:: socket.sethostname name socket.sethostname .. availability:: Unix. @@ -1107,7 +1107,7 @@ to sockets. Bind the socket to *address*. The socket must not already be bound. (The format of *address* depends on the address family --- see above.) - .. audit-event:: socket.bind "self address" + .. audit-event:: socket.bind self,address socket.socket.bind .. method:: socket.close() @@ -1145,7 +1145,7 @@ to sockets. :exc:`InterruptedError` exception if the connection is interrupted by a signal (or the exception raised by the signal handler). - .. audit-event:: socket.connect "self address" + .. audit-event:: socket.connect self,address socket.socket.connect .. versionchanged:: 3.5 The method now waits until the connection completes instead of raising an @@ -1163,7 +1163,7 @@ to sockets. :c:data:`errno` variable. This is useful to support, for example, asynchronous connects. - .. audit-event:: socket.connect "self address" + .. audit-event:: socket.connect self,address socket.socket.connect_ex .. method:: socket.detach() @@ -1505,7 +1505,7 @@ to sockets. bytes sent. (The format of *address* depends on the address family --- see above.) - .. audit-event:: socket.sendto "self address" + .. audit-event:: socket.sendto self,address socket.socket.sendto .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise @@ -1546,7 +1546,7 @@ to sockets. .. availability:: most Unix platforms, possibly others. - .. audit-event:: socket.sendmsg "self address" + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg .. versionadded:: 3.3 diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e0411fed2f9..67ea2b1d776 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -224,7 +224,7 @@ Module functions and constants More information about this feature, including a list of recognized options, can be found in the `SQLite URI documentation `_. - .. audit-event:: sqlite3.connect "database" + .. audit-event:: sqlite3.connect database sqlite3.connect .. versionchanged:: 3.4 Added the *uri* parameter. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index ad49d7f2017..af338795dae 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -585,7 +585,7 @@ functions. with Popen(["ifconfig"], stdout=PIPE) as proc: log.write(proc.stdout.read()) - .. audit-event:: subprocess.Popen "executable args cwd env" + .. audit-event:: subprocess.Popen executable,args,cwd,env subprocess.Popen Popen and the other functions in this module that use it raise an :ref:`auditing event ` ``subprocess.Popen`` with arguments diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index c073431c894..0b53ee00c99 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -86,6 +86,9 @@ always available. The native equivalent of this function is :c:func:`PySys_Audit`. Using the native function is preferred when possible. + See the :ref:`audit events table ` for all events raised by + ``CPython``. + .. versionadded:: 3.8 @@ -166,7 +169,7 @@ always available. This function should be used for internal and specialized purposes only. - .. audit-event:: sys._current_frames + .. audit-event:: sys._current_frames "" sys._current_frames .. function:: breakpointhook() @@ -675,7 +678,7 @@ always available. that is deeper than the call stack, :exc:`ValueError` is raised. The default for *depth* is zero, returning the frame at the top of the call stack. - .. audit-event:: sys._getframe + .. audit-event:: sys._getframe "" sys._getframe .. impl-detail:: @@ -1190,7 +1193,7 @@ always available. ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends on the event type. - .. audit-event:: sys.setprofile + .. audit-event:: sys.setprofile "" sys.setprofile The events have the following meaning: @@ -1312,7 +1315,7 @@ always available. For more information on code and frame objects, refer to :ref:`types`. - .. audit-event:: sys.settrace + .. audit-event:: sys.settrace "" sys.settrace .. impl-detail:: @@ -1334,9 +1337,9 @@ always available. first time. The *finalizer* will be called when an asynchronous generator is about to be garbage collected. - .. audit-event:: sys.set_asyncgen_hooks_firstiter + .. audit-event:: sys.set_asyncgen_hooks_firstiter "" sys.set_asyncgen_hooks - .. audit-event:: sys.set_asyncgen_hooks_finalizer + .. audit-event:: sys.set_asyncgen_hooks_finalizer "" sys.set_asyncgen_hooks Two auditing events are raised because the underlying API consists of two calls, each of which must raise its own event. diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index d238136567c..0262a9b0841 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -141,7 +141,7 @@ Telnet Objects Do not try to reopen an already connected instance. - .. audit-event:: telnetlib.Telnet.open "self host port" + .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open .. method:: Telnet.msg(msg, *args) @@ -178,7 +178,7 @@ Telnet Objects block if the connection is blocked. May raise :exc:`OSError` if the connection is closed. - .. audit-event:: telnetlib.Telnet.write "self buffer" + .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write .. versionchanged:: 3.3 This method used to raise :exc:`socket.error`, which is now an alias diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 98069f2442c..0793e43df01 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -62,7 +62,7 @@ The module defines the following user-callable items: The :py:data:`os.O_TMPFILE` flag is used if it is available and works (Linux-specific, requires Linux kernel 3.11 or later). - .. audit-event:: tempfile.mkstemp "full-path" + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile .. versionchanged:: 3.5 @@ -87,7 +87,7 @@ The module defines the following user-callable items: attribute is the underlying true file object. This file-like object can be used in a :keyword:`with` statement, just like a normal file. - .. audit-event:: tempfile.mkstemp "full-path" + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile .. versionchanged:: 3.8 Added *errors* parameter. @@ -134,7 +134,7 @@ The module defines the following user-callable items: The directory can be explicitly cleaned up by calling the :func:`cleanup` method. - .. audit-event:: tempfile.mkdtemp "full-path" + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory .. versionadded:: 3.2 @@ -183,7 +183,7 @@ The module defines the following user-callable items: file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. - .. audit-event:: tempfile.mkstemp "full-path" + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to @@ -206,7 +206,7 @@ The module defines the following user-callable items: :func:`mkdtemp` returns the absolute pathname of the new directory. - .. audit-event:: tempfile.mkdtemp "full-path" + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp .. versionchanged:: 3.5 *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 9f8869c87f6..3b750896669 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -95,7 +95,7 @@ The :mod:`urllib.request` module defines the following functions: parameter to ``urllib.urlopen``, can be obtained by using :class:`ProxyHandler` objects. - .. audit-event:: urllib.Request "fullurl data headers method" + .. audit-event:: urllib.Request fullurl,data,headers,method urllib.request.urlopen The default opener raises an :ref:`auditing event ` ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 85e932d5a22..b7bfb655a71 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -64,7 +64,7 @@ The following functions are defined: may work and start the operating system's associated program. However, this is neither supported nor portable. - .. audit-event:: webbrowser.open "url" + .. audit-event:: webbrowser.open url webbrowser.open .. function:: open_new(url) diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8bf7eb699c4..9b38f8a6899 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -23,7 +23,7 @@ from docutils import nodes, utils from sphinx import addnodes from sphinx.builders import Builder from sphinx.locale import translators -from sphinx.util import status_iterator +from sphinx.util import status_iterator, logging from sphinx.util.nodes import split_explicit_title from sphinx.writers.html import HTMLTranslator from sphinx.writers.text import TextWriter, TextTranslator @@ -157,7 +157,7 @@ class AuditEvent(Directive): has_content = True required_arguments = 1 - optional_arguments = 1 + optional_arguments = 2 final_argument_whitespace = True _label = [ @@ -166,21 +166,49 @@ class AuditEvent(Directive): "Raises an :ref:`auditing event ` {name} with arguments {args}.", ] + @property + def logger(self): + cls = type(self) + return logging.getLogger(cls.__module__ + "." + cls.__name__) + def run(self): + name = self.arguments[0] if len(self.arguments) >= 2 and self.arguments[1]: - args = [ - "``{}``".format(a.strip()) - for a in self.arguments[1].strip("'\"").split() - if a.strip() - ] + args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) + args = [a for a in args if a] else: args = [] label = translators['sphinx'].gettext(self._label[min(2, len(args))]) - text = label.format(name="``{}``".format(self.arguments[0]), - args=", ".join(args)) + text = label.format(name="``{}``".format(name), + args=", ".join("``{}``".format(a) for a in args if a)) - pnode = nodes.paragraph(text, classes=["audit-hook"]) + env = self.state.document.settings.env + if not hasattr(env, 'all_audit_events'): + env.all_audit_events = {} + + new_info = { + 'source': [], + 'args': args + } + info = env.all_audit_events.setdefault(name, new_info) + if info is not new_info: + if not self._do_args_match(info['args'], new_info['args']): + self.logger.warn( + "Mismatched arguments for audit-event {}: {!r} != {!r}" + .format(name, info['args'], new_info['args']) + ) + + if len(self.arguments) >= 3 and self.arguments[2]: + target = self.arguments[2] + ids = [] + else: + target = "audit_event_{}_{}".format(name, len(info['source'])) + target = re.sub(r'\W', '_', label) + ids = [target] + info['source'].append((env.docname, target)) + + pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) if self.content: self.state.nested_parse(self.content, self.content_offset, pnode) else: @@ -189,6 +217,37 @@ class AuditEvent(Directive): return [pnode] + # This list of sets are allowable synonyms for event argument names. + # If two names are in the same set, they are treated as equal for the + # purposes of warning. This won't help if number of arguments is + # different! + _SYNONYMS = [ + {"file", "path", "fd"}, + ] + + def _do_args_match(self, args1, args2): + if args1 == args2: + return True + if len(args1) != len(args2): + return False + for a1, a2 in zip(args1, args2): + if a1 == a2: + continue + if any(a1 in s and a2 in s for s in self._SYNONYMS): + continue + return False + return True + + +class audit_event_list(nodes.General, nodes.Element): + pass + + +class AuditEventListDirective(Directive): + + def run(self): + return [audit_event_list('')] + # Support for documenting decorators @@ -394,7 +453,7 @@ class PydocTopicsBuilder(Builder): 'building topics... ', length=len(pydoc_topic_labels)): if label not in self.env.domaindata['std']['labels']: - self.warn('label %r not in documentation' % label) + self.env.logger.warn('label %r not in documentation' % label) continue docname, labelid, sectname = self.env.domaindata['std']['labels'][label] doctree = self.env.get_and_resolve_doctree(docname, self) @@ -458,12 +517,72 @@ def parse_pdb_command(env, sig, signode): return fullname +def process_audit_events(app, doctree, fromdocname): + for node in doctree.traverse(audit_event_list): + break + else: + return + + env = app.builder.env + + table = nodes.table(cols=3) + group = nodes.tgroup( + '', + nodes.colspec(colwidth=30), + nodes.colspec(colwidth=55), + nodes.colspec(colwidth=15), + ) + head = nodes.thead() + body = nodes.tbody() + + table += group + group += head + group += body + + row = nodes.row() + row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) + row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) + head += row + + for name in sorted(getattr(env, "all_audit_events", ())): + audit_event = env.all_audit_events[name] + + row = nodes.row() + node = nodes.paragraph('', nodes.Text(name)) + row += nodes.entry('', node) + + node = nodes.paragraph() + for i, a in enumerate(audit_event['args']): + if i: + node += nodes.Text(", ") + node += nodes.literal(a, nodes.Text(a)) + row += nodes.entry('', node) + + node = nodes.paragraph() + for i, (doc, label) in enumerate(audit_event['source'], start=1): + if isinstance(label, str): + ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) + ref['refuri'] = "{}#{}".format( + app.builder.get_relative_uri(fromdocname, doc), + label, + ) + node += ref + row += nodes.entry('', node) + + body += row + + for node in doctree.traverse(audit_event_list): + node.replace_self(table) + + def setup(app): app.add_role('issue', issue_role) app.add_role('source', source_role) app.add_directive('impl-detail', ImplementationDetail) app.add_directive('availability', Availability) app.add_directive('audit-event', AuditEvent) + app.add_directive('audit-event-table', AuditEventListDirective) app.add_directive('deprecated-removed', DeprecatedRemoved) app.add_builder(PydocTopicsBuilder) app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) @@ -478,4 +597,5 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('doctree-resolved', process_audit_events) return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 27bf6daea61..58a46bca4a3 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -148,7 +148,7 @@ class FTP: self.timeout = timeout if source_address is not None: self.source_address = source_address - sys.audit("ftplib.FTP.connect", self, self.host, self.port) + sys.audit("ftplib.connect", self, self.host, self.port) self.sock = socket.create_connection((self.host, self.port), self.timeout, source_address=self.source_address) self.af = self.sock.family @@ -189,7 +189,7 @@ class FTP: def putline(self, line): if '\r' in line or '\n' in line: raise ValueError('an illegal newline character should not be contained') - sys.audit("ftplib.FTP.sendcmd", self, line) + sys.audit("ftplib.sendcmd", self, line) line = line + CRLF if self.debugging > 1: print('*put*', self.sanitize(line)) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index face45ba5fa..822d9d6fdd8 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -289,7 +289,7 @@ class IMAP4: # (which is used by socket.create_connection()) expects None # as a default value for host. host = None if not self.host else self.host - sys.audit("imaplib.IMAP4.open", self, self.host, self.port) + sys.audit("imaplib.open", self, self.host, self.port) return socket.create_connection((host, self.port)) def open(self, host = '', port = IMAP4_PORT): @@ -319,7 +319,7 @@ class IMAP4: def send(self, data): """Send data to remote.""" - sys.audit("imaplib.IMAP4.send", self, data) + sys.audit("imaplib.send", self, data) self.sock.sendall(data) diff --git a/Lib/nntplib.py b/Lib/nntplib.py index e7bbc853c07..1b7e83af01a 100644 --- a/Lib/nntplib.py +++ b/Lib/nntplib.py @@ -414,7 +414,7 @@ class _NNTPBase: def _putline(self, line): """Internal: send one line to the server, appending CRLF. The `line` must be a bytes-like object.""" - sys.audit("nntplib.NNTP.putline", self, line) + sys.audit("nntplib.putline", self, line) line = line + _CRLF if self.debugging > 1: print('*put*', repr(line)) self.file.write(line) @@ -1042,7 +1042,7 @@ class NNTP(_NNTPBase): """ self.host = host self.port = port - sys.audit("nntplib.NNTP", self, host, port) + sys.audit("nntplib.connect", self, host, port) self.sock = socket.create_connection((host, port), timeout) file = None try: @@ -1074,7 +1074,7 @@ if _have_ssl: """This works identically to NNTP.__init__, except for the change in default port and the `ssl_context` argument for SSL connections. """ - sys.audit("nntplib.NNTP", self, host, port) + sys.audit("nntplib.connect", self, host, port) self.sock = socket.create_connection((host, port), timeout) file = None try: diff --git a/Lib/poplib.py b/Lib/poplib.py index ced0a2d53c5..e3bd2ab1ebc 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -100,7 +100,7 @@ class POP3: self.host = host self.port = port self._tls_established = False - sys.audit("poplib.POP3", self, host, port) + sys.audit("poplib.connect", self, host, port) self.sock = self._create_socket(timeout) self.file = self.sock.makefile('rb') self._debugging = 0 @@ -111,7 +111,7 @@ class POP3: def _putline(self, line): if self._debugging > 1: print('*put*', repr(line)) - sys.audit("poplib.POP3.putline", self, line) + sys.audit("poplib.putline", self, line) self.sock.sendall(line + CRLF) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 01f3d4386f5..a634f7ae733 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -335,7 +335,7 @@ class SMTP: port = self.default_port if self.debuglevel > 0: self._print_debug('connect:', (host, port)) - sys.audit("smtplib.SMTP.connect", self, host, port) + sys.audit("smtplib.connect", self, host, port) self.sock = self._get_socket(host, port, self.timeout) self.file = None (code, msg) = self.getreply() @@ -353,7 +353,7 @@ class SMTP: # should not be used, but 'data' needs to convert the string to # binary itself anyway, so that's not a problem. s = s.encode(self.command_encoding) - sys.audit("smtplib.SMTP.send", self, s) + sys.audit("smtplib.send", self, s) try: self.sock.sendall(s) except OSError: