Merge branch 'master' into fix-issue-38490

.azure-pipelines/windows-release/stage-pack-msix.yml

@@ -120,10 +120,11 @@ jobs:
       artifactName: unsigned_msix
       downloadPath: $(Build.BinariesDirectory)
 
+  # MSIX must be signed and timestamped simultaneously
   - powershell: |
       $failed = $true
       foreach ($retry in 1..3) {
-          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix)
+          signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix)
           if ($?) {
               $failed = $false
               break

.azure-pipelines/windows-release/stage-pack-nuget.yml

@@ -4,7 +4,7 @@ jobs:
   condition: and(succeeded(), eq(variables['DoNuget'], 'true'))
 
   pool:
-    vmImage: windows-2019
+    name: 'Windows Release'
 
   workspace:
     clean: all
@@ -36,6 +36,14 @@ jobs:
       nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive
     displayName: 'Create nuget package'
 
+  - powershell: |
+      gci *.nupkg | %{
+        nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite
+      }
+    displayName: 'Sign nuget package'
+    workingDirectory: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), variables['SigningCertificate'])
+
   - task: PublishBuildArtifacts@1
     displayName: 'Publish Artifact: nuget'
     inputs:

.azure-pipelines/windows-release/stage-sign.yml

@@ -57,7 +57,7 @@ jobs:
       $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }})
       $failed = $true
       foreach ($retry in 1..10) {
-          signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files
+          signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files
           if ($?) {
               $failed = $false
               break

Doc/Makefile

@@ -215,12 +215,12 @@ serve:
 
 # for development releases: always build
 autobuild-dev:
-	make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1'
+	make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
 	-make suspicious
 
 # for quick rebuilds (HTML only)
 autobuild-dev-html:
-	make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1'
+	make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
 
 # for stable releases: only build if not in pre-release stage (alpha, beta)
 # release candidate downloads are okay, since the stable tree can be in that stage

Doc/c-api/arg.rst

@@ -482,7 +482,8 @@ API Functions
    *min* and no more than *max*; *min* and *max* may be equal.  Additional
    arguments must be passed to the function, each of which should be a pointer to a
    :c:type:`PyObject*` variable; these will be filled in with the values from
-   *args*; they will contain borrowed references.  The variables which correspond
+   *args*; they will contain :term:`borrowed references <borrowed reference>`.
+   The variables which correspond
    to optional parameters not given by *args* will not be filled in; these should
    be initialized by the caller. This function returns true on success and false if
    *args* is not a tuple or contains the wrong number of elements; an exception

Doc/c-api/exceptions.rst

@@ -182,8 +182,8 @@ For convenience, some of these functions will always return a
 .. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename)
 
    Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename
-   is given as a C string.  *filename* is decoded from the filesystem encoding
-   (:func:`os.fsdecode`).
+   is given as a C string.  *filename* is decoded from the :term:`filesystem
+   encoding and error handler`.
 
 
 .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr)
@@ -266,7 +266,7 @@ For convenience, some of these functions will always return a
 .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset)
 
    Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string
-   decoded from the filesystem encoding (:func:`os.fsdecode`).
+   decoded from the :term:`filesystem encoding and error handler`.
 
    .. versionadded:: 3.2
 
@@ -343,7 +343,7 @@ an error value).
 
    Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and
    *module* are UTF-8 encoded strings, and *filename* is decoded from the
-   filesystem encoding (:func:`os.fsdecode`).
+   :term:`filesystem encoding and error handler`.
 
 
 .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...)

Doc/c-api/init.rst

@@ -151,8 +151,9 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
 
 .. c:var:: int Py_LegacyWindowsFSEncodingFlag
 
-   If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8
-   encoding for the filesystem encoding.
+   If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error
+   handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler,
+   for the :term:`filesystem encoding and error handler`.
 
    Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment
    variable is set to a non-empty string.
@@ -1076,7 +1077,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
 
    Get the current frame of the Python thread state *tstate*.
 
-   Return a strong reference. Return ``NULL`` if no frame is currently
+   Return a :term:`strong reference`. Return ``NULL`` if no frame is currently
    executing.
 
    See also :c:func:`PyEval_GetFrame`.

Doc/c-api/intro.rst

@@ -326,7 +326,7 @@ when it's no longer needed---or passing on this responsibility (usually to its
 caller). When a function passes ownership of a reference on to its caller, the
 caller is said to receive a *new* reference.  When no ownership is transferred,
 the caller is said to *borrow* the reference. Nothing needs to be done for a
-borrowed reference.
+:term:`borrowed reference`.
 
 Conversely, when a calling function passes in a reference to an  object, there
 are two possibilities: the function *steals* a  reference to the object, or it

Doc/c-api/module.rst

@@ -264,7 +264,7 @@ of the following two module creation functions:
       instead; only use this if you are sure you need it.
 
 Before it is returned from in the initialization function, the resulting module
-object is typically populated using functions like :c:func:`PyModule_AddObject`.
+object is typically populated using functions like :c:func:`PyModule_AddObjectRef`.
 
 .. _multi-phase-initialization:
 
@@ -437,26 +437,102 @@ a function called from a module execution slot (if using multi-phase
 initialization), can use the following functions to help initialize the module
 state:
 
+.. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value)
+
+   Add an object to *module* as *name*.  This is a convenience function which
+   can be used from the module's initialization function.
+
+   On success, return ``0``. On error, raise an exception and return ``-1``.
+
+   Return ``NULL`` if *value* is ``NULL``. It must be called with an exception
+   raised in this case.
+
+   Example usage::
+
+       static int
+       add_spam(PyObject *module, int value)
+       {
+           PyObject *obj = PyLong_FromLong(value);
+           if (obj == NULL) {
+               return -1;
+           }
+           int res = PyModule_AddObjectRef(module, "spam", obj);
+           Py_DECREF(obj);
+           return res;
+        }
+
+   The example can also be written without checking explicitly if *obj* is
+   ``NULL``::
+
+       static int
+       add_spam(PyObject *module, int value)
+       {
+           PyObject *obj = PyLong_FromLong(value);
+           int res = PyModule_AddObjectRef(module, "spam", obj);
+           Py_XDECREF(obj);
+           return res;
+        }
+
+   Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in
+   this case, since *obj* can be ``NULL``.
+
+   .. versionadded:: 3.10
+
+
 .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value)
 
-   Add an object to *module* as *name*.  This is a convenience function which can
-   be used from the module's initialization function.  This steals a reference to
-   *value* on success. Return ``-1`` on error, ``0`` on success.
+   Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to
+   *value* on success (if it returns ``0``).
+
+   The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is
+   easy to introduce reference leaks by misusing the
+   :c:func:`PyModule_AddObject` function.
 
    .. note::
 
-      Unlike other functions that steal references, ``PyModule_AddObject()`` only
-      decrements the reference count of *value* **on success**.
+      Unlike other functions that steal references, ``PyModule_AddObject()``
+      only decrements the reference count of *value* **on success**.
 
       This means that its return value must be checked, and calling code must
-      :c:func:`Py_DECREF` *value* manually on error. Example usage::
+      :c:func:`Py_DECREF` *value* manually on error.
+
+   Example usage::
+
+      static int
+      add_spam(PyObject *module, int value)
+      {
+          PyObject *obj = PyLong_FromLong(value);
+          if (obj == NULL) {
+              return -1;
+          }
+          if (PyModule_AddObject(module, "spam", obj) < 0) {
+              Py_DECREF(obj);
+              return -1;
+          }
+          // PyModule_AddObject() stole a reference to obj:
+          // Py_DECREF(obj) is not needed here
+          return 0;
+      }
+
+   The example can also be written without checking explicitly if *obj* is
+   ``NULL``::
+
+      static int
+      add_spam(PyObject *module, int value)
+      {
+          PyObject *obj = PyLong_FromLong(value);
+          if (PyModule_AddObject(module, "spam", obj) < 0) {
+              Py_XDECREF(obj);
+              return -1;
+          }
+          // PyModule_AddObject() stole a reference to obj:
+          // Py_DECREF(obj) is not needed here
+          return 0;
+      }
+
+   Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in
+   this case, since *obj* can be ``NULL``.
 
-         Py_INCREF(spam);
-         if (PyModule_AddObject(module, "spam", spam) < 0) {
-             Py_DECREF(module);
-             Py_DECREF(spam);
-             return NULL;
-         }
 
 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value)
 

Doc/c-api/refcounting.rst

@@ -13,8 +13,14 @@ objects.
 
 .. c:function:: void Py_INCREF(PyObject *o)
 
-   Increment the reference count for object *o*.  The object must not be ``NULL``; if
-   you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`.
+   Increment the reference count for object *o*.
+
+   This function is usually used to convert a :term:`borrowed reference` to a
+   :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be
+   used to create a new :term:`strong reference`.
+
+   The object must not be ``NULL``; if you aren't sure that it isn't
+   ``NULL``, use :c:func:`Py_XINCREF`.
 
 
 .. c:function:: void Py_XINCREF(PyObject *o)
@@ -22,13 +28,55 @@ objects.
    Increment the reference count for object *o*.  The object may be ``NULL``, in
    which case the macro has no effect.
 
+   See also :c:func:`Py_XNewRef`.
+
+
+.. c:function:: PyObject* Py_NewRef(PyObject *o)
+
+   Create a new :term:`strong reference` to an object: increment the reference
+   count of the object *o* and return the object *o*.
+
+   When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF`
+   should be called on it to decrement the object reference count.
+
+   The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be
+   ``NULL``.
+
+   For example::
+
+       Py_INCREF(obj);
+       self->attr = obj;
+
+   can be written as::
+
+       self->attr = Py_NewRef(obj);
+
+   See also :c:func:`Py_INCREF`.
+
+   .. versionadded:: 3.10
+
+
+.. c:function:: PyObject* Py_XNewRef(PyObject *o)
+
+   Similar to :c:func:`Py_NewRef`, but the object *o* can be NULL.
+
+   If the object *o* is ``NULL``, the function just returns ``NULL``.
+
+   .. versionadded:: 3.10
+
 
 .. c:function:: void Py_DECREF(PyObject *o)
 
-   Decrement the reference count for object *o*.  The object must not be ``NULL``; if
-   you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`.  If the reference
-   count reaches zero, the object's type's deallocation function (which must not be
-   ``NULL``) is invoked.
+   Decrement the reference count for object *o*.
+
+   If the reference count reaches zero, the object's type's deallocation
+   function (which must not be ``NULL``) is invoked.
+
+   This function is usually used to delete a :term:`strong reference` before
+   exiting its scope.
+
+   The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``,
+   use :c:func:`Py_XDECREF`.
 
    .. warning::
 

Doc/c-api/reflection.rst

@@ -35,7 +35,7 @@ Reflection
 
    Get the *frame* next outer frame.
 
-   Return a strong reference, or ``NULL`` if *frame* has no outer frame.
+   Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame.
 
    *frame* must not be ``NULL``.
 
@@ -46,7 +46,7 @@ Reflection
 
    Get the *frame* code.
 
-   Return a strong reference.
+   Return a :term:`strong reference`.
 
    *frame* must not be ``NULL``. The result (frame code) cannot be ``NULL``.
 

Doc/c-api/structures.rst

@@ -66,7 +66,7 @@ the definition of all other Python objects.
 
    Get the type of the Python object *o*.
 
-   Return a borrowed reference.
+   Return a :term:`borrowed reference`.
 
    .. versionchanged:: 3.10
       :c:func:`Py_TYPE()` is changed to the inline static function.
@@ -263,10 +263,12 @@ There are these calling conventions:
    of :c:type:`PyObject*` values indicating the arguments and the third
    parameter is the number of arguments (the length of the array).
 
-   This is not part of the :ref:`limited API <stable>`.
-
    .. versionadded:: 3.7
 
+   .. versionchanged:: 3.10
+
+      ``METH_FASTCALL`` is now part of the stable ABI.
+
 
 .. data:: METH_FASTCALL | METH_KEYWORDS
 

Doc/c-api/sys.rst

@@ -118,22 +118,21 @@ Operating System Utilities
 
 .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size)
 
-   Decode a byte string from the locale encoding with the :ref:`surrogateescape
-   error handler <surrogateescape>`: undecodable bytes are decoded as
-   characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a
-   surrogate character, escape the bytes using the surrogateescape error
-   handler instead of decoding them.
+   .. warning::
+      This function should not be called directly: use the :c:type:`PyConfig`
+      API with the :c:func:`PyConfig_SetBytesString` function which ensures
+      that :ref:`Python is preinitialized <c-preinit>`.
 
-   Encoding, highest priority to lowest priority:
+      This function must not be called before :ref:`Python is preinitialized
+      <c-preinit>` and so that the LC_CTYPE locale is properly configured: see
+      the :c:func:`Py_PreInitialize` function.
 
-   * ``UTF-8`` on macOS, Android, and VxWorks;
-   * ``UTF-8`` on Windows if :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero;
-   * ``UTF-8`` if the Python UTF-8 mode is enabled;
-   * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``,
-     ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias),
-     and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the
-     ``ISO-8859-1`` encoding.
-   * the current locale encoding.
+   Decode a byte string from the :term:`filesystem encoding and error handler`.
+   If the error handler is :ref:`surrogateescape error handler
+   <surrogateescape>`, undecodable bytes are decoded as characters in range
+   U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate
+   character, the bytes are escaped using the surrogateescape error handler
+   instead of decoding them.
 
    Return a pointer to a newly allocated wide character string, use
    :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write
@@ -143,6 +142,10 @@ Operating System Utilities
    not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to
    ``(size_t)-2`` on decoding error.
 
+   The :term:`filesystem encoding and error handler` are selected by
+   :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and
+   :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
+
    Decoding errors should never happen, unless there is a bug in the C
    library.
 
@@ -157,7 +160,8 @@ Operating System Utilities
    .. versionadded:: 3.5
 
    .. versionchanged:: 3.7
-      The function now uses the UTF-8 encoding in the UTF-8 mode.
+      The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode
+      <utf8-mode>`.
 
    .. versionchanged:: 3.8
       The function now uses the UTF-8 encoding on Windows if
@@ -166,22 +170,10 @@ Operating System Utilities
 
 .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
 
-   Encode a wide character string to the locale encoding with the
-   :ref:`surrogateescape error handler <surrogateescape>`: surrogate characters
-   in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF.
-
-   Encoding, highest priority to lowest priority:
-
-   * ``UTF-8`` on macOS, Android, and VxWorks;
-   * ``UTF-8`` on Windows if :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero;
-   * ``UTF-8`` if the Python UTF-8 mode is enabled;
-   * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``,
-     ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias),
-     and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the
-     ``ISO-8859-1`` encoding.
-   * the current locale encoding.
-
-   The function uses the UTF-8 encoding in the Python UTF-8 mode.
+   Encode a wide character string to the :term:`filesystem encoding and error
+   handler`. If the error handler is :ref:`surrogateescape error handler
+   <surrogateescape>`, surrogate characters in the range U+DC80..U+DCFF are
+   converted to bytes 0x80..0xFF.
 
    Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free`
    to free the memory. Return ``NULL`` on encoding error or memory allocation
@@ -190,9 +182,18 @@ Operating System Utilities
    If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on
    success,  or set to the index of the invalid character on encoding error.
 
+   The :term:`filesystem encoding and error handler` are selected by
+   :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and
+   :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
+
    Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back
    to a wide character string.
 
+   .. warning::
+      This function must not be called before :ref:`Python is preinitialized
+      <c-preinit>` and so that the LC_CTYPE locale is properly configured: see
+      the :c:func:`Py_PreInitialize` function.
+
    .. seealso::
 
       The :c:func:`PyUnicode_EncodeFSDefault` and
@@ -201,7 +202,8 @@ Operating System Utilities
    .. versionadded:: 3.5
 
    .. versionchanged:: 3.7
-      The function now uses the UTF-8 encoding in the UTF-8 mode.
+      The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode
+      <utf8-mode>`.
 
    .. versionchanged:: 3.8
       The function now uses the UTF-8 encoding on Windows if

Doc/c-api/type.rst

@@ -169,6 +169,10 @@ The following functions and structs are used to create
 
    .. versionadded:: 3.9
 
+   .. versionchanged:: 3.10
+
+      The function now accepts NULL ``tp_doc`` slot.
+
 .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
 
    Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``.
@@ -259,5 +263,3 @@ The following functions and structs are used to create
 
       The desired value of the slot. In most cases, this is a pointer
       to a function.
-
-      May not be ``NULL``.

Doc/c-api/typeobj.rst

@@ -1213,8 +1213,9 @@ and :c:type:`PyType_Type` effectively act as defaults.)
    :func:`~gc.get_referents` function will include it.
 
    .. warning::
-       When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members
-       that the instance *owns* (by having strong references to them) must be
+       When implementing :c:member:`~PyTypeObject.tp_traverse`, only the
+       members that the instance *owns* (by having :term:`strong references
+       <strong reference>` to them) must be
        visited. For instance, if an object supports weak references via the
        :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting
        the linked list (what *tp_weaklist* points to) must **not** be

Doc/c-api/unicode.rst

@@ -783,7 +783,7 @@ system.
    :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
    Python startup).
 
-   This function ignores the Python UTF-8 mode.
+   This function ignores the :ref:`Python UTF-8 Mode <utf8-mode>`.
 
    .. seealso::
 
@@ -819,7 +819,7 @@ system.
    :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at
    Python startup).
 
-   This function ignores the Python UTF-8 mode.
+   This function ignores the :ref:`Python UTF-8 Mode <utf8-mode>`.
 
    .. seealso::
 
@@ -878,8 +878,7 @@ conversion function:
 
 .. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size)
 
-   Decode a string using :c:data:`Py_FileSystemDefaultEncoding` and the
-   :c:data:`Py_FileSystemDefaultEncodeErrors` error handler.
+   Decode a string from the :term:`filesystem encoding and error handler`.
 
    If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the
    locale encoding.
@@ -899,8 +898,8 @@ conversion function:
 
 .. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s)
 
-   Decode a null-terminated string using :c:data:`Py_FileSystemDefaultEncoding`
-   and the :c:data:`Py_FileSystemDefaultEncodeErrors` error handler.
+   Decode a null-terminated string from the :term:`filesystem encoding and
+   error handler`.
 
    If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the
    locale encoding.

Doc/c-api/veryhigh.rst

@@ -112,9 +112,9 @@ the same library that the Python runtime is using.
 
    Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read
    from *fp* instead of an in-memory string. *filename* should be the name of
-   the file, it is decoded from the filesystem encoding
-   (:func:`sys.getfilesystemencoding`).  If *closeit* is true, the file is
-   closed before PyRun_SimpleFileExFlags returns.
+   the file, it is decoded from :term:`filesystem encoding and error handler`.
+   If *closeit* is true, the file is closed before
+   ``PyRun_SimpleFileExFlags()`` returns.
 
    .. note::
       On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``).
@@ -132,7 +132,7 @@ the same library that the Python runtime is using.
    Read and execute a single statement from a file associated with an
    interactive device according to the *flags* argument.  The user will be
    prompted using ``sys.ps1`` and ``sys.ps2``.  *filename* is decoded from the
-   filesystem encoding (:func:`sys.getfilesystemencoding`).
+   :term:`filesystem encoding and error handler`.
 
    Returns ``0`` when the input was
    executed successfully, ``-1`` if there was an exception, or an error code
@@ -151,9 +151,8 @@ the same library that the Python runtime is using.
 
    Read and execute statements from a file associated with an interactive device
    until EOF is reached.  The user will be prompted using ``sys.ps1`` and
-   ``sys.ps2``.  *filename* is decoded from the filesystem encoding
-   (:func:`sys.getfilesystemencoding`).  Returns ``0`` at EOF or a negative
-   number upon failure.
+   ``sys.ps2``.  *filename* is decoded from the :term:`filesystem encoding and
+   error handler`.  Returns ``0`` at EOF or a negative number upon failure.
 
 
 .. c:var:: int (*PyOS_InputHook)(void)
@@ -206,8 +205,8 @@ the same library that the Python runtime is using.
    Parse Python source code from *str* using the start token *start* according to
    the *flags* argument.  The result can be used to create a code object which can
    be evaluated efficiently. This is useful if a code fragment must be evaluated
-   many times. *filename* is decoded from the filesystem encoding
-   (:func:`sys.getfilesystemencoding`).
+   many times. *filename* is decoded from the :term:`filesystem encoding and
+   error handler`.
 
 
 .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
@@ -262,7 +261,7 @@ the same library that the Python runtime is using.
 
    Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from
    *fp* instead of an in-memory string. *filename* should be the name of the file,
-   it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`).
+   it is decoded from the :term:`filesystem encoding and error handler`.
    If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags`
    returns.
 
@@ -301,7 +300,7 @@ the same library that the Python runtime is using.
 .. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize)
 
    Like :c:func:`Py_CompileStringObject`, but *filename* is a byte string
-   decoded from the filesystem encoding (:func:`os.fsdecode`).
+   decoded from the :term:`filesystem encoding and error handler`.
 
    .. versionadded:: 3.2
 

Doc/c-api/weakref.rst

@@ -57,10 +57,10 @@ as much as it can.
 
    .. note::
 
-      This function returns a **borrowed reference** to the referenced object.
+      This function returns a :term:`borrowed reference` to the referenced object.
       This means that you should always call :c:func:`Py_INCREF` on the object
-      except if you know that it cannot be destroyed while you are still
-      using it.
+      except it cannot be destroyed before the last usage of the borrowed
+      reference.
 
 
 .. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref)

Doc/data/refcounts.dat

@@ -3007,6 +3007,9 @@ Py_GetVersion:const char*:::
 Py_INCREF:void:::
 Py_INCREF:PyObject*:o:+1:
 
+Py_NewRef:void:::
+Py_NewRef:PyObject*:o:+1:
+
 Py_Initialize:void:::
 
 Py_IsInitialized:int:::
@@ -3028,6 +3031,9 @@ Py_XDECREF:PyObject*:o:-1:if o is not NULL
 Py_XINCREF:void:::
 Py_XINCREF:PyObject*:o:+1:if o is not NULL
 
+Py_XNewRef:void:::
+Py_XNewRef:PyObject*:o:+1:if o is not NULL
+
 _PyImport_Fini:void:::
 
 _PyObject_New:PyObject*::+1:

Doc/glossary.rst

@@ -158,6 +158,18 @@ Glossary
       See also :term:`text file` for a file object able to read and write
       :class:`str` objects.
 
+   borrowed reference
+      In the Python's C API, a borrowed reference is a reference to an object.
+      It does not modify the object reference count. It becomes a dangling
+      pointer if the object is destroyed. For example, a garbage collection can
+      remove the last :term:`strong reference` to the object and so destroy it.
+
+      Calling :c:func:`Py_INCREF` on the :term:`borrowed reference` is
+      recommended to convert it to a :term:`strong reference` in-place, except
+      if the object cannot be destroyed before the last usage of the borrowed
+      reference. The :c:func:`Py_NewRef` function can be used to create a new
+      :term:`strong reference`.
+
    bytes-like object
       An object that supports the :ref:`bufferobjects` and can
       export a C-:term:`contiguous` buffer. This includes all :class:`bytes`,
@@ -386,6 +398,25 @@ Glossary
    file-like object
       A synonym for :term:`file object`.
 
+   filesystem encoding and error handler
+      Encoding and error handler used by Python to decode bytes from the
+      operating system and encode Unicode to the operating system.
+
+      The filesystem encoding must guarantee to successfully decode all bytes
+      below 128. If the file system encoding fails to provide this guarantee,
+      API functions can raise :exc:`UnicodeError`.
+
+      The :func:`sys.getfilesystemencoding` and
+      :func:`sys.getfilesystemencodeerrors` functions can be used to get the
+      filesystem encoding and error handler.
+
+      The :term:`filesystem encoding and error handler` are configured at
+      Python startup by the :c:func:`PyConfig_Read` function: see
+      :c:member:`~PyConfig.filesystem_encoding` and
+      :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
+
+      See also the :term:`locale encoding`.
+
    finder
       An object that tries to find the :term:`loader` for a module that is
       being imported.
@@ -673,6 +704,18 @@ Glossary
       thread removes *key* from *mapping* after the test, but before the lookup.
       This issue can be solved with locks or by using the EAFP approach.
 
+   locale encoding
+      On Unix, it is the encoding of the LC_CTYPE locale. It can be set with
+      ``locale.setlocale(locale.LC_CTYPE, new_locale)``.
+
+      On Windows, it is is the ANSI code page (ex: ``cp1252``).
+
+      ``locale.getpreferredencoding(False)`` can be used to get the locale
+      encoding.
+
+      Python uses the :term:`filesystem encoding and error handler` to convert
+      between Unicode filenames and bytes filenames.
+
    list
       A built-in Python :term:`sequence`.  Despite its name it is more akin
       to an array in other languages than to a linked list since access to
@@ -1069,6 +1112,18 @@ Glossary
       an :term:`expression` or one of several constructs with a keyword, such
       as :keyword:`if`, :keyword:`while` or :keyword:`for`.
 
+   strong reference
+      In the Python's C API, a strong reference is a reference to an object
+      which increments object reference count when it is created and
+      decrements the object reference count when it is deleted.
+
+      The :c:func:`Py_NewRef` function can be used to create a strong reference
+      to an object. Usually, the :c:func:`Py_DECREF` function must be called on
+      the strong reference before exiting the scope of the strong reference, to
+      avoid leaking one reference.
+
+      See also :term:`borrowed reference`.
+
    text encoding
       A codec which encodes Unicode strings to bytes.
 

Doc/howto/descriptor.rst

@@ -112,7 +112,7 @@ different, updated answers each time::
 Besides showing how descriptors can run computations, this example also
 reveals the purpose of the parameters to :meth:`__get__`.  The *self*
 parameter is *size*, an instance of *DirectorySize*.  The *obj* parameter is
-either *g* or *s*, an instance of *Directory*.  It is *obj* parameter that
+either *g* or *s*, an instance of *Directory*.  It is the *obj* parameter that
 lets the :meth:`__get__` method learn the target directory.  The *objtype*
 parameter is the class *Directory*.
 
@@ -183,7 +183,7 @@ logged, but that the regular attribute *name* is not logged::
     INFO:root:Accessing 'age' giving 40
     40
 
-One major issue with this example is the private name *_age* is hardwired in
+One major issue with this example is that the private name *_age* is hardwired in
 the *LoggedAgeAccess* class.  That means that each instance can only have one
 logged attribute and that its name is unchangeable.  In the next example,
 we'll fix that problem.
@@ -192,7 +192,7 @@ we'll fix that problem.
 Customized names
 ----------------
 
-When a class uses descriptors, it can inform each descriptor about what
+When a class uses descriptors, it can inform each descriptor about which
 variable name was used.
 
 In this example, the :class:`Person` class has two descriptor instances,
@@ -233,7 +233,7 @@ be recorded, giving each descriptor its own *public_name* and *private_name*::
 
 An interactive session shows that the :class:`Person` class has called
 :meth:`__set_name__` so that the field names would be recorded.  Here
-we call :func:`vars` to lookup the descriptor without triggering it::
+we call :func:`vars` to look up the descriptor without triggering it::
 
     >>> vars(vars(Person)['name'])
     {'public_name': 'name', 'private_name': '_name'}
@@ -614,8 +614,8 @@ Sometimes it is desirable for a descriptor to know what class variable name it
 was assigned to.  When a new class is created, the :class:`type` metaclass
 scans the dictionary of the new class.  If any of the entries are descriptors
 and if they define :meth:`__set_name__`, that method is called with two
-arguments.  The *owner* is the class where the descriptor is used, the *name*
-is class variable the descriptor was assigned to.
+arguments.  The *owner* is the class where the descriptor is used, and the
+*name* is the class variable the descriptor was assigned to.
 
 The implementation details are in :c:func:`type_new()` and
 :c:func:`set_names()` in :source:`Objects/typeobject.c`.
@@ -703,7 +703,7 @@ Properties
 ----------
 
 Calling :func:`property` is a succinct way of building a data descriptor that
-triggers function calls upon access to an attribute.  Its signature is::
+triggers a function call upon access to an attribute.  Its signature is::
 
     property(fget=None, fset=None, fdel=None, doc=None) -> property
 
@@ -803,7 +803,7 @@ roughly equivalent to::
 
 To support automatic creation of methods, functions include the
 :meth:`__get__` method for binding methods during attribute access.  This
-means that functions are non-data descriptors which return bound methods
+means that functions are non-data descriptors that return bound methods
 during dotted lookup from an instance.  Here's how it works::
 
     class Function:
@@ -1016,7 +1016,7 @@ attributes stored in ``__slots__``::
 
     class Immutable:
 
-        __slots__ = ('_dept', '_name')          # Replace instance dictionary
+        __slots__ = ('_dept', '_name')          # Replace the instance dictionary
 
         def __init__(self, dept, name):
             self._dept = dept                   # Store to private attribute
@@ -1086,7 +1086,7 @@ by member descriptors::
 
 The :meth:`type.__new__` method takes care of adding member objects to class
 variables.  The :meth:`object.__new__` method takes care of creating instances
-that have slots instead of a instance dictionary.  Here is a rough equivalent
+that have slots instead of an instance dictionary.  Here is a rough equivalent
 in pure Python::
 
     class Type(type):

Doc/howto/unicode.rst

@@ -609,9 +609,9 @@ implemented by converting the Unicode string into some encoding that
 varies depending on the system.  Today Python is converging on using
 UTF-8: Python on MacOS has used UTF-8 for several versions, and Python
 3.6 switched to using UTF-8 on Windows as well.  On Unix systems,
-there will only be a filesystem encoding if you've set the ``LANG`` or
-``LC_CTYPE`` environment variables; if you haven't, the default
-encoding is again UTF-8.
+there will only be a :term:`filesystem encoding <filesystem encoding and error
+handler>`. if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if
+you haven't, the default encoding is again UTF-8.
 
 The :func:`sys.getfilesystemencoding` function returns the encoding to use on
 your current system, in case you want to do the encoding manually, but there's
@@ -633,8 +633,8 @@ provided the directory path as bytes or a Unicode string.  If you pass a
 Unicode string as the path, filenames will be decoded using the filesystem's
 encoding and a list of Unicode strings will be returned, while passing a byte
 path will return the filenames as bytes.  For example,
-assuming the default filesystem encoding is UTF-8, running the following
-program::
+assuming the default :term:`filesystem encoding <filesystem encoding and error
+handler>` is UTF-8, running the following program::
 
    fn = 'filename\u4500abc'
    f = open(fn, 'w')

Doc/library/ast.rst

@@ -80,12 +80,13 @@ Node classes
                   end_col_offset
 
       Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have
-      :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset`
-      attributes.  The :attr:`lineno` and :attr:`end_lineno` are the first and
-      last line numbers of source text span (1-indexed so the first line is line 1)
-      and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding
-      UTF-8 byte offsets of the first and last tokens that generated the node.
-      The UTF-8 offset is recorded because the parser uses UTF-8 internally.
+      :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and
+      :attr:`end_col_offset` attributes.  The :attr:`lineno` and :attr:`end_lineno`
+      are the first and last line numbers of source text span (1-indexed so the
+      first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset`
+      are the corresponding UTF-8 byte offsets of the first and last tokens that
+      generated the node. The UTF-8 offset is recorded because the parser uses
+      UTF-8 internally.
 
       Note that the end positions are not required by the compiler and are
       therefore optional. The end offset is *after* the last symbol, for example

Doc/library/concurrent.futures.rst

@@ -236,9 +236,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
    An :class:`Executor` subclass that executes calls asynchronously using a pool
    of at most *max_workers* processes.  If *max_workers* is ``None`` or not
    given, it will default to the number of processors on the machine.
-   If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError`
+   If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError`
    will be raised.
-   On Windows, *max_workers* must be equal or lower than ``61``. If it is not
+   On Windows, *max_workers* must be less than or equal to ``61``. If it is not
    then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then
    the default chosen will be at most ``61``, even if more processors are
    available.
@@ -250,7 +250,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock.
    each worker process; *initargs* is a tuple of arguments passed to the
    initializer.  Should *initializer* raise an exception, all currently
    pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`,
-   as well any attempt to submit more jobs to the pool.
+   as well as any attempt to submit more jobs to the pool.
 
    .. versionchanged:: 3.3
       When one of the worker processes terminates abruptly, a

Doc/library/contextlib.rst

@@ -126,6 +126,31 @@ Functions and classes provided:
 
    .. versionadded:: 3.7
 
+   Context managers defined with :func:`asynccontextmanager` can be used
+   either as decorators or with :keyword:`async with` statements::
+
+     import time
+
+     async def timeit():
+         now = time.monotonic()
+         try:
+             yield
+         finally:
+             print(f'it took {time.monotonic() - now}s to run')
+
+      @timeit()
+      async def main():
+          # ... async code ...
+
+   When used as a decorator, a new generator instance is implicitly created on
+   each function call. This allows the otherwise "one-shot" context managers
+   created by :func:`asynccontextmanager` to meet the requirement that context
+   managers support multiple invocations in order to be used as decorators.
+
+  .. versionchanged:: 3.10
+     Async context managers created with :func:`asynccontextmanager` can
+     be used as decorators.
+
 
 .. function:: closing(thing)
 
@@ -218,8 +243,26 @@ Functions and classes provided:
           with cm as file:
               # Perform processing on the file
 
+   It can also be used as a stand-in for
+   :ref:`asynchronous context managers <async-context-managers>`::
+
+       async def send_http(session=None):
+          if not session:
+              # If no http session, create it with aiohttp
+              cm = aiohttp.ClientSession()
+          else:
+              # Caller is responsible for closing the session
+              cm = nullcontext(session)
+
+          async with cm as session:
+              # Send http requests with session
+
    .. versionadded:: 3.7
 
+   .. versionchanged:: 3.10
+      :term:`asynchronous context manager` support was added.
+
+
 
 .. function:: suppress(*exceptions)
 
@@ -384,6 +427,45 @@ Functions and classes provided:
    .. versionadded:: 3.2
 
 
+.. class:: AsyncContextDecorator
+
+   Similar to :class:`ContextDecorator` but only for asynchronous functions.
+
+   Example of ``AsyncContextDecorator``::
+
+      from asyncio import run
+      from contextlib import AsyncContextDecorator
+
+      class mycontext(AsyncContextDecorator):
+          async def __aenter__(self):
+              print('Starting')
+              return self
+
+          async def __aexit__(self, *exc):
+              print('Finishing')
+              return False
+
+      >>> @mycontext()
+      ... async def function():
+      ...     print('The bit in the middle')
+      ...
+      >>> run(function())
+      Starting
+      The bit in the middle
+      Finishing
+
+      >>> async def function():
+      ...    async with mycontext():
+      ...         print('The bit in the middle')
+      ...
+      >>> run(function())
+      Starting
+      The bit in the middle
+      Finishing
+
+   .. versionadded:: 3.10
+
+
 .. class:: ExitStack()
 
    A context manager that is designed to make it easy to programmatically

Doc/library/devmode.rst

@@ -93,6 +93,9 @@ The Python Development Mode does not prevent the :option:`-O` command line
 option from removing :keyword:`assert` statements nor from setting
 :const:`__debug__` to ``False``.
 
+The Python Development Mode can only be enabled at the Python startup. Its
+value can be read from :data:`sys.flags.dev_mode <sys.flags>`.
+
 .. versionchanged:: 3.8
    The :class:`io.IOBase` destructor now logs ``close()`` exceptions.
 

Doc/library/exceptions.rst

@@ -313,8 +313,8 @@ The following exceptions are the exceptions that are usually raised.
    .. versionchanged:: 3.4
       The :attr:`filename` attribute is now the original file name passed to
       the function, instead of the name encoded to or decoded from the
-      filesystem encoding.  Also, the *filename2* constructor argument and
-      attribute was added.
+      :term:`filesystem encoding and error handler`. Also, the *filename2*
+      constructor argument and attribute was added.
 
 
 .. exception:: OverflowError

Doc/library/locale.rst

@@ -315,21 +315,25 @@ The :mod:`locale` module defines the following exception and functions:
 
 .. function:: getpreferredencoding(do_setlocale=True)
 
-   Return the encoding used for text data, according to user preferences.  User
-   preferences are expressed differently on different systems, and might not be
-   available programmatically on some systems, so this function only returns a
-   guess.
+   Return the :term:`locale encoding` used for text data, according to user
+   preferences.  User preferences are expressed differently on different
+   systems, and might not be available programmatically on some systems, so
+   this function only returns a guess.
 
-   On some systems, it is necessary to invoke :func:`setlocale` to obtain the user
-   preferences, so this function is not thread-safe. If invoking setlocale is not
-   necessary or desired, *do_setlocale* should be set to ``False``.
+   On some systems, it is necessary to invoke :func:`setlocale` to obtain the
+   user preferences, so this function is not thread-safe. If invoking setlocale
+   is not necessary or desired, *do_setlocale* should be set to ``False``.
 
-   On Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), always
-   return ``'UTF-8'``, the locale and the *do_setlocale* argument are ignored.
+   On Android or if the :ref:`Python UTF-8 Mode <utf8-mode>` is enabled, always
+   return ``'UTF-8'``, the :term:`locale encoding` and the *do_setlocale*
+   argument are ignored.
+
+   The :ref:`Python preinitialization <c-preinit>` configures the LC_CTYPE
+   locale. See also the :term:`filesystem encoding and error handler`.
 
    .. versionchanged:: 3.7
-      The function now always returns ``UTF-8`` on Android or if the UTF-8 mode
-      is enabled.
+      The function now always returns ``UTF-8`` on Android or if the
+      :ref:`Python UTF-8 Mode <utf8-mode>` is enabled.
 
 
 .. function:: normalize(localename)

Doc/library/os.rst

@@ -68,8 +68,13 @@ File Names, Command Line Arguments, and Environment Variables
 In Python, file names, command line arguments, and environment variables are
 represented using the string type. On some systems, decoding these strings to
 and from bytes is necessary before passing them to the operating system. Python
-uses the file system encoding to perform this conversion (see
-:func:`sys.getfilesystemencoding`).
+uses the :term:`filesystem encoding and error handler` to perform this
+conversion (see :func:`sys.getfilesystemencoding`).
+
+The :term:`filesystem encoding and error handler` are configured at Python
+startup by the :c:func:`PyConfig_Read` function: see
+:c:member:`~PyConfig.filesystem_encoding` and
+:c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
 
 .. versionchanged:: 3.1
    On some systems, conversion using the file system encoding may fail. In this
@@ -79,9 +84,72 @@ uses the file system encoding to perform this conversion (see
    original byte on encoding.
 
 
-The file system encoding must guarantee to successfully decode all bytes
-below 128. If the file system encoding fails to provide this guarantee, API
-functions may raise UnicodeErrors.
+The :term:`file system encoding <filesystem encoding and error handler>` must
+guarantee to successfully decode all bytes below 128. If the file system
+encoding fails to provide this guarantee, API functions can raise
+:exc:`UnicodeError`.
+
+See also the :term:`locale encoding`.
+
+
+.. _utf8-mode:
+
+Python UTF-8 Mode
+-----------------
+
+.. versionadded:: 3.7
+   See :pep:`540` for more details.
+
+The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage
+of the UTF-8 encoding:
+
+* Use UTF-8 as the :term:`filesystem encoding <filesystem encoding and error
+  handler>`.
+* :func:`sys.getfilesystemencoding()` returns ``'UTF-8'``.
+* :func:`locale.getpreferredencoding()` returns ``'UTF-8'`` (the *do_setlocale*
+  argument has no effect).
+* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use
+  UTF-8 as their text encoding, with the ``surrogateescape``
+  :ref:`error handler <error-handlers>` being enabled for :data:`sys.stdin`
+  and :data:`sys.stdout` (:data:`sys.stderr` continues to use
+  ``backslashreplace`` as it does in the default locale-aware mode)
+* On Unix, :func:`os.device_encoding` returns ``'UTF-8'``. rather than the
+  device encoding.
+
+Note that the standard stream settings in UTF-8 mode can be overridden by
+:envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware
+mode).
+
+As a consequence of the changes in those lower level APIs, other higher
+level APIs also exhibit different default behaviours:
+
+* Command line arguments, environment variables and filenames are decoded
+  to text using the UTF-8 encoding.
+* :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding.
+* :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8
+  encoding by default. However, they still use the strict error handler by
+  default so that attempting to open a binary file in text mode is likely
+  to raise an exception rather than producing nonsense data.
+
+The :ref:`Python UTF-8 Mode <utf8-mode>` is enabled if the LC_CTYPE locale is
+``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read`
+function).
+
+It can be enabled or disabled using the :option:`-X utf8 <-X>` command line
+option and the :envvar:`PYTHONUTF8` environment variable.
+
+If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the
+interpreter defaults to using the current locale settings, *unless* the current
+locale is identified as a legacy ASCII-based locale (as described for
+:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or
+fails. In such legacy locales, the interpreter will default to enabling UTF-8
+mode unless explicitly instructed not to do so.
+
+The Python UTF-8 Mode can only be enabled at the Python startup. Its value
+can be read from :data:`sys.flags.utf8_mode <sys.flags>`.
+
+See also the :ref:`UTF-8 mode on Windows <win-utf8-mode>`
+and the :term:`filesystem encoding and error handler`.
 
 
 .. _os-procinfo:
@@ -165,9 +233,9 @@ process and user.
 
 .. function:: fsencode(filename)
 
-   Encode :term:`path-like <path-like object>` *filename* to the filesystem
-   encoding with ``'surrogateescape'`` error handler, or ``'strict'`` on
-   Windows; return :class:`bytes` unchanged.
+   Encode :term:`path-like <path-like object>` *filename* to the
+   :term:`filesystem encoding and error handler`; return :class:`bytes`
+   unchanged.
 
    :func:`fsdecode` is the reverse function.
 
@@ -181,8 +249,8 @@ process and user.
 .. function:: fsdecode(filename)
 
    Decode the :term:`path-like <path-like object>` *filename* from the
-   filesystem encoding with ``'surrogateescape'`` error handler, or ``'strict'``
-   on Windows; return :class:`str` unchanged.
+   :term:`filesystem encoding and error handler`; return :class:`str`
+   unchanged.
 
    :func:`fsencode` is the reverse function.
 
@@ -742,6 +810,12 @@ as internal buffering of data.
    Return a string describing the encoding of the device associated with *fd*
    if it is connected to a terminal; else return :const:`None`.
 
+   On Unix, if the :ref:`Python UTF-8 Mode <utf8-mode>` is enabled, return
+   ``'UTF-8'`` rather than the device encoding.
+
+   .. versionchanged:: 3.10
+      On Unix, the function now implements the Python UTF-8 Mode.
+
 
 .. function:: dup(fd)
 
@@ -3246,7 +3320,7 @@ These functions are all available on Linux only.
    Removes the extended filesystem attribute *attribute* from *path*.
    *attribute* should be bytes or str (directly or indirectly through the
    :class:`PathLike` interface). If it is a string, it is encoded
-   with the filesystem encoding.
+   with the :term:`filesystem encoding and error handler`.
 
    This function can support :ref:`specifying a file descriptor <path_fd>` and
    :ref:`not following symlinks <follow_symlinks>`.
@@ -3262,7 +3336,7 @@ These functions are all available on Linux only.
    Set the extended filesystem attribute *attribute* on *path* to *value*.
    *attribute* must be a bytes or str with no embedded NULs (directly or
    indirectly through the :class:`PathLike` interface). If it is a str,
-   it is encoded with the filesystem encoding.  *flags* may be
+   it is encoded with the :term:`filesystem encoding and error handler`.  *flags* may be
    :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is
    given and the attribute does not exist, ``EEXISTS`` will be raised.
    If :data:`XATTR_CREATE` is given and the attribute already exists, the

Doc/library/platform.rst

@@ -209,13 +209,6 @@ Windows Platform
    which means the OS version uses debugging code, i.e. code that checks arguments,
    ranges, etc.
 
-   .. note::
-
-      This function works best with Mark Hammond's
-      :mod:`win32all` package installed, but also on Python 2.3 and
-      later (support for this was added in Python 2.6). It obviously
-      only runs on Win32 compatible platforms.
-
 .. function:: win32_edition()
 
    Returns a string representing the current Windows edition.  Possible

Doc/library/sys.rst

@@ -196,6 +196,18 @@ always available.
 
    .. audit-event:: sys._current_frames "" sys._current_frames
 
+.. function:: _current_exceptions()
+
+   Return a dictionary mapping each thread's identifier to the topmost exception
+   currently active in that thread at the time the function is called.
+   If a thread is not currently handling an exception, it is not included in
+   the result dictionary.
+
+   This is most useful for statistical profiling.
+
+   This function should be used for internal and specialized purposes only.
+
+   .. audit-event:: sys._current_exceptions "" sys._current_exceptions
 
 .. function:: breakpointhook()
 
@@ -615,21 +627,24 @@ always available.
 
 .. function:: getfilesystemencoding()
 
-   Return the name of the encoding used to convert between Unicode
-   filenames and bytes filenames.
+   Get the :term:`filesystem encoding <filesystem encoding and error handler>`:
+   the encoding used with the :term:`filesystem error handler <filesystem
+   encoding and error handler>` to convert between Unicode filenames and bytes
+   filenames. The filesystem error handler is returned from
+   :func:`getfilesystemencoding`.
 
    For best compatibility, str should be used for filenames in all cases,
    although representing filenames as bytes is also supported. Functions
    accepting or returning filenames should support either str or bytes and
    internally convert to the system's preferred representation.
 
-   This encoding is always ASCII-compatible.
-
    :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that
    the correct encoding and errors mode are used.
 
-   The filesystem encoding is initialized from
-   :c:member:`PyConfig.filesystem_encoding`.
+   The :term:`filesystem encoding and error handler` are configured at Python
+   startup by the :c:func:`PyConfig_Read` function: see
+   :c:member:`~PyConfig.filesystem_encoding` and
+   :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
 
    .. versionchanged:: 3.2
       :func:`getfilesystemencoding` result cannot be ``None`` anymore.
@@ -639,20 +654,25 @@ always available.
       and :func:`_enablelegacywindowsfsencoding` for more information.
 
    .. versionchanged:: 3.7
-      Return 'utf-8' in the UTF-8 mode.
+      Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode <utf8-mode>` is
+      enabled.
 
 
 .. function:: getfilesystemencodeerrors()
 
-   Return the name of the error mode used to convert between Unicode filenames
-   and bytes filenames. The encoding name is returned from
+   Get the :term:`filesystem error handler <filesystem encoding and error
+   handler>`: the error handler used with the :term:`filesystem encoding
+   <filesystem encoding and error handler>` to convert between Unicode
+   filenames and bytes filenames. The filesystem encoding is returned from
    :func:`getfilesystemencoding`.
 
    :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that
    the correct encoding and errors mode are used.
 
-   The filesystem error handler is initialized from
-   :c:member:`PyConfig.filesystem_errors`.
+   The :term:`filesystem encoding and error handler` are configured at Python
+   startup by the :c:func:`PyConfig_Read` function: see
+   :c:member:`~PyConfig.filesystem_encoding` and
+   :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
 
    .. versionadded:: 3.6
 
@@ -1445,8 +1465,9 @@ always available.
 
 .. function:: _enablelegacywindowsfsencoding()
 
-   Changes the default filesystem encoding and errors mode to 'mbcs' and
-   'replace' respectively, for consistency with versions of Python prior to 3.6.
+   Changes the :term:`filesystem encoding and error handler` to 'mbcs' and
+   'replace' respectively, for consistency with versions of Python prior to
+   3.6.
 
    This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING`
    environment variable before launching Python.
@@ -1476,9 +1497,8 @@ always available.
    returned by the :func:`open` function.  Their parameters are chosen as
    follows:
 
-   * The character encoding is platform-dependent.  Non-Windows
-     platforms use the locale encoding (see
-     :meth:`locale.getpreferredencoding()`).
+   * The encoding and error handling are is initialized from
+     :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`.
 
      On Windows, UTF-8 is used for the console device.  Non-character
      devices such as disk files and pipes use the system locale
@@ -1486,7 +1506,7 @@ always available.
      devices such as NUL (i.e. where ``isatty()`` returns ``True``) use the
      value of the console input and output codepages at startup,
      respectively for stdin and stdout/stderr. This defaults to the
-     system locale encoding if the process is not initially attached
+     system :term:`locale encoding` if the process is not initially attached
      to a console.
 
      The special behaviour of the console can be overridden

Doc/library/threading.rst

@@ -121,6 +121,17 @@ This module defines the following functions:
    :meth:`~Thread.run` method is called.
 
 
+.. function:: gettrace()
+
+   .. index::
+      single: trace function
+      single: debugger
+
+   Get the trace function as set by :func:`settrace`.
+
+   .. versionadded:: 3.10
+
+
 .. function:: setprofile(func)
 
    .. index:: single: profile function
@@ -130,6 +141,15 @@ This module defines the following functions:
    :meth:`~Thread.run` method is called.
 
 
+.. function:: getprofile()
+
+   .. index:: single: profile function
+
+   Get the profiler function as set by :func:`setprofile`.
+
+   .. versionadded:: 3.10
+
+
 .. function:: stack_size([size])
 
    Return the thread stack size used when creating new threads.  The optional

Doc/library/traceback.rst

@@ -36,7 +36,8 @@ The module defines the following functions:
        Added negative *limit* support.
 
 
-.. function:: print_exception(etype, value, tb, limit=None, file=None, chain=True)
+.. function:: print_exception(exc, /[, value, tb], limit=None, \
+                              file=None, chain=True)
 
    Print exception information and stack trace entries from traceback object
    *tb* to *file*. This differs from :func:`print_tb` in the following
@@ -45,7 +46,7 @@ The module defines the following functions:
    * if *tb* is not ``None``, it prints a header ``Traceback (most recent
      call last):``
 
-   * it prints the exception *etype* and *value* after the stack trace
+   * it prints the exception type and *value* after the stack trace
 
    .. index:: single: ^ (caret); marker
 
@@ -53,6 +54,10 @@ The module defines the following functions:
      format, it prints the line where the syntax error occurred with a caret
      indicating the approximate position of the error.
 
+   Since Python 3.10, instead of passing *value* and *tb*, an exception object
+   can be passed as the first argument. If *value* and *tb* are provided, the
+   first argument is ignored in order to provide backwards compatibility.
+
    The optional *limit* argument has the same meaning as for :func:`print_tb`.
    If *chain* is true (the default), then chained exceptions (the
    :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be
@@ -62,6 +67,10 @@ The module defines the following functions:
    .. versionchanged:: 3.5
       The *etype* argument is ignored and inferred from the type of *value*.
 
+   .. versionchanged:: 3.10
+      The *etype* parameter has been renamed to *exc* and is now
+      positional-only.
+
 
 .. function:: print_exc(limit=None, file=None, chain=True)
 
@@ -121,18 +130,26 @@ The module defines the following functions:
    text line is not ``None``.
 
 
-.. function:: format_exception_only(etype, value)
+.. function:: format_exception_only(exc, /[, value])
 
-   Format the exception part of a traceback.  The arguments are the exception
-   type and value such as given by ``sys.last_type`` and ``sys.last_value``.
-   The return value is a list of strings, each ending in a newline.  Normally,
-   the list contains a single string; however, for :exc:`SyntaxError`
-   exceptions, it contains several lines that (when printed) display detailed
-   information about where the syntax error occurred.  The message indicating
-   which exception occurred is the always last string in the list.
+   Format the exception part of a traceback using an exception value such as
+   given by ``sys.last_value``.  The return value is a list of strings, each
+   ending in a newline.  Normally, the list contains a single string; however,
+   for :exc:`SyntaxError` exceptions, it contains several lines that (when
+   printed) display detailed information about where the syntax error occurred.
+   The message indicating which exception occurred is the always last string in
+   the list.
+
+   Since Python 3.10, instead of passing *value*, an exception object
+   can be passed as the first argument.  If *value* is provided, the first
+   argument is ignored in order to provide backwards compatibility.
+
+   .. versionchanged:: 3.10
+      The *etype* parameter has been renamed to *exc* and is now
+      positional-only.
 
 
-.. function:: format_exception(etype, value, tb, limit=None, chain=True)
+.. function:: format_exception(exc, /[, value, tb], limit=None, chain=True)
 
    Format a stack trace and the exception information.  The arguments  have the
    same meaning as the corresponding arguments to :func:`print_exception`.  The
@@ -143,6 +160,10 @@ The module defines the following functions:
    .. versionchanged:: 3.5
       The *etype* argument is ignored and inferred from the type of *value*.
 
+   .. versionchanged:: 3.10
+      This function's behavior and signature were modified to match
+      :func:`print_exception`.
+
 
 .. function:: format_exc(limit=None, chain=True)
 

Doc/library/xml.etree.elementtree.rst

@@ -455,6 +455,12 @@ Supported XPath syntax
 |                       | has the given value.  The value cannot contain       |
 |                       | quotes.                                              |
 +-----------------------+------------------------------------------------------+
+| ``[@attrib!='value']``| Selects all elements for which the given attribute   |
+|                       | does not have the given value. The value cannot      |
+|                       | contain quotes.                                      |
+|                       |                                                      |
+|                       | .. versionadded:: 3.10                               |
++-----------------------+------------------------------------------------------+
 | ``[tag]``             | Selects all elements that have a child named         |
 |                       | ``tag``.  Only immediate children are supported.     |
 +-----------------------+------------------------------------------------------+
@@ -463,10 +469,22 @@ Supported XPath syntax
 |                       |                                                      |
 |                       | .. versionadded:: 3.7                                |
 +-----------------------+------------------------------------------------------+
+| ``[.!='text']``       | Selects all elements whose complete text content,    |
+|                       | including descendants, does not equal the given      |
+|                       | ``text``.                                            |
+|                       |                                                      |
+|                       | .. versionadded:: 3.10                               |
++-----------------------+------------------------------------------------------+
 | ``[tag='text']``      | Selects all elements that have a child named         |
 |                       | ``tag`` whose complete text content, including       |
 |                       | descendants, equals the given ``text``.              |
 +-----------------------+------------------------------------------------------+
+| ``[tag!='text']``     | Selects all elements that have a child named         |
+|                       | ``tag`` whose complete text content, including       |
+|                       | descendants, does not equal the given ``text``.      |
+|                       |                                                      |
+|                       | .. versionadded:: 3.10                               |
++-----------------------+------------------------------------------------------+
 | ``[position]``        | Selects all elements that are located at the given   |
 |                       | position.  The position can be either an integer     |
 |                       | (1 is the first position), the expression ``last()`` |

Doc/tools/static/switchers.js

@@ -1,156 +0,0 @@
-(function() {
-  'use strict';
-
-  // Parses versions in URL segments like:
-  // "3", "dev", "release/2.7" or "3.6rc2"
-  var version_regexs = [
-    '(?:\\d)',
-    '(?:\\d\\.\\d[\\w\\d\\.]*)',
-    '(?:dev)',
-    '(?:release/\\d.\\d[\\x\\d\\.]*)'];
-
-  var all_versions = {
-    '3.10': 'dev (3.10)',
-    '3.9': 'pre (3.9)',
-    '3.8': '3.8',
-    '3.7': '3.7',
-    '3.6': '3.6',
-    '2.7': '2.7',
-  };
-
-  var all_languages = {
-      'en': 'English',
-      'fr': 'French',
-      'ja': 'Japanese',
-      'ko': 'Korean',
-      'pt-br': 'Brazilian Portuguese',
-      'zh-cn': 'Simplified Chinese',
-  };
-
-  function build_version_select(current_version, current_release) {
-    var buf = ['<select>'];
-
-    $.each(all_versions, function(version, title) {
-      buf.push('<option value="' + version + '"');
-      if (version == current_version)
-        buf.push(' selected="selected">' + current_release + '</option>');
-      else
-        buf.push('>' + title + '</option>');
-    });
-
-    buf.push('</select>');
-    return buf.join('');
-  }
-
-  function build_language_select(current_language) {
-    var buf = ['<select>'];
-
-    $.each(all_languages, function(language, title) {
-      if (language == current_language)
-        buf.push('<option value="' + language + '" selected="selected">' +
-                 all_languages[current_language] + '</option>');
-      else
-        buf.push('<option value="' + language + '">' + title + '</option>');
-    });
-    if (!(current_language in all_languages)) {
-        // In case we're browsing a language that is not yet in all_languages.
-        buf.push('<option value="' + current_language + '" selected="selected">' +
-                 current_language + '</option>');
-        all_languages[current_language] = current_language;
-    }
-    buf.push('</select>');
-    return buf.join('');
-  }
-
-  function navigate_to_first_existing(urls) {
-    // Navigate to the first existing URL in urls.
-    var url = urls.shift();
-    if (urls.length == 0) {
-      window.location.href = url;
-      return;
-    }
-    $.ajax({
-      url: url,
-      success: function() {
-        window.location.href = url;
-      },
-      error: function() {
-        navigate_to_first_existing(urls);
-      }
-    });
-  }
-
-  function on_version_switch() {
-    var selected_version = $(this).children('option:selected').attr('value') + '/';
-    var url = window.location.href;
-    var current_language = language_segment_from_url(url);
-    var current_version = version_segment_in_url(url);
-    var new_url = url.replace('.org/' + current_language + current_version,
-                              '.org/' + current_language + selected_version);
-    if (new_url != url) {
-      navigate_to_first_existing([
-        new_url,
-        url.replace('.org/' + current_language + current_version,
-                    '.org/' + selected_version),
-        'https://docs.python.org/' + current_language + selected_version,
-        'https://docs.python.org/' + selected_version,
-        'https://docs.python.org/'
-      ]);
-    }
-  }
-
-  function on_language_switch() {
-    var selected_language = $(this).children('option:selected').attr('value') + '/';
-    var url = window.location.href;
-    var current_language = language_segment_from_url(url);
-    var current_version = version_segment_in_url(url);
-    if (selected_language == 'en/') // Special 'default' case for english.
-      selected_language = '';
-    var new_url = url.replace('.org/' + current_language + current_version,
-                              '.org/' + selected_language + current_version);
-    if (new_url != url) {
-      navigate_to_first_existing([
-        new_url,
-        'https://docs.python.org/'
-      ]);
-    }
-  }
-
-  // Returns the path segment of the language as a string, like 'fr/'
-  // or '' if not found.
-  function language_segment_from_url(url) {
-    var language_regexp = '\.org/([a-z]{2}(?:-[a-z]{2})?/)';
-    var match = url.match(language_regexp);
-    if (match !== null)
-        return match[1];
-    return '';
-  }
-
-  // Returns the path segment of the version as a string, like '3.6/'
-  // or '' if not found.
-  function version_segment_in_url(url) {
-    var language_segment = '(?:[a-z]{2}(?:-[a-z]{2})?/)';
-    var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)';
-    var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')';
-    var match = url.match(version_regexp);
-    if (match !== null)
-      return match[1];
-    return ''
-  }
-
-  $(document).ready(function() {
-    var release = DOCUMENTATION_OPTIONS.VERSION;
-    var language_segment = language_segment_from_url(window.location.href);
-    var current_language = language_segment.replace(/\/+$/g, '') || 'en';
-    var version = release.substr(0, 3);
-    var version_select = build_version_select(version, release);
-
-    $('.version_switcher_placeholder').html(version_select);
-    $('.version_switcher_placeholder select').bind('change', on_version_switch);
-
-    var language_select = build_language_select(current_language);
-
-    $('.language_switcher_placeholder').html(language_select);
-    $('.language_switcher_placeholder select').bind('change', on_language_switch);
-  });
-})();

Doc/tools/templates/dummy.html

@@ -6,3 +6,12 @@ In extensions/pyspecific.py:
 {% trans %}CPython implementation detail:{% endtrans %}
 {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %}
 {% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %}
+
+
+In docsbuild-scripts, when rewriting indexsidebar.html with actual versions:
+
+{% trans %}in development{% endtrans %}
+{% trans %}pre-release{% endtrans %}
+{% trans %}stable{% endtrans %}
+{% trans %}security-fixes{% endtrans %}
+{% trans %}EOL{% endtrans %}

Doc/tools/templates/indexsidebar.html

@@ -2,12 +2,8 @@
 <p><a href="{{ pathto('download') }}">{% trans %}Download these documents{% endtrans %}</a></p>
 <h3>{% trans %}Docs by version{% endtrans %}</h3>
 <ul>
-  <li><a href="https://docs.python.org/3.10/">{% trans %}Python 3.10 (in development){% endtrans %}</a></li>
-  <li><a href="https://docs.python.org/3.9/">{% trans %}Python 3.9 (pre-release){% endtrans %}</a></li>
-  <li><a href="https://docs.python.org/3.8/">{% trans %}Python 3.8 (stable){% endtrans %}</a></li>
-  <li><a href="https://docs.python.org/3.7/">{% trans %}Python 3.7 (stable){% endtrans %}</a></li>
-  <li><a href="https://docs.python.org/3.6/">{% trans %}Python 3.6 (security-fixes){% endtrans %}</a></li>
-  <li><a href="https://docs.python.org/2.7/">{% trans %}Python 2.7 (EOL){% endtrans %}</a></li>
+  <li><a href="https://docs.python.org/">{% trans %}Stable{% endtrans %}</a></li>
+  <li><a href="https://docs.python.org/dev/">{% trans %}In development{% endtrans %}</a></li>
   <li><a href="https://www.python.org/doc/versions/">{% trans %}All versions{% endtrans %}</a></li>
 </ul>
 

Doc/tools/templates/layout.html

@@ -12,22 +12,14 @@
 
 {% block rootrellink %}
 {{ super() }}
-    <li>
-      {%- if switchers is defined %}
-      <span class="language_switcher_placeholder">{{ language or 'en' }}</span>
-      <span class="version_switcher_placeholder">{{ release }}</span>
-      <a href="{{ pathto('index') }}">{% trans %}Documentation {% endtrans %}</a>{{ reldelim1 }}
-      {%- else %}
+    <li id="cpython-language-and-version">
       <a href="{{ pathto('index') }}">{{ shorttitle }}</a>{{ reldelim1 }}
-      {%- endif %}
     </li>
 {% endblock %}
 
 {% block extrahead %}
     <link rel="canonical" href="https://docs.python.org/3/{{pagename}}.html" />
     {% if builder != "htmlhelp" %}
-      {% if switchers is defined and not embedded %}
-      <script type="text/javascript" src="{{ pathto('_static/switchers.js', 1) }}"></script>{% endif %}
       {% if pagename == 'whatsnew/changelog' and not embedded %}
       <script type="text/javascript" src="{{ pathto('_static/changelog_search.js', 1) }}"></script>{% endif %}
     {% endif %}

Doc/tutorial/errors.rst

@@ -273,15 +273,15 @@ Exception Chaining
 ==================
 
 The :keyword:`raise` statement allows an optional :keyword:`from` which enables
-chaining exceptions by setting the ``__cause__`` attribute of the raised
-exception. For example::
+chaining exceptions. For example::
 
-    raise RuntimeError from OSError
+    # exc must be exception instance or None.
+    raise RuntimeError from exc
 
 This can be useful when you are transforming exceptions. For example::
 
     >>> def func():
-    ...    raise IOError
+    ...     raise IOError
     ...
     >>> try:
     ...     func()
@@ -297,12 +297,11 @@ This can be useful when you are transforming exceptions. For example::
     <BLANKLINE>
     Traceback (most recent call last):
       File "<stdin>", line 4, in <module>
-    RuntimeError
+    RuntimeError: Failed to open database
 
-The expression following the :keyword:`from` must be either an exception or
-``None``. Exception chaining happens automatically when an exception is raised
-inside an exception handler or :keyword:`finally` section. Exception chaining
-can be disabled by using ``from None`` idiom:
+Exception chaining happens automatically when an exception is raised inside an
+:keyword:`except` or :keyword:`finally` section. Exception chaining can be
+disabled by using ``from None`` idiom:
 
     >>> try:
     ...     open('database.sqlite')
@@ -313,6 +312,8 @@ can be disabled by using ``from None`` idiom:
       File "<stdin>", line 4, in <module>
     RuntimeError
 
+For more information about chaining mechanics, see :ref:`bltin-exceptions`.
+
 
 .. _tut-userexceptions:
 

Doc/using/cmdline.rst

@@ -447,10 +447,9 @@ Miscellaneous options
    * ``-X dev``: enable :ref:`Python Development Mode <devmode>`, introducing
      additional runtime checks that are too expensive to be enabled by
      default.
-   * ``-X utf8`` enables UTF-8 mode for operating system interfaces, overriding
-     the default locale-aware mode. ``-X utf8=0`` explicitly disables UTF-8
-     mode (even when it would otherwise activate automatically).
-     See :envvar:`PYTHONUTF8` for more details.
+   * ``-X utf8`` enables the :ref:`Python UTF-8 Mode <utf8-mode>`.
+     ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode <utf8-mode>`
+     (even when it would otherwise activate automatically).
    * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel
      tree rooted at the given directory instead of to the code tree. See also
      :envvar:`PYTHONPYCACHEPREFIX`.
@@ -810,9 +809,10 @@ conflict.
 
 .. envvar:: PYTHONLEGACYWINDOWSFSENCODING
 
-   If set to a non-empty string, the default filesystem encoding and errors mode
-   will revert to their pre-3.6 values of 'mbcs' and 'replace', respectively.
-   Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used.
+   If set to a non-empty string, the default :term:`filesystem encoding and
+   error handler` mode will revert to their pre-3.6 values of 'mbcs' and
+   'replace', respectively.  Otherwise, the new defaults 'utf-8' and
+   'surrogatepass' are used.
 
    This may also be enabled at runtime with
    :func:`sys._enablelegacywindowsfsencoding()`.
@@ -898,54 +898,14 @@ conflict.
 
 .. envvar:: PYTHONUTF8
 
-   If set to ``1``, enables the interpreter's UTF-8 mode, where ``UTF-8`` is
-   used as the text encoding for system interfaces, regardless of the
-   current locale setting.
+   If set to ``1``, enable the :ref:`Python UTF-8 Mode <utf8-mode>`.
 
-   This means that:
-
-    * :func:`sys.getfilesystemencoding()` returns ``'UTF-8'`` (the locale
-      encoding is ignored).
-    * :func:`locale.getpreferredencoding()` returns ``'UTF-8'`` (the locale
-      encoding is ignored, and the function's ``do_setlocale`` parameter has no
-      effect).
-    * :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use
-      UTF-8 as their text encoding, with the ``surrogateescape``
-      :ref:`error handler <error-handlers>` being enabled for :data:`sys.stdin`
-      and :data:`sys.stdout` (:data:`sys.stderr` continues to use
-      ``backslashreplace`` as it does in the default locale-aware mode)
-
-   As a consequence of the changes in those lower level APIs, other higher
-   level APIs also exhibit different default behaviours:
-
-    * Command line arguments, environment variables and filenames are decoded
-      to text using the UTF-8 encoding.
-    * :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding.
-    * :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8
-      encoding by default. However, they still use the strict error handler by
-      default so that attempting to open a binary file in text mode is likely
-      to raise an exception rather than producing nonsense data.
-
-   Note that the standard stream settings in UTF-8 mode can be overridden by
-   :envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware
-   mode).
-
-   If set to ``0``, the interpreter runs in its default locale-aware mode.
+   If set to ``0``, disable the :ref:`Python UTF-8 Mode <utf8-mode>`.
 
    Setting any other non-empty string causes an error during interpreter
    initialisation.
 
-   If this environment variable is not set at all, then the interpreter defaults
-   to using the current locale settings, *unless* the current locale is
-   identified as a legacy ASCII-based locale
-   (as described for :envvar:`PYTHONCOERCECLOCALE`), and locale coercion is
-   either disabled or fails. In such legacy locales, the interpreter will
-   default to enabling UTF-8 mode unless explicitly instructed not to do so.
-
-   Also available as the :option:`-X` ``utf8`` option.
-
    .. versionadded:: 3.7
-      See :pep:`540` for more details.
 
 
 Debug-mode variables

Doc/using/windows.rst

@@ -614,21 +614,14 @@ Page).  Python uses it for the default encoding of text files (e.g.
 This may cause issues because UTF-8 is widely used on the internet
 and most Unix systems, including WSL (Windows Subsystem for Linux).
 
-You can use UTF-8 mode to change the default text encoding to UTF-8.
-You can enable UTF-8 mode via the ``-X utf8`` command line option, or
-the ``PYTHONUTF8=1`` environment variable.  See :envvar:`PYTHONUTF8` for
-enabling UTF-8 mode, and :ref:`setting-envvars` for how to modify
-environment variables.
+You can use the :ref:`Python UTF-8 Mode <utf8-mode>` to change the default text
+encoding to UTF-8. You can enable the :ref:`Python UTF-8 Mode <utf8-mode>` via
+the ``-X utf8`` command line option, or the ``PYTHONUTF8=1`` environment
+variable.  See :envvar:`PYTHONUTF8` for enabling UTF-8 mode, and
+:ref:`setting-envvars` for how to modify environment variables.
 
-When UTF-8 mode is enabled:
-
-* :func:`locale.getpreferredencoding` returns ``'UTF-8'`` instead of
-  the system encoding.  This function is used for the default text
-  encoding in many places, including :func:`open`, :class:`Popen`,
-  :meth:`Path.read_text`, etc.
-* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr`
-  all use UTF-8 as their text encoding.
-* You can still use the system encoding via the "mbcs" codec.
+When the :ref:`Python UTF-8 Mode <utf8-mode>` is enabled, you can still use the
+system encoding (the ANSI Code Page) via the "mbcs" codec.
 
 Note that adding ``PYTHONUTF8=1`` to the default environment variables
 will affect all Python 3.7+ applications on your system.
@@ -641,7 +634,8 @@ temporarily or use the ``-X utf8`` command line option.
    on Windows for:
 
    * Console I/O including standard I/O (see :pep:`528` for details).
-   * The filesystem encoding (see :pep:`529` for details).
+   * The :term:`filesystem encoding <filesystem encoding and error handler>`
+     (see :pep:`529` for details).
 
 
 .. _launcher:

Doc/whatsnew/3.10.rst

@@ -176,6 +176,13 @@ codecs
 Add a :func:`codecs.unregister` function to unregister a codec search function.
 (Contributed by Hai Shi in :issue:`41842`.)
 
+contextlib
+----------
+
+Add a :func:`contextlib.aclosing` context manager to safely close async generators
+and objects representing asynchronously released resources.
+(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.)
+
 curses
 ------
 
@@ -186,6 +193,12 @@ by :func:`curses.color_content`, :func:`curses.init_color`,
 support is provided by the underlying ncurses library.
 (Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.)
 
+doctest
+-------
+
+When a module does not define ``__loader__``, fall back to ``__spec__.loader``.
+(Contributed by Brett Cannon in :issue:`42133`.)
+
 encodings
 ---------
 :func:`encodings.normalize_encoding` now ignores non-ASCII characters.
@@ -198,6 +211,18 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and
 :func:`~glob.iglob` which allow to specify the root directory for searching.
 (Contributed by Serhiy Storchaka in :issue:`38144`.)
 
+inspect
+-------
+
+When a module does not define ``__loader__``, fall back to ``__spec__.loader``.
+(Contributed by Brett Cannon in :issue:`42133`.)
+
+linecache
+---------
+
+When a module does not define ``__loader__``, fall back to ``__spec__.loader``.
+(Contributed by Brett Cannon in :issue:`42133`.)
+
 os
 --
 
@@ -225,6 +250,12 @@ Added :func:`~statistics.covariance`, Pearson's
 :func:`~statistics.linear_regression` functions.
 (Contributed by Tymoteusz Wołodźko in :issue:`38490`.)
 
+site
+----
+
+When a module does not define ``__loader__``, fall back to ``__spec__.loader``.
+(Contributed by Brett Cannon in :issue:`42133`.)
+
 sys
 ---
 
@@ -232,6 +263,23 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line
 arguments passed to the Python executable.
 (Contributed by Victor Stinner in :issue:`23427`.)
 
+threading
+---------
+
+Added :func:`threading.gettrace` and :func:`threading.getprofile` to
+retrieve the functions set by :func:`threading.settrace` and
+:func:`threading.setprofile` respectively.
+(Contributed by Mario Corchero in :issue:`42251`.)
+
+traceback
+---------
+
+The :func:`~traceback.format_exception`,
+:func:`~traceback.format_exception_only`, and
+:func:`~traceback.print_exception` functions can now take an exception object
+as a positional-only argument.
+(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.)
+
 types
 -----
 
@@ -328,6 +376,15 @@ This section lists previously described changes and other bugfixes
 that may require changes to your code.
 
 
+Changes in the Python API
+-------------------------
+
+* The *etype* parameters of the :func:`~traceback.format_exception`,
+  :func:`~traceback.format_exception_only`, and
+  :func:`~traceback.print_exception` functions in the :mod:`traceback` module
+  have been renamed to *exc*.
+  (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.)
+
 
 Build Changes
 =============
@@ -374,6 +431,19 @@ New Features
 * Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API.
   (Contributed by Alex Gaynor in :issue:`41784`.)
 
+* Added :c:func:`PyModule_AddObjectRef` function: similar to
+  :c:func:`PyModule_AddObjectRef` but don't steal a reference to the value on
+  success.
+  (Contributed by Victor Stinner in :issue:`1635741`.)
+
+* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the
+  reference count of an object and return the object.
+  (Contributed by Victor Stinner in :issue:`42262`.)
+
+* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc``
+  slot.
+  (Contributed by Hai Shi in :issue:`41832`.)
+
 
 Porting to Python 3.10
 ----------------------

Doc/whatsnew/3.7.rst

@@ -232,7 +232,8 @@ PEP 540: Forced UTF-8 Runtime Mode
 -----------------------------------
 
 The new :option:`-X` ``utf8`` command line option and :envvar:`PYTHONUTF8`
-environment variable can be used to enable the CPython *UTF-8 mode*.
+environment variable can be used to enable the :ref:`Python UTF-8 Mode
+<utf8-mode>`.
 
 When in UTF-8 mode, CPython ignores the locale settings, and uses the
 UTF-8 encoding by default.  The error handlers for :data:`sys.stdin` and

Include/Python-ast.h

@@ -1,4 +1,4 @@
-/* File automatically generated by Parser/asdl_c.py. */
+// File automatically generated by Parser/asdl_c.py.
 
 #ifndef Py_PYTHON_AST_H
 #define Py_PYTHON_AST_H

Include/boolobject.h

@@ -22,8 +22,8 @@ PyAPI_DATA(struct _longobject) _Py_FalseStruct, _Py_TrueStruct;
 #define Py_True ((PyObject *) &_Py_TrueStruct)
 
 /* Macros for returning Py_True or Py_False, respectively */
-#define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True
-#define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False
+#define Py_RETURN_TRUE return Py_NewRef(Py_True)
+#define Py_RETURN_FALSE return Py_NewRef(Py_False)
 
 /* Function to return a bool from a C long */
 PyAPI_FUNC(PyObject *) PyBool_FromLong(long);

Include/cpython/initconfig.h

@@ -41,7 +41,7 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
 
 /* --- PyPreConfig ----------------------------------------------- */
 
-typedef struct {
+typedef struct PyPreConfig {
     int _config_init;     /* _PyConfigInitEnum value */
 
     /* Parse Py_PreInitializeFromBytesArgs() arguments?
@@ -127,273 +127,84 @@ PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
 
 /* --- PyConfig ---------------------------------------------- */
 
-typedef struct {
+/* This structure is best documented in the Doc/c-api/init_config.rst file. */
+typedef struct PyConfig {
     int _config_init;     /* _PyConfigInitEnum value */
 
-    int isolated;         /* Isolated mode? see PyPreConfig.isolated */
-    int use_environment;  /* Use environment variables? see PyPreConfig.use_environment */
-    int dev_mode;         /* Python Development Mode? See PyPreConfig.dev_mode */
-
-    /* Install signal handlers? Yes by default. */
+    int isolated;
+    int use_environment;
+    int dev_mode;
     int install_signal_handlers;
-
-    int use_hash_seed;      /* PYTHONHASHSEED=x */
+    int use_hash_seed;
     unsigned long hash_seed;
-
-    /* Enable faulthandler?
-       Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */
     int faulthandler;
-
-    /* Enable tracemalloc?
-       Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */
     int tracemalloc;
-
-    int import_time;        /* PYTHONPROFILEIMPORTTIME, -X importtime */
-    int show_ref_count;     /* -X showrefcount */
-    int dump_refs;          /* PYTHONDUMPREFS */
-    int malloc_stats;       /* PYTHONMALLOCSTATS */
-
-    /* Python filesystem encoding and error handler:
-       sys.getfilesystemencoding() and sys.getfilesystemencodeerrors().
-
-       The Doc/c-api/init_config.rst documentation explains how Python selects
-       the filesystem encoding and error handler.
-
-      _PyUnicode_InitEncodings() updates the encoding name to the Python codec
-      name. For example, "ANSI_X3.4-1968" is replaced with "ascii". It also
-      sets Py_FileSystemDefaultEncoding to filesystem_encoding and
-      sets Py_FileSystemDefaultEncodeErrors to filesystem_errors. */
+    int import_time;
+    int show_ref_count;
+    int dump_refs;
+    int malloc_stats;
     wchar_t *filesystem_encoding;
     wchar_t *filesystem_errors;
-
-    wchar_t *pycache_prefix;  /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
-    int parse_argv;           /* Parse argv command line arguments? */
-
-    /* Command line arguments (sys.argv).
-
-       Set parse_argv to 1 to parse argv as Python command line arguments
-       and then strip Python arguments from argv.
-
-       If argv is empty, an empty string is added to ensure that sys.argv
-       always exists and is never empty. */
+    wchar_t *pycache_prefix;
+    int parse_argv;
+    PyWideStringList orig_argv;
     PyWideStringList argv;
-
-    /* Program name:
-
-       - If Py_SetProgramName() was called, use its value.
-       - On macOS, use PYTHONEXECUTABLE environment variable if set.
-       - If WITH_NEXT_FRAMEWORK macro is defined, use __PYVENV_LAUNCHER__
-         environment variable is set.
-       - Use argv[0] if available and non-empty.
-       - Use "python" on Windows, or "python3 on other platforms. */
-    wchar_t *program_name;
-
-    PyWideStringList xoptions;     /* Command line -X options */
-
-    /* Warnings options: lowest to highest priority. warnings.filters
-       is built in the reverse order (highest to lowest priority). */
+    PyWideStringList xoptions;
     PyWideStringList warnoptions;
-
-    /* If equal to zero, disable the import of the module site and the
-       site-dependent manipulations of sys.path that it entails. Also disable
-       these manipulations if site is explicitly imported later (call
-       site.main() if you want them to be triggered).
-
-       Set to 0 by the -S command line option. If set to -1 (default), it is
-       set to !Py_NoSiteFlag. */
     int site_import;
-
-    /* Bytes warnings:
-
-       * If equal to 1, issue a warning when comparing bytes or bytearray with
-         str or bytes with int.
-       * If equal or greater to 2, issue an error.
-
-       Incremented by the -b command line option. If set to -1 (default), inherit
-       Py_BytesWarningFlag value. */
     int bytes_warning;
-
-    /* If greater than 0, enable inspect: when a script is passed as first
-       argument or the -c option is used, enter interactive mode after
-       executing the script or the command, even when sys.stdin does not appear
-       to be a terminal.
-
-       Incremented by the -i command line option. Set to 1 if the PYTHONINSPECT
-       environment variable is non-empty. If set to -1 (default), inherit
-       Py_InspectFlag value. */
     int inspect;
-
-    /* If greater than 0: enable the interactive mode (REPL).
-
-       Incremented by the -i command line option. If set to -1 (default),
-       inherit Py_InteractiveFlag value. */
     int interactive;
-
-    /* Optimization level.
-
-       Incremented by the -O command line option. Set by the PYTHONOPTIMIZE
-       environment variable. If set to -1 (default), inherit Py_OptimizeFlag
-       value. */
     int optimization_level;
-
-    /* If greater than 0, enable the debug mode: turn on parser debugging
-       output (for expert only, depending on compilation options).
-
-       Incremented by the -d command line option. Set by the PYTHONDEBUG
-       environment variable. If set to -1 (default), inherit Py_DebugFlag
-       value. */
     int parser_debug;
-
-    /* If equal to 0, Python won't try to write ``.pyc`` files on the
-       import of source modules.
-
-       Set to 0 by the -B command line option and the PYTHONDONTWRITEBYTECODE
-       environment variable. If set to -1 (default), it is set to
-       !Py_DontWriteBytecodeFlag. */
     int write_bytecode;
-
-    /* If greater than 0, enable the verbose mode: print a message each time a
-       module is initialized, showing the place (filename or built-in module)
-       from which it is loaded.
-
-       If greater or equal to 2, print a message for each file that is checked
-       for when searching for a module. Also provides information on module
-       cleanup at exit.
-
-       Incremented by the -v option. Set by the PYTHONVERBOSE environment
-       variable. If set to -1 (default), inherit Py_VerboseFlag value. */
     int verbose;
-
-    /* If greater than 0, enable the quiet mode: Don't display the copyright
-       and version messages even in interactive mode.
-
-       Incremented by the -q option. If set to -1 (default), inherit
-       Py_QuietFlag value. */
     int quiet;
-
-   /* If greater than 0, don't add the user site-packages directory to
-      sys.path.
-
-      Set to 0 by the -s and -I command line options , and the PYTHONNOUSERSITE
-      environment variable. If set to -1 (default), it is set to
-      !Py_NoUserSiteDirectory. */
     int user_site_directory;
-
-    /* If non-zero, configure C standard steams (stdio, stdout,
-       stderr):
-
-       - Set O_BINARY mode on Windows.
-       - If buffered_stdio is equal to zero, make streams unbuffered.
-         Otherwise, enable streams buffering if interactive is non-zero. */
     int configure_c_stdio;
-
-    /* If equal to 0, enable unbuffered mode: force the stdout and stderr
-       streams to be unbuffered.
-
-       Set to 0 by the -u option. Set by the PYTHONUNBUFFERED environment
-       variable.
-       If set to -1 (default), it is set to !Py_UnbufferedStdioFlag. */
     int buffered_stdio;
-
-    /* Encoding of sys.stdin, sys.stdout and sys.stderr.
-       Value set from PYTHONIOENCODING environment variable and
-       Py_SetStandardStreamEncoding() function.
-       See also 'stdio_errors' attribute. */
     wchar_t *stdio_encoding;
-
-    /* Error handler of sys.stdin and sys.stdout.
-       Value set from PYTHONIOENCODING environment variable and
-       Py_SetStandardStreamEncoding() function.
-       See also 'stdio_encoding' attribute. */
     wchar_t *stdio_errors;
-
 #ifdef MS_WINDOWS
-    /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys
-       standard streams.
-
-       Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to
-       a non-empty string. If set to -1 (default), inherit
-       Py_LegacyWindowsStdioFlag value.
-
-       See PEP 528 for more details. */
     int legacy_windows_stdio;
 #endif
-
-    /* Value of the --check-hash-based-pycs command line option:
-
-       - "default" means the 'check_source' flag in hash-based pycs
-         determines invalidation
-       - "always" causes the interpreter to hash the source file for
-         invalidation regardless of value of 'check_source' bit
-       - "never" causes the interpreter to always assume hash-based pycs are
-         valid
-
-       The default value is "default".
-
-       See PEP 552 "Deterministic pycs" for more details. */
     wchar_t *check_hash_pycs_mode;
 
     /* --- Path configuration inputs ------------ */
-
-    /* If greater than 0, suppress _PyPathConfig_Calculate() warnings on Unix.
-       The parameter has no effect on Windows.
-
-       If set to -1 (default), inherit !Py_FrozenFlag value. */
     int pathconfig_warnings;
-
-    wchar_t *pythonpath_env; /* PYTHONPATH environment variable */
-    wchar_t *home;          /* PYTHONHOME environment variable,
-                               see also Py_SetPythonHome(). */
+    wchar_t *program_name;
+    wchar_t *pythonpath_env;
+    wchar_t *home;
+    wchar_t *platlibdir;
 
     /* --- Path configuration outputs ----------- */
-
-    int module_search_paths_set;  /* If non-zero, use module_search_paths */
-    PyWideStringList module_search_paths;  /* sys.path paths. Computed if
-                                       module_search_paths_set is equal
-                                       to zero. */
-
-    wchar_t *executable;        /* sys.executable */
-    wchar_t *base_executable;   /* sys._base_executable */
-    wchar_t *prefix;            /* sys.prefix */
-    wchar_t *base_prefix;       /* sys.base_prefix */
-    wchar_t *exec_prefix;       /* sys.exec_prefix */
-    wchar_t *base_exec_prefix;  /* sys.base_exec_prefix */
-    wchar_t *platlibdir;        /* sys.platlibdir */
+    int module_search_paths_set;
+    PyWideStringList module_search_paths;
+    wchar_t *executable;
+    wchar_t *base_executable;
+    wchar_t *prefix;
+    wchar_t *base_prefix;
+    wchar_t *exec_prefix;
+    wchar_t *base_exec_prefix;
 
     /* --- Parameter only used by Py_Main() ---------- */
-
-    /* Skip the first line of the source ('run_filename' parameter), allowing use of non-Unix forms of
-       "#!cmd".  This is intended for a DOS specific hack only.
-
-       Set by the -x command line option. */
     int skip_source_first_line;
-
-    wchar_t *run_command;   /* -c command line argument */
-    wchar_t *run_module;    /* -m command line argument */
-    wchar_t *run_filename;  /* Trailing command line argument without -c or -m */
+    wchar_t *run_command;
+    wchar_t *run_module;
+    wchar_t *run_filename;
 
     /* --- Private fields ---------------------------- */
 
-    /* Install importlib? If set to 0, importlib is not initialized at all.
-       Needed by freeze_importlib. */
+    // Install importlib? If equals to 0, importlib is not initialized at all.
+    // Needed by freeze_importlib.
     int _install_importlib;
 
-    /* If equal to 0, stop Python initialization before the "main" phase */
+    // If equal to 0, stop Python initialization before the "main" phase.
     int _init_main;
 
-    /* If non-zero, disallow threads, subprocesses, and fork.
-       Default: 0. */
+    // If non-zero, disallow threads, subprocesses, and fork.
+    // Default: 0.
     int _isolated_interpreter;
-
-    /* The list of the original command line arguments passed to the Python
-       executable.
-
-       If 'orig_argv' list is empty and 'argv' is not a list only containing an
-       empty string, PyConfig_Read() copies 'argv' into 'orig_argv' before
-       modifying 'argv' (if 'parse_argv is non-zero).
-
-       _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */
-    PyWideStringList orig_argv;
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

Include/cpython/object.h

@@ -296,6 +296,8 @@ PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *);
 PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
 PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *);
 PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *);
+struct PyModuleDef;
+PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef(PyTypeObject *, struct PyModuleDef *);
 
 struct _Py_Identifier;
 PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);

Include/cpython/pystate.h

@@ -167,6 +167,11 @@ PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void);
 */
 PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void);
 
+/* The implementation of sys._current_exceptions()  Returns a dict mapping
+   thread id to that thread's current exception.
+*/
+PyAPI_FUNC(PyObject *) _PyThread_CurrentExceptions(void);
+
 /* Routines for advanced debuggers, requested by David Beazley.
    Don't use unless you know what you are doing! */
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void);
@@ -188,6 +193,36 @@ PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc(
 
 PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp);
 
+/* Get a copy of the current interpreter configuration.
+
+   Return 0 on success. Raise an exception and return -1 on error.
+
+   The caller must initialize 'config', using PyConfig_InitPythonConfig()
+   for example.
+
+   Python must be preinitialized to call this method.
+   The caller must hold the GIL. */
+PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy(
+    struct PyConfig *config);
+
+/* Set the configuration of the current interpreter.
+
+   This function should be called during or just after the Python
+   initialization.
+
+   Update the sys module with the new configuration. If the sys module was
+   modified directly after the Python initialization, these changes are lost.
+
+   Some configuration like faulthandler or warnoptions can be updated in the
+   configuration, but don't reconfigure Python (don't enable/disable
+   faulthandler and don't reconfigure warnings filters).
+
+   Return 0 on success. Raise an exception and return -1 on error.
+
+   The configuration should come from _PyInterpreterState_GetConfigCopy(). */
+PyAPI_FUNC(int) _PyInterpreterState_SetConfig(
+    const struct PyConfig *config);
+
 // Get the configuration of the currrent interpreter.
 // The caller must hold the GIL.
 PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void);

Include/internal/pycore_ast.h

@@ -0,0 +1,234 @@
+// File automatically generated by Parser/asdl_c.py.
+
+#ifndef Py_INTERNAL_AST_H
+#define Py_INTERNAL_AST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+struct ast_state {
+    int initialized;
+    PyObject *AST_type;
+    PyObject *Add_singleton;
+    PyObject *Add_type;
+    PyObject *And_singleton;
+    PyObject *And_type;
+    PyObject *AnnAssign_type;
+    PyObject *Assert_type;
+    PyObject *Assign_type;
+    PyObject *AsyncFor_type;
+    PyObject *AsyncFunctionDef_type;
+    PyObject *AsyncWith_type;
+    PyObject *Attribute_type;
+    PyObject *AugAssign_type;
+    PyObject *Await_type;
+    PyObject *BinOp_type;
+    PyObject *BitAnd_singleton;
+    PyObject *BitAnd_type;
+    PyObject *BitOr_singleton;
+    PyObject *BitOr_type;
+    PyObject *BitXor_singleton;
+    PyObject *BitXor_type;
+    PyObject *BoolOp_type;
+    PyObject *Break_type;
+    PyObject *Call_type;
+    PyObject *ClassDef_type;
+    PyObject *Compare_type;
+    PyObject *Constant_type;
+    PyObject *Continue_type;
+    PyObject *Del_singleton;
+    PyObject *Del_type;
+    PyObject *Delete_type;
+    PyObject *DictComp_type;
+    PyObject *Dict_type;
+    PyObject *Div_singleton;
+    PyObject *Div_type;
+    PyObject *Eq_singleton;
+    PyObject *Eq_type;
+    PyObject *ExceptHandler_type;
+    PyObject *Expr_type;
+    PyObject *Expression_type;
+    PyObject *FloorDiv_singleton;
+    PyObject *FloorDiv_type;
+    PyObject *For_type;
+    PyObject *FormattedValue_type;
+    PyObject *FunctionDef_type;
+    PyObject *FunctionType_type;
+    PyObject *GeneratorExp_type;
+    PyObject *Global_type;
+    PyObject *GtE_singleton;
+    PyObject *GtE_type;
+    PyObject *Gt_singleton;
+    PyObject *Gt_type;
+    PyObject *IfExp_type;
+    PyObject *If_type;
+    PyObject *ImportFrom_type;
+    PyObject *Import_type;
+    PyObject *In_singleton;
+    PyObject *In_type;
+    PyObject *Interactive_type;
+    PyObject *Invert_singleton;
+    PyObject *Invert_type;
+    PyObject *IsNot_singleton;
+    PyObject *IsNot_type;
+    PyObject *Is_singleton;
+    PyObject *Is_type;
+    PyObject *JoinedStr_type;
+    PyObject *LShift_singleton;
+    PyObject *LShift_type;
+    PyObject *Lambda_type;
+    PyObject *ListComp_type;
+    PyObject *List_type;
+    PyObject *Load_singleton;
+    PyObject *Load_type;
+    PyObject *LtE_singleton;
+    PyObject *LtE_type;
+    PyObject *Lt_singleton;
+    PyObject *Lt_type;
+    PyObject *MatMult_singleton;
+    PyObject *MatMult_type;
+    PyObject *Mod_singleton;
+    PyObject *Mod_type;
+    PyObject *Module_type;
+    PyObject *Mult_singleton;
+    PyObject *Mult_type;
+    PyObject *Name_type;
+    PyObject *NamedExpr_type;
+    PyObject *Nonlocal_type;
+    PyObject *NotEq_singleton;
+    PyObject *NotEq_type;
+    PyObject *NotIn_singleton;
+    PyObject *NotIn_type;
+    PyObject *Not_singleton;
+    PyObject *Not_type;
+    PyObject *Or_singleton;
+    PyObject *Or_type;
+    PyObject *Pass_type;
+    PyObject *Pow_singleton;
+    PyObject *Pow_type;
+    PyObject *RShift_singleton;
+    PyObject *RShift_type;
+    PyObject *Raise_type;
+    PyObject *Return_type;
+    PyObject *SetComp_type;
+    PyObject *Set_type;
+    PyObject *Slice_type;
+    PyObject *Starred_type;
+    PyObject *Store_singleton;
+    PyObject *Store_type;
+    PyObject *Sub_singleton;
+    PyObject *Sub_type;
+    PyObject *Subscript_type;
+    PyObject *Try_type;
+    PyObject *Tuple_type;
+    PyObject *TypeIgnore_type;
+    PyObject *UAdd_singleton;
+    PyObject *UAdd_type;
+    PyObject *USub_singleton;
+    PyObject *USub_type;
+    PyObject *UnaryOp_type;
+    PyObject *While_type;
+    PyObject *With_type;
+    PyObject *YieldFrom_type;
+    PyObject *Yield_type;
+    PyObject *__dict__;
+    PyObject *__doc__;
+    PyObject *__module__;
+    PyObject *_attributes;
+    PyObject *_fields;
+    PyObject *alias_type;
+    PyObject *annotation;
+    PyObject *arg;
+    PyObject *arg_type;
+    PyObject *args;
+    PyObject *argtypes;
+    PyObject *arguments_type;
+    PyObject *asname;
+    PyObject *ast;
+    PyObject *attr;
+    PyObject *bases;
+    PyObject *body;
+    PyObject *boolop_type;
+    PyObject *cause;
+    PyObject *cmpop_type;
+    PyObject *col_offset;
+    PyObject *comparators;
+    PyObject *comprehension_type;
+    PyObject *context_expr;
+    PyObject *conversion;
+    PyObject *ctx;
+    PyObject *decorator_list;
+    PyObject *defaults;
+    PyObject *elt;
+    PyObject *elts;
+    PyObject *end_col_offset;
+    PyObject *end_lineno;
+    PyObject *exc;
+    PyObject *excepthandler_type;
+    PyObject *expr_context_type;
+    PyObject *expr_type;
+    PyObject *finalbody;
+    PyObject *format_spec;
+    PyObject *func;
+    PyObject *generators;
+    PyObject *handlers;
+    PyObject *id;
+    PyObject *ifs;
+    PyObject *is_async;
+    PyObject *items;
+    PyObject *iter;
+    PyObject *key;
+    PyObject *keys;
+    PyObject *keyword_type;
+    PyObject *keywords;
+    PyObject *kind;
+    PyObject *kw_defaults;
+    PyObject *kwarg;
+    PyObject *kwonlyargs;
+    PyObject *left;
+    PyObject *level;
+    PyObject *lineno;
+    PyObject *lower;
+    PyObject *mod_type;
+    PyObject *module;
+    PyObject *msg;
+    PyObject *name;
+    PyObject *names;
+    PyObject *op;
+    PyObject *operand;
+    PyObject *operator_type;
+    PyObject *ops;
+    PyObject *optional_vars;
+    PyObject *orelse;
+    PyObject *posonlyargs;
+    PyObject *returns;
+    PyObject *right;
+    PyObject *simple;
+    PyObject *slice;
+    PyObject *step;
+    PyObject *stmt_type;
+    PyObject *tag;
+    PyObject *target;
+    PyObject *targets;
+    PyObject *test;
+    PyObject *type;
+    PyObject *type_comment;
+    PyObject *type_ignore_type;
+    PyObject *type_ignores;
+    PyObject *unaryop_type;
+    PyObject *upper;
+    PyObject *value;
+    PyObject *values;
+    PyObject *vararg;
+    PyObject *withitem_type;
+};
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_AST_H */
+

Include/internal/pycore_initconfig.h

@@ -44,6 +44,8 @@ struct pyruntimestate;
 #define _PyStatus_UPDATE_FUNC(err) \
     do { err.func = _PyStatus_GET_FUNC(); } while (0)
 
+PyObject* _PyErr_SetFromPyStatus(PyStatus status);
+
 /* --- PyWideStringList ------------------------------------------------ */
 
 #define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL}
@@ -150,12 +152,16 @@ extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
 extern PyStatus _PyConfig_InitPathConfig(PyConfig *config);
+extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);
 extern PyStatus _PyConfig_SetPyArgv(
     PyConfig *config,
     const _PyArgv *args);
 
+PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config);
+PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict);
+
 
 /* --- Function used for testing ---------------------------------- */
 

Include/internal/pycore_interp.h

@@ -8,10 +8,11 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-#include "pycore_atomic.h"    /* _Py_atomic_address */
-#include "pycore_gil.h"       /* struct _gil_runtime_state  */
-#include "pycore_gc.h"        /* struct _gc_runtime_state */
-#include "pycore_warnings.h"  /* struct _warnings_runtime_state */
+#include "pycore_atomic.h"        // _Py_atomic_address
+#include "pycore_ast.h"           // struct ast_state
+#include "pycore_gil.h"           // struct _gil_runtime_state
+#include "pycore_gc.h"            // struct _gc_runtime_state
+#include "pycore_warnings.h"      // struct _warnings_runtime_state
 
 struct _pending_calls {
     PyThread_type_lock lock;
@@ -258,15 +259,11 @@ struct _is {
     struct _Py_async_gen_state async_gen;
     struct _Py_context_state context;
     struct _Py_exc_state exc_state;
+
+    struct ast_state ast;
 };
 
-/* Used by _PyImport_Cleanup() */
 extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp);
-
-extern PyStatus _PyInterpreterState_SetConfig(
-    PyInterpreterState *interp,
-    const PyConfig *config);
-
 extern void _PyInterpreterState_Clear(PyThreadState *tstate);
 
 

Include/internal/pycore_pylifecycle.h

@@ -44,7 +44,7 @@ extern PyStatus _PySys_Create(
     PyObject **sysmod_p);
 extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
 extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
-extern int _PySys_InitMain(PyThreadState *tstate);
+extern int _PySys_UpdateConfig(PyThreadState *tstate);
 extern PyStatus _PyExc_Init(PyThreadState *tstate);
 extern PyStatus _PyErr_Init(void);
 extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod);
@@ -84,7 +84,7 @@ extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
 extern void _PyTraceMalloc_Fini(void);
 extern void _PyWarnings_Fini(PyInterpreterState *interp);
-extern void _PyAST_Fini(PyThreadState *tstate);
+extern void _PyAST_Fini(PyInterpreterState *interp);
 
 extern PyStatus _PyGILState_Init(PyThreadState *tstate);
 extern void _PyGILState_Fini(PyThreadState *tstate);

Include/internal/pycore_unionobject.h

@@ -10,6 +10,7 @@ extern "C" {
 
 PyAPI_FUNC(PyObject *) _Py_Union(PyObject *args);
 PyAPI_DATA(PyTypeObject) _Py_UnionType;
+PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject* self, PyObject* param);
 
 #ifdef __cplusplus
 }

Include/methodobject.h

@@ -73,7 +73,7 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *,
 
 #define METH_COEXIST   0x0040
 
-#ifndef Py_LIMITED_API
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03100000
 #define METH_FASTCALL  0x0080
 #endif
 

Include/modsupport.h

@@ -136,7 +136,15 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords(
 void _PyArg_Fini(void);
 #endif   /* Py_LIMITED_API */
 
-PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *);
+// Add an attribute with name 'name' and value 'obj' to the module 'mod.
+// On success, return 0 on success.
+// On error, raise an exception and return -1.
+PyAPI_FUNC(int) PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value);
+
+// Similar to PyModule_AddObjectRef() but steal a reference to 'obj'
+// (Py_DECREF(obj)) on success (if it returns 0).
+PyAPI_FUNC(int) PyModule_AddObject(PyObject *mod, const char *, PyObject *value);
+
 PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long);
 PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *);
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000

Include/object.h

@@ -526,6 +526,32 @@ they can have object code that is not dependent on Python compilation flags.
 PyAPI_FUNC(void) Py_IncRef(PyObject *);
 PyAPI_FUNC(void) Py_DecRef(PyObject *);
 
+// Create a new strong reference to an object:
+// increment the reference count of the object and return the object.
+PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj);
+
+// Similar to Py_NewRef(), but the object can be NULL.
+PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj);
+
+static inline PyObject* _Py_NewRef(PyObject *obj)
+{
+    Py_INCREF(obj);
+    return obj;
+}
+
+static inline PyObject* _Py_XNewRef(PyObject *obj)
+{
+    Py_XINCREF(obj);
+    return obj;
+}
+
+// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI.
+// Names overriden with macros by static inline functions for best
+// performances.
+#define Py_NewRef(obj) _Py_NewRef(obj)
+#define Py_XNewRef(obj) _Py_XNewRef(obj)
+
+
 /*
 _Py_NoneStruct is an object of undefined type which can be used in contexts
 where NULL (nil) is not suitable (since NULL often means 'error').
@@ -536,7 +562,7 @@ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */
 #define Py_None (&_Py_NoneStruct)
 
 /* Macro for returning Py_None from a function */
-#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
+#define Py_RETURN_NONE return Py_NewRef(Py_None)
 
 /*
 Py_NotImplemented is a singleton used to signal that an operation is
@@ -546,8 +572,7 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */
 #define Py_NotImplemented (&_Py_NotImplementedStruct)
 
 /* Macro for returning Py_NotImplemented from a function */
-#define Py_RETURN_NOTIMPLEMENTED \
-    return Py_INCREF(Py_NotImplemented), Py_NotImplemented
+#define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented)
 
 /* Rich comparison opcodes */
 #define Py_LT 0

Include/patchlevel.h

@@ -20,10 +20,10 @@
 #define PY_MINOR_VERSION        10
 #define PY_MICRO_VERSION        0
 #define PY_RELEASE_LEVEL        PY_RELEASE_LEVEL_ALPHA
-#define PY_RELEASE_SERIAL       1
+#define PY_RELEASE_SERIAL       2
 
 /* Version as a string */
-#define PY_VERSION              "3.10.0a1+"
+#define PY_VERSION              "3.10.0a2+"
 /*--end constants--*/
 
 /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2.

Lib/_osx_support.py

@@ -110,6 +110,26 @@ def _get_system_version():
 
     return _SYSTEM_VERSION
 
+_SYSTEM_VERSION_TUPLE = None
+def _get_system_version_tuple():
+    """
+    Return the macOS system version as a tuple
+
+    The return value is safe to use to compare
+    two version numbers.
+    """
+    global _SYSTEM_VERSION_TUPLE
+    if _SYSTEM_VERSION_TUPLE is None:
+        osx_version = _get_system_version()
+        if osx_version:
+            try:
+                _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.'))
+            except ValueError:
+                _SYSTEM_VERSION_TUPLE = ()
+
+    return _SYSTEM_VERSION_TUPLE
+   
+
 def _remove_original_values(_config_vars):
     """Remove original unmodified values for testing"""
     # This is needed for higher-level cross-platform tests of get_platform.
@@ -132,14 +152,18 @@ def _supports_universal_builds():
     # builds, in particular -isysroot and -arch arguments to the compiler. This
     # is in support of allowing 10.4 universal builds to run on 10.3.x systems.
 
-    osx_version = _get_system_version()
-    if osx_version:
-        try:
-            osx_version = tuple(int(i) for i in osx_version.split('.'))
-        except ValueError:
-            osx_version = ''
+    osx_version = _get_system_version_tuple()
     return bool(osx_version >= (10, 4)) if osx_version else False
 
+def _supports_arm64_builds():
+    """Returns True if arm64 builds are supported on this system"""
+    # There are two sets of systems supporting macOS/arm64 builds:
+    # 1. macOS 11 and later, unconditionally
+    # 2. macOS 10.15 with Xcode 12.2 or later
+    # For now the second category is ignored.
+    osx_version = _get_system_version_tuple()
+    return osx_version >= (11, 0) if osx_version else False
+
 
 def _find_appropriate_compiler(_config_vars):
     """Find appropriate C compiler for extension module builds"""
@@ -331,6 +355,12 @@ def compiler_fixup(compiler_so, cc_args):
             except ValueError:
                 break
 
+    elif not _supports_arm64_builds():
+        # Look for "-arch arm64" and drop that
+        for idx in range(len(compiler_so)):
+            if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64":
+                del compiler_so[idx:idx+2]
+
     if 'ARCHFLAGS' in os.environ and not stripArch:
         # User specified different -arch flags in the environ,
         # see also distutils.sysconfig
@@ -481,6 +511,8 @@ def get_platform_osx(_config_vars, osname, release, machine):
 
             if len(archs) == 1:
                 machine = archs[0]
+            elif archs == ('arm64', 'x86_64'):
+                machine = 'universal2'
             elif archs == ('i386', 'ppc'):
                 machine = 'fat'
             elif archs == ('i386', 'x86_64'):

Lib/asyncio/base_futures.py

@@ -1,6 +1,7 @@
 __all__ = ()
 
 import reprlib
+from _thread import get_ident
 
 from . import format_helpers
 
@@ -41,6 +42,16 @@ def _format_callbacks(cb):
     return f'cb=[{cb}]'
 
 
+# bpo-42183: _repr_running is needed for repr protection
+# when a Future or Task result contains itself directly or indirectly.
+# The logic is borrowed from @reprlib.recursive_repr decorator.
+# Unfortunately, the direct decorator usage is impossible because of
+# AttributeError: '_asyncio.Task' object has no attribute '__module__' error.
+#
+# After fixing this thing we can return to the decorator based approach.
+_repr_running = set()
+
+
 def _future_repr_info(future):
     # (Future) -> str
     """helper function for Future.__repr__"""
@@ -49,9 +60,17 @@ def _future_repr_info(future):
         if future._exception is not None:
             info.append(f'exception={future._exception!r}')
         else:
-            # use reprlib to limit the length of the output, especially
-            # for very long strings
-            result = reprlib.repr(future._result)
+            key = id(future), get_ident()
+            if key in _repr_running:
+                result = '...'
+            else:
+                _repr_running.add(key)
+                try:
+                    # use reprlib to limit the length of the output, especially
+                    # for very long strings
+                    result = reprlib.repr(future._result)
+                finally:
+                    _repr_running.discard(key)
             info.append(f'result={result}')
     if future._callbacks:
         info.append(_format_callbacks(future._callbacks))

Lib/contextlib.py

@@ -80,6 +80,22 @@ class ContextDecorator(object):
         return inner
 
 
+class AsyncContextDecorator(object):
+    "A base class or mixin that enables async context managers to work as decorators."
+
+    def _recreate_cm(self):
+        """Return a recreated instance of self.
+        """
+        return self
+
+    def __call__(self, func):
+        @wraps(func)
+        async def inner(*args, **kwds):
+            async with self._recreate_cm():
+                return await func(*args, **kwds)
+        return inner
+
+
 class _GeneratorContextManagerBase:
     """Shared functionality for @contextmanager and @asynccontextmanager."""
 
@@ -167,9 +183,16 @@ class _GeneratorContextManager(_GeneratorContextManagerBase,
 
 
 class _AsyncGeneratorContextManager(_GeneratorContextManagerBase,
-                                    AbstractAsyncContextManager):
+                                    AbstractAsyncContextManager,
+                                    AsyncContextDecorator):
     """Helper for @asynccontextmanager."""
 
+    def _recreate_cm(self):
+        # _AGCM instances are one-shot context managers, so the
+        # ACM must be recreated each time a decorated function is
+        # called
+        return self.__class__(self.func, self.args, self.kwds)
+
     async def __aenter__(self):
         try:
             return await self.gen.__anext__()
@@ -681,7 +704,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager):
         return received_exc and suppressed_exc
 
 
-class nullcontext(AbstractContextManager):
+class nullcontext(AbstractContextManager, AbstractAsyncContextManager):
     """Context manager that does no additional processing.
 
     Used as a stand-in for a normal context manager, when a particular
@@ -700,3 +723,9 @@ class nullcontext(AbstractContextManager):
 
     def __exit__(self, *excinfo):
         pass
+
+    async def __aenter__(self):
+        return self.enter_result
+
+    async def __aexit__(self, *excinfo):
+        pass

Lib/ctypes/macholib/dyld.py

@@ -6,6 +6,11 @@ import os
 from ctypes.macholib.framework import framework_info
 from ctypes.macholib.dylib import dylib_info
 from itertools import *
+try:
+    from _ctypes import _dyld_shared_cache_contains_path
+except ImportError:
+    def _dyld_shared_cache_contains_path(*args):
+        raise NotImplementedError
 
 __all__ = [
     'dyld_find', 'framework_find',
@@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None):
                 dyld_executable_path_search(name, executable_path),
                 dyld_default_search(name, env),
             ), env):
+
         if os.path.isfile(path):
             return path
+        try:
+            if _dyld_shared_cache_contains_path(path):
+                return path
+        except NotImplementedError:
+            pass
+
     raise ValueError("dylib %s could not be found" % (name,))
 
 def framework_find(fn, executable_path=None, env=None):

Lib/ctypes/test/test_macholib.py

@@ -45,19 +45,22 @@ def find_lib(name):
 class MachOTest(unittest.TestCase):
     @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
     def test_find(self):
-
-        self.assertEqual(find_lib('pthread'),
-                             '/usr/lib/libSystem.B.dylib')
+        # On Mac OS 11, system dylibs are only present in the shared cache,
+        # so symlinks like libpthread.dylib -> libSystem.B.dylib will not
+        # be resolved by dyld_find
+        self.assertIn(find_lib('pthread'),
+                              ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))
 
         result = find_lib('z')
         # Issue #21093: dyld default search path includes $HOME/lib and
         # /usr/local/lib before /usr/lib, which caused test failures if
         # a local copy of libz exists in one of them. Now ignore the head
         # of the path.
-        self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
+        self.assertRegex(result, r".*/lib/libz.*\.dylib")
 
-        self.assertEqual(find_lib('IOKit'),
-                             '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
+        self.assertIn(find_lib('IOKit'),
+                              ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
+                              '/System/Library/Frameworks/IOKit.framework/IOKit'))
 
 if __name__ == "__main__":
     unittest.main()

Lib/distutils/tests/test_build_ext.py

@@ -493,7 +493,7 @@ class BuildExtTestCase(TempdirManager,
         # format the target value as defined in the Apple
         # Availability Macros.  We can't use the macro names since
         # at least one value we test with will not exist yet.
-        if target[1] < 10:
+        if target[:2] < (10, 10):
             # for 10.1 through 10.9.x -> "10n0"
             target = '%02d%01d0' % target
         else:

Lib/doctest.py

@@ -222,13 +222,17 @@ def _load_testfile(filename, package, module_relative, encoding):
     if module_relative:
         package = _normalize_module(package, 3)
         filename = _module_relative_path(package, filename)
-        if getattr(package, '__loader__', None) is not None:
-            if hasattr(package.__loader__, 'get_data'):
-                file_contents = package.__loader__.get_data(filename)
-                file_contents = file_contents.decode(encoding)
-                # get_data() opens files as 'rb', so one must do the equivalent
-                # conversion as universal newlines would do.
-                return _newline_convert(file_contents), filename
+        if (loader := getattr(package, '__loader__', None)) is None:
+            try:
+                loader = package.__spec__.loader
+            except AttributeError:
+                pass
+        if hasattr(loader, 'get_data'):
+            file_contents = loader.get_data(filename)
+            file_contents = file_contents.decode(encoding)
+            # get_data() opens files as 'rb', so one must do the equivalent
+            # conversion as universal newlines would do.
+            return _newline_convert(file_contents), filename
     with open(filename, encoding=encoding) as f:
         return f.read(), filename
 

Lib/inspect.py

@@ -707,10 +707,13 @@ def getsourcefile(object):
     if os.path.exists(filename):
         return filename
     # only return a non-existent filename if the module has a PEP 302 loader
-    if getattr(getmodule(object, filename), '__loader__', None) is not None:
+    module = getmodule(object, filename)
+    if getattr(module, '__loader__', None) is not None:
+        return filename
+    elif getattr(getattr(module, "__spec__", None), "loader", None) is not None:
         return filename
     # or it is in the linecache
-    if filename in linecache.cache:
+    elif filename in linecache.cache:
         return filename
 
 def getabsfile(object, _filename=None):

Lib/linecache.py

@@ -165,9 +165,14 @@ def lazycache(filename, module_globals):
     if not filename or (filename.startswith('<') and filename.endswith('>')):
         return False
     # Try for a __loader__, if available
-    if module_globals and '__loader__' in module_globals:
-        name = module_globals.get('__name__')
-        loader = module_globals['__loader__']
+    if module_globals and '__name__' in module_globals:
+        name = module_globals['__name__']
+        if (loader := module_globals.get('__loader__')) is None:
+            if spec := module_globals.get('__spec__'):
+                try:
+                    loader = spec.loader
+                except AttributeError:
+                    pass
         get_source = getattr(loader, 'get_source', None)
 
         if name and get_source:

Lib/logging/__init__.py

@@ -1148,6 +1148,10 @@ class FileHandler(StreamHandler):
         self.encoding = encoding
         self.errors = errors
         self.delay = delay
+        # bpo-26789: FileHandler keeps a reference to the builtin open()
+        # function to be able to open or reopen the file during Python
+        # finalization.
+        self._builtin_open = open
         if delay:
             #We don't open the stream, but we still need to call the
             #Handler constructor to set level, formatter, lock etc.
@@ -1183,8 +1187,9 @@ class FileHandler(StreamHandler):
         Open the current base file with the (original) mode and encoding.
         Return the resulting stream.
         """
-        return open(self.baseFilename, self.mode, encoding=self.encoding,
-                    errors=self.errors)
+        open_func = self._builtin_open
+        return open_func(self.baseFilename, self.mode,
+                         encoding=self.encoding, errors=self.errors)
 
     def emit(self, record):
         """

Lib/plistlib.py

@@ -477,7 +477,7 @@ class _BinaryPlistParser:
             return self._read_object(top_object)
 
         except (OSError, IndexError, struct.error, OverflowError,
-                UnicodeDecodeError):
+                ValueError):
             raise InvalidFileException()
 
     def _get_size(self, tokenL):
@@ -493,7 +493,7 @@ class _BinaryPlistParser:
     def _read_ints(self, n, size):
         data = self._fp.read(size * n)
         if size in _BINARY_FORMAT:
-            return struct.unpack('>' + _BINARY_FORMAT[size] * n, data)
+            return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data)
         else:
             if not size or len(data) != size * n:
                 raise InvalidFileException()
@@ -553,14 +553,22 @@ class _BinaryPlistParser:
         elif tokenH == 0x40:  # data
             s = self._get_size(tokenL)
             result = self._fp.read(s)
+            if len(result) != s:
+                raise InvalidFileException()
 
         elif tokenH == 0x50:  # ascii string
             s = self._get_size(tokenL)
-            result =  self._fp.read(s).decode('ascii')
+            data = self._fp.read(s)
+            if len(data) != s:
+                raise InvalidFileException()
+            result = data.decode('ascii')
 
         elif tokenH == 0x60:  # unicode string
-            s = self._get_size(tokenL)
-            result = self._fp.read(s * 2).decode('utf-16be')
+            s = self._get_size(tokenL) * 2
+            data = self._fp.read(s)
+            if len(data) != s:
+                raise InvalidFileException()
+            result = data.decode('utf-16be')
 
         elif tokenH == 0x80:  # UID
             # used by Key-Archiver plist files
@@ -585,9 +593,11 @@ class _BinaryPlistParser:
             obj_refs = self._read_refs(s)
             result = self._dict_type()
             self._objects[ref] = result
-            for k, o in zip(key_refs, obj_refs):
-                result[self._read_object(k)] = self._read_object(o)
-
+            try:
+                for k, o in zip(key_refs, obj_refs):
+                    result[self._read_object(k)] = self._read_object(o)
+            except TypeError:
+                raise InvalidFileException()
         else:
             raise InvalidFileException()
 
@@ -601,7 +611,7 @@ def _count_to_size(count):
     elif count < 1 << 16:
         return 2
 
-    elif count << 1 << 32:
+    elif count < 1 << 32:
         return 4
 
     else:

Lib/pydoc_data/topics.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Autogenerated by Sphinx on Mon Oct  5 18:27:28 2020
+# Autogenerated by Sphinx on Tue Nov  3 00:01:01 2020
 topics = {'assert': 'The "assert" statement\n'
            '**********************\n'
            '\n'
@@ -433,11 +433,9 @@ topics = {'assert': 'The "assert" statement\n'
           '\n'
           'Execution of Python coroutines can be suspended and resumed at '
           'many\n'
-          'points (see *coroutine*).  Inside the body of a coroutine '
-          'function,\n'
-          '"await" and "async" identifiers become reserved keywords; "await"\n'
-          'expressions, "async for" and "async with" can only be used in\n'
-          'coroutine function bodies.\n'
+          'points (see *coroutine*). "await" expressions, "async for" and '
+          '"async\n'
+          'with" can only be used in the body of a coroutine function.\n'
           '\n'
           'Functions defined with "async def" syntax are always coroutine\n'
           'functions, even if they do not contain "await" or "async" '
@@ -453,6 +451,10 @@ topics = {'assert': 'The "assert" statement\n'
           '       do_stuff()\n'
           '       await some_coroutine()\n'
           '\n'
+          'Changed in version 3.7: "await" and "async" are now keywords;\n'
+          'previously they were only treated as such inside the body of a\n'
+          'coroutine function.\n'
+          '\n'
           '\n'
           'The "async for" statement\n'
           '=========================\n'
@@ -700,6 +702,11 @@ topics = {'assert': 'The "assert" statement\n'
                      'syntax or\n'
                      '     built-in functions. See Special method lookup.\n'
                      '\n'
+                     '   For certain sensitive attribute accesses, raises an '
+                     'auditing event\n'
+                     '   "object.__getattr__" with arguments "obj" and '
+                     '"name".\n'
+                     '\n'
                      'object.__setattr__(self, name, value)\n'
                      '\n'
                      '   Called when an attribute assignment is attempted.  '
@@ -716,6 +723,11 @@ topics = {'assert': 'The "assert" statement\n'
                      'for example,\n'
                      '   "object.__setattr__(self, name, value)".\n'
                      '\n'
+                     '   For certain sensitive attribute assignments, raises '
+                     'an auditing\n'
+                     '   event "object.__setattr__" with arguments "obj", '
+                     '"name", "value".\n'
+                     '\n'
                      'object.__delattr__(self, name)\n'
                      '\n'
                      '   Like "__setattr__()" but for attribute deletion '
@@ -724,6 +736,11 @@ topics = {'assert': 'The "assert" statement\n'
                      'obj.name" is\n'
                      '   meaningful for the object.\n'
                      '\n'
+                     '   For certain sensitive attribute deletions, raises an '
+                     'auditing event\n'
+                     '   "object.__delattr__" with arguments "obj" and '
+                     '"name".\n'
+                     '\n'
                      'object.__dir__(self)\n'
                      '\n'
                      '   Called when "dir()" is called on the object. A '
@@ -1464,8 +1481,8 @@ topics = {'assert': 'The "assert" statement\n'
                    '\n'
                    '   Called when the instance is “called” as a function; if '
                    'this method\n'
-                   '   is defined, "x(arg1, arg2, ...)" is a shorthand for\n'
-                   '   "x.__call__(arg1, arg2, ...)".\n',
+                   '   is defined, "x(arg1, arg2, ...)" roughly translates to\n'
+                   '   "type(x).__call__(x, arg1, ...)".\n',
  'calls': 'Calls\n'
           '*****\n'
           '\n'
@@ -2766,20 +2783,11 @@ topics = {'assert': 'The "assert" statement\n'
              'parameter list.  These annotations can be any valid Python '
              'expression.\n'
              'The presence of annotations does not change the semantics of a\n'
-             'function.  The annotation values are available as values of a\n'
+             'function.  The annotation values are available as string values '
+             'in a\n'
              'dictionary keyed by the parameters’ names in the '
              '"__annotations__"\n'
-             'attribute of the function object.  If the "annotations" import '
-             'from\n'
-             '"__future__" is used, annotations are preserved as strings at '
-             'runtime\n'
-             'which enables postponed evaluation.  Otherwise, they are '
-             'evaluated\n'
-             'when the function definition is executed.  In this case '
-             'annotations\n'
-             'may be evaluated in a different order than they appear in the '
-             'source\n'
-             'code.\n'
+             'attribute of the function object.\n'
              '\n'
              'It is also possible to create anonymous functions (functions not '
              'bound\n'
@@ -2949,12 +2957,9 @@ topics = {'assert': 'The "assert" statement\n'
              '\n'
              'Execution of Python coroutines can be suspended and resumed at '
              'many\n'
-             'points (see *coroutine*).  Inside the body of a coroutine '
-             'function,\n'
-             '"await" and "async" identifiers become reserved keywords; '
-             '"await"\n'
-             'expressions, "async for" and "async with" can only be used in\n'
-             'coroutine function bodies.\n'
+             'points (see *coroutine*). "await" expressions, "async for" and '
+             '"async\n'
+             'with" can only be used in the body of a coroutine function.\n'
              '\n'
              'Functions defined with "async def" syntax are always coroutine\n'
              'functions, even if they do not contain "await" or "async" '
@@ -2970,6 +2975,10 @@ topics = {'assert': 'The "assert" statement\n'
              '       do_stuff()\n'
              '       await some_coroutine()\n'
              '\n'
+             'Changed in version 3.7: "await" and "async" are now keywords;\n'
+             'previously they were only treated as such inside the body of a\n'
+             'coroutine function.\n'
+             '\n'
              '\n'
              'The "async for" statement\n'
              '-------------------------\n'
@@ -3461,16 +3470,21 @@ topics = {'assert': 'The "assert" statement\n'
                   '   on the value to determine if the result is true or '
                   'false.\n'
                   '\n'
-                  '   By default, "__ne__()" delegates to "__eq__()" and '
-                  'inverts the\n'
-                  '   result unless it is "NotImplemented".  There are no '
-                  'other implied\n'
-                  '   relationships among the comparison operators, for '
-                  'example, the\n'
-                  '   truth of "(x<y or x==y)" does not imply "x<=y". To '
-                  'automatically\n'
-                  '   generate ordering operations from a single root '
-                  'operation, see\n'
+                  '   By default, "object" implements "__eq__()" by using '
+                  '"is", returning\n'
+                  '   "NotImplemented" in the case of a false comparison: '
+                  '"True if x is y\n'
+                  '   else NotImplemented". For "__ne__()", by default it '
+                  'delegates to\n'
+                  '   "__eq__()" and inverts the result unless it is '
+                  '"NotImplemented".\n'
+                  '   There are no other implied relationships among the '
+                  'comparison\n'
+                  '   operators or default implementations; for example, the '
+                  'truth of\n'
+                  '   "(x<y or x==y)" does not imply "x<=y". To automatically '
+                  'generate\n'
+                  '   ordering operations from a single root operation, see\n'
                   '   "functools.total_ordering()".\n'
                   '\n'
                   '   See the paragraph on "__hash__()" for some important '
@@ -5859,20 +5873,11 @@ topics = {'assert': 'The "assert" statement\n'
              'parameter list.  These annotations can be any valid Python '
              'expression.\n'
              'The presence of annotations does not change the semantics of a\n'
-             'function.  The annotation values are available as values of a\n'
+             'function.  The annotation values are available as string values '
+             'in a\n'
              'dictionary keyed by the parameters’ names in the '
              '"__annotations__"\n'
-             'attribute of the function object.  If the "annotations" import '
-             'from\n'
-             '"__future__" is used, annotations are preserved as strings at '
-             'runtime\n'
-             'which enables postponed evaluation.  Otherwise, they are '
-             'evaluated\n'
-             'when the function definition is executed.  In this case '
-             'annotations\n'
-             'may be evaluated in a different order than they appear in the '
-             'source\n'
-             'code.\n'
+             'attribute of the function object.\n'
              '\n'
              'It is also possible to create anonymous functions (functions not '
              'bound\n'
@@ -6395,8 +6400,8 @@ topics = {'assert': 'The "assert" statement\n'
            '\n'
            '* other future statements.\n'
            '\n'
-           'The only feature in Python 3.7 that requires using the future\n'
-           'statement is "annotations".\n'
+           'The only feature that requires using the future statement is\n'
+           '"annotations" (see **PEP 563**).\n'
            '\n'
            'All historical features enabled by the future statement are still\n'
            'recognized by Python 3.  The list includes "absolute_import",\n'
@@ -8242,16 +8247,21 @@ topics = {'assert': 'The "assert" statement\n'
                  '   on the value to determine if the result is true or '
                  'false.\n'
                  '\n'
-                 '   By default, "__ne__()" delegates to "__eq__()" and '
-                 'inverts the\n'
-                 '   result unless it is "NotImplemented".  There are no other '
-                 'implied\n'
-                 '   relationships among the comparison operators, for '
-                 'example, the\n'
-                 '   truth of "(x<y or x==y)" does not imply "x<=y". To '
-                 'automatically\n'
-                 '   generate ordering operations from a single root '
-                 'operation, see\n'
+                 '   By default, "object" implements "__eq__()" by using "is", '
+                 'returning\n'
+                 '   "NotImplemented" in the case of a false comparison: "True '
+                 'if x is y\n'
+                 '   else NotImplemented". For "__ne__()", by default it '
+                 'delegates to\n'
+                 '   "__eq__()" and inverts the result unless it is '
+                 '"NotImplemented".\n'
+                 '   There are no other implied relationships among the '
+                 'comparison\n'
+                 '   operators or default implementations; for example, the '
+                 'truth of\n'
+                 '   "(x<y or x==y)" does not imply "x<=y". To automatically '
+                 'generate\n'
+                 '   ordering operations from a single root operation, see\n'
                  '   "functools.total_ordering()".\n'
                  '\n'
                  '   See the paragraph on "__hash__()" for some important '
@@ -8481,6 +8491,10 @@ topics = {'assert': 'The "assert" statement\n'
                  'syntax or\n'
                  '     built-in functions. See Special method lookup.\n'
                  '\n'
+                 '   For certain sensitive attribute accesses, raises an '
+                 'auditing event\n'
+                 '   "object.__getattr__" with arguments "obj" and "name".\n'
+                 '\n'
                  'object.__setattr__(self, name, value)\n'
                  '\n'
                  '   Called when an attribute assignment is attempted.  This '
@@ -8497,6 +8511,11 @@ topics = {'assert': 'The "assert" statement\n'
                  'example,\n'
                  '   "object.__setattr__(self, name, value)".\n'
                  '\n'
+                 '   For certain sensitive attribute assignments, raises an '
+                 'auditing\n'
+                 '   event "object.__setattr__" with arguments "obj", "name", '
+                 '"value".\n'
+                 '\n'
                  'object.__delattr__(self, name)\n'
                  '\n'
                  '   Like "__setattr__()" but for attribute deletion instead '
@@ -8505,6 +8524,10 @@ topics = {'assert': 'The "assert" statement\n'
                  'obj.name" is\n'
                  '   meaningful for the object.\n'
                  '\n'
+                 '   For certain sensitive attribute deletions, raises an '
+                 'auditing event\n'
+                 '   "object.__delattr__" with arguments "obj" and "name".\n'
+                 '\n'
                  'object.__dir__(self)\n'
                  '\n'
                  '   Called when "dir()" is called on the object. A sequence '
@@ -9298,8 +9321,8 @@ topics = {'assert': 'The "assert" statement\n'
                  '\n'
                  '   Called when the instance is “called” as a function; if '
                  'this method\n'
-                 '   is defined, "x(arg1, arg2, ...)" is a shorthand for\n'
-                 '   "x.__call__(arg1, arg2, ...)".\n'
+                 '   is defined, "x(arg1, arg2, ...)" roughly translates to\n'
+                 '   "type(x).__call__(x, arg1, ...)".\n'
                  '\n'
                  '\n'
                  'Emulating container types\n'
@@ -11054,9 +11077,10 @@ topics = {'assert': 'The "assert" statement\n'
  'subscriptions': 'Subscriptions\n'
                   '*************\n'
                   '\n'
-                  'A subscription selects an item of a sequence (string, tuple '
-                  'or list)\n'
-                  'or mapping (dictionary) object:\n'
+                  'Subscription of a sequence (string, tuple or list) or '
+                  'mapping\n'
+                  '(dictionary) object usually selects an item from the '
+                  'collection:\n'
                   '\n'
                   '   subscription ::= primary "[" expression_list "]"\n'
                   '\n'
@@ -11107,7 +11131,13 @@ topics = {'assert': 'The "assert" statement\n'
                   '\n'
                   'A string’s items are characters.  A character is not a '
                   'separate data\n'
-                  'type but a string of exactly one character.\n',
+                  'type but a string of exactly one character.\n'
+                  '\n'
+                  'Subscription of certain *classes* or *types* creates a '
+                  'generic alias.\n'
+                  'In this case, user-defined classes can support subscription '
+                  'by\n'
+                  'providing a "__class_getitem__()" classmethod.\n',
  'truth': 'Truth Value Testing\n'
           '*******************\n'
           '\n'
@@ -11353,6 +11383,27 @@ topics = {'assert': 'The "assert" statement\n'
           'representation\n'
           '   in computers.\n'
           '\n'
+          '   The string representations of the numeric classes, computed by\n'
+          '   "__repr__()" and "__str__()", have the following properties:\n'
+          '\n'
+          '   * They are valid numeric literals which, when passed to their '
+          'class\n'
+          '     constructor, produce an object having the value of the '
+          'original\n'
+          '     numeric.\n'
+          '\n'
+          '   * The representation is in base 10, when possible.\n'
+          '\n'
+          '   * Leading zeros, possibly excepting a single zero before a '
+          'decimal\n'
+          '     point, are not shown.\n'
+          '\n'
+          '   * Trailing zeros, possibly excepting a single zero after a '
+          'decimal\n'
+          '     point, are not shown.\n'
+          '\n'
+          '   * A sign is shown only when the number is negative.\n'
+          '\n'
           '   Python distinguishes between integers, floating point numbers, '
           'and\n'
           '   complex numbers:\n'
@@ -12404,6 +12455,21 @@ topics = {'assert': 'The "assert" statement\n'
                  'positional\n'
                  '   argument and a possibly empty set of keyword arguments.\n'
                  '\n'
+                 '   Dictionaries can be created by several means:\n'
+                 '\n'
+                 '   * Use a comma-separated list of "key: value" pairs within '
+                 'braces:\n'
+                 '     "{\'jack\': 4098, \'sjoerd\': 4127}" or "{4098: '
+                 "'jack', 4127:\n"
+                 '     \'sjoerd\'}"\n'
+                 '\n'
+                 '   * Use a dict comprehension: "{}", "{x: x ** 2 for x in '
+                 'range(10)}"\n'
+                 '\n'
+                 '   * Use the type constructor: "dict()", "dict([(\'foo\', '
+                 "100), ('bar',\n"
+                 '     200)])", "dict(foo=100, bar=200)"\n'
+                 '\n'
                  '   If no positional argument is given, an empty dictionary '
                  'is created.\n'
                  '   If a positional argument is given and it is a mapping '

Lib/shutil.py

@@ -711,7 +711,7 @@ def rmtree(path, ignore_errors=False, onerror=None):
         try:
             fd = os.open(path, os.O_RDONLY)
         except Exception:
-            onerror(os.lstat, path, sys.exc_info())
+            onerror(os.open, path, sys.exc_info())
             return
         try:
             if os.path.samestat(orig_st, os.fstat(fd)):

Lib/site.py

@@ -105,8 +105,15 @@ def makepath(*paths):
 def abs_paths():
     """Set all module __file__ and __cached__ attributes to an absolute path"""
     for m in set(sys.modules.values()):
-        if (getattr(getattr(m, '__loader__', None), '__module__', None) not in
-                ('_frozen_importlib', '_frozen_importlib_external')):
+        loader_module = None
+        try:
+            loader_module = m.__loader__.__module__
+        except AttributeError:
+            try:
+                loader_module = m.__spec__.loader.__module__
+            except AttributeError:
+                pass
+        if loader_module not in {'_frozen_importlib', '_frozen_importlib_external'}:
             continue   # don't mess with a PEP 302-supplied __file__
         try:
             m.__file__ = os.path.abspath(m.__file__)

Lib/socketserver.py

@@ -128,7 +128,6 @@ import selectors
 import os
 import sys
 import threading
-import contextlib
 from io import BufferedIOBase
 from time import monotonic as time
 
@@ -629,55 +628,6 @@ if hasattr(os, "fork"):
             self.collect_children(blocking=self.block_on_close)
 
 
-class _Threads(list):
-    """
-    Joinable list of all non-daemon threads.
-    """
-    def __init__(self):
-        self._lock = threading.Lock()
-
-    def append(self, thread):
-        if thread.daemon:
-            return
-        with self._lock:
-            super().append(thread)
-
-    def remove(self, thread):
-        with self._lock:
-            # should not happen, but safe to ignore
-            with contextlib.suppress(ValueError):
-                super().remove(thread)
-
-    def remove_current(self):
-        """Remove a current non-daemon thread."""
-        thread = threading.current_thread()
-        if not thread.daemon:
-            self.remove(thread)
-
-    def pop_all(self):
-        with self._lock:
-            self[:], result = [], self[:]
-        return result
-
-    def join(self):
-        for thread in self.pop_all():
-            thread.join()
-
-
-class _NoThreads:
-    """
-    Degenerate version of _Threads.
-    """
-    def append(self, thread):
-        pass
-
-    def join(self):
-        pass
-
-    def remove_current(self):
-        pass
-
-
 class ThreadingMixIn:
     """Mix-in class to handle each request in a new thread."""
 
@@ -686,9 +636,9 @@ class ThreadingMixIn:
     daemon_threads = False
     # If true, server_close() waits until all non-daemonic threads terminate.
     block_on_close = True
-    # Threads object
+    # For non-daemonic threads, list of threading.Threading objects
     # used by server_close() to wait for all threads completion.
-    _threads = _NoThreads()
+    _threads = None
 
     def process_request_thread(self, request, client_address):
         """Same as in BaseServer but as a thread.
@@ -701,24 +651,27 @@ class ThreadingMixIn:
         except Exception:
             self.handle_error(request, client_address)
         finally:
-            try:
-                self.shutdown_request(request)
-            finally:
-                self._threads.remove_current()
+            self.shutdown_request(request)
 
     def process_request(self, request, client_address):
         """Start a new thread to process the request."""
-        if self.block_on_close:
-            vars(self).setdefault('_threads', _Threads())
         t = threading.Thread(target = self.process_request_thread,
                              args = (request, client_address))
         t.daemon = self.daemon_threads
-        self._threads.append(t)
+        if not t.daemon and self.block_on_close:
+            if self._threads is None:
+                self._threads = []
+            self._threads.append(t)
         t.start()
 
     def server_close(self):
         super().server_close()
-        self._threads.join()
+        if self.block_on_close:
+            threads = self._threads
+            self._threads = None
+            if threads:
+                for thread in threads:
+                    thread.join()
 
 
 if hasattr(os, "fork"):

Lib/test/_test_embed_set_config.py

@@ -0,0 +1,261 @@
+# bpo-42260: Test _PyInterpreterState_GetConfigCopy()
+# and _PyInterpreterState_SetConfig().
+#
+# Test run in a subinterpreter since set_config(get_config())
+# does reset sys attributes to their state of the Python startup
+# (before the site module is run).
+
+import _testinternalcapi
+import os
+import sys
+import unittest
+
+
+MS_WINDOWS = (os.name == 'nt')
+MAX_HASH_SEED = 4294967295
+
+class SetConfigTests(unittest.TestCase):
+    def setUp(self):
+        self.old_config = _testinternalcapi.get_config()
+        self.sys_copy = dict(sys.__dict__)
+
+    def tearDown(self):
+        _testinternalcapi.set_config(self.old_config)
+        sys.__dict__.clear()
+        sys.__dict__.update(self.sys_copy)
+
+    def set_config(self, **kwargs):
+        _testinternalcapi.set_config(self.old_config | kwargs)
+
+    def check(self, **kwargs):
+        self.set_config(**kwargs)
+        for key, value in kwargs.items():
+            self.assertEqual(getattr(sys, key), value,
+                             (key, value))
+
+    def test_set_invalid(self):
+        invalid_uint = -1
+        NULL = None
+        invalid_wstr = NULL
+        # PyWideStringList strings must be non-NULL
+        invalid_wstrlist = ["abc", NULL, "def"]
+
+        type_tests = []
+        value_tests = [
+            # enum
+            ('_config_init', 0),
+            ('_config_init', 4),
+            # unsigned long
+            ("hash_seed", -1),
+            ("hash_seed", MAX_HASH_SEED + 1),
+        ]
+
+        # int (unsigned)
+        options = [
+            '_config_init',
+            'isolated',
+            'use_environment',
+            'dev_mode',
+            'install_signal_handlers',
+            'use_hash_seed',
+            'faulthandler',
+            'tracemalloc',
+            'import_time',
+            'show_ref_count',
+            'dump_refs',
+            'malloc_stats',
+            'parse_argv',
+            'site_import',
+            'bytes_warning',
+            'inspect',
+            'interactive',
+            'optimization_level',
+            'parser_debug',
+            'write_bytecode',
+            'verbose',
+            'quiet',
+            'user_site_directory',
+            'configure_c_stdio',
+            'buffered_stdio',
+            'pathconfig_warnings',
+            'module_search_paths_set',
+            'skip_source_first_line',
+            '_install_importlib',
+            '_init_main',
+            '_isolated_interpreter',
+        ]
+        if MS_WINDOWS:
+            options.append('legacy_windows_stdio')
+        for key in options:
+            value_tests.append((key, invalid_uint))
+            type_tests.append((key, "abc"))
+            type_tests.append((key, 2.0))
+
+        # wchar_t*
+        for key in (
+            'filesystem_encoding',
+            'filesystem_errors',
+            'stdio_encoding',
+            'stdio_errors',
+            'check_hash_pycs_mode',
+            'program_name',
+            'platlibdir',
+            # optional wstr:
+            # 'pythonpath_env'
+            # 'home'
+            # 'pycache_prefix'
+            # 'run_command'
+            # 'run_module'
+            # 'run_filename'
+            # 'executable'
+            # 'prefix'
+            # 'exec_prefix'
+            # 'base_executable'
+            # 'base_prefix'
+            # 'base_exec_prefix'
+        ):
+            value_tests.append((key, invalid_wstr))
+            type_tests.append((key, b'bytes'))
+            type_tests.append((key, 123))
+
+        # PyWideStringList
+        for key in (
+            'orig_argv',
+            'argv',
+            'xoptions',
+            'warnoptions',
+            'module_search_paths',
+        ):
+            value_tests.append((key, invalid_wstrlist))
+            type_tests.append((key, 123))
+            type_tests.append((key, "abc"))
+            type_tests.append((key, [123]))
+            type_tests.append((key, [b"bytes"]))
+
+
+        if MS_WINDOWS:
+            value_tests.append(('legacy_windows_stdio', invalid_uint))
+
+        for exc_type, tests in (
+            (ValueError, value_tests),
+            (TypeError, type_tests),
+        ):
+            for key, value in tests:
+                config = self.old_config | {key: value}
+                with self.subTest(key=key, value=value, exc_type=exc_type):
+                    with self.assertRaises(exc_type):
+                        _testinternalcapi.set_config(config)
+
+    def test_flags(self):
+        for sys_attr, key, value in (
+            ("debug", "parser_debug", 1),
+            ("inspect", "inspect", 2),
+            ("interactive", "interactive", 3),
+            ("optimize", "optimization_level", 4),
+            ("verbose", "verbose", 1),
+            ("bytes_warning", "bytes_warning", 10),
+            ("quiet", "quiet", 11),
+            ("isolated", "isolated", 12),
+        ):
+            with self.subTest(sys=sys_attr, key=key, value=value):
+                self.set_config(**{key: value, 'parse_argv': 0})
+                self.assertEqual(getattr(sys.flags, sys_attr), value)
+
+        self.set_config(write_bytecode=0)
+        self.assertEqual(sys.flags.dont_write_bytecode, True)
+        self.assertEqual(sys.dont_write_bytecode, True)
+
+        self.set_config(write_bytecode=1)
+        self.assertEqual(sys.flags.dont_write_bytecode, False)
+        self.assertEqual(sys.dont_write_bytecode, False)
+
+        self.set_config(user_site_directory=0, isolated=0)
+        self.assertEqual(sys.flags.no_user_site, 1)
+        self.set_config(user_site_directory=1, isolated=0)
+        self.assertEqual(sys.flags.no_user_site, 0)
+
+        self.set_config(site_import=0)
+        self.assertEqual(sys.flags.no_site, 1)
+        self.set_config(site_import=1)
+        self.assertEqual(sys.flags.no_site, 0)
+
+        self.set_config(dev_mode=0)
+        self.assertEqual(sys.flags.dev_mode, False)
+        self.set_config(dev_mode=1)
+        self.assertEqual(sys.flags.dev_mode, True)
+
+        self.set_config(use_environment=0, isolated=0)
+        self.assertEqual(sys.flags.ignore_environment, 1)
+        self.set_config(use_environment=1, isolated=0)
+        self.assertEqual(sys.flags.ignore_environment, 0)
+
+        self.set_config(use_hash_seed=1, hash_seed=0)
+        self.assertEqual(sys.flags.hash_randomization, 0)
+        self.set_config(use_hash_seed=0, hash_seed=0)
+        self.assertEqual(sys.flags.hash_randomization, 1)
+        self.set_config(use_hash_seed=1, hash_seed=123)
+        self.assertEqual(sys.flags.hash_randomization, 1)
+
+    def test_options(self):
+        self.check(warnoptions=[])
+        self.check(warnoptions=["default", "ignore"])
+
+        self.set_config(xoptions=[])
+        self.assertEqual(sys._xoptions, {})
+        self.set_config(xoptions=["dev", "tracemalloc=5"])
+        self.assertEqual(sys._xoptions, {"dev": True, "tracemalloc": "5"})
+
+    def test_pathconfig(self):
+        self.check(
+            executable='executable',
+            prefix="prefix",
+            base_prefix="base_prefix",
+            exec_prefix="exec_prefix",
+            base_exec_prefix="base_exec_prefix",
+            platlibdir="platlibdir")
+
+        self.set_config(base_executable="base_executable")
+        self.assertEqual(sys._base_executable, "base_executable")
+
+        # When base_xxx is NULL, value is copied from xxxx
+        self.set_config(
+            executable='executable',
+            prefix="prefix",
+            exec_prefix="exec_prefix",
+            base_executable=None,
+            base_prefix=None,
+            base_exec_prefix=None)
+        self.assertEqual(sys._base_executable, "executable")
+        self.assertEqual(sys.base_prefix, "prefix")
+        self.assertEqual(sys.base_exec_prefix, "exec_prefix")
+
+    def test_path(self):
+        self.set_config(module_search_paths_set=1,
+                        module_search_paths=['a', 'b', 'c'])
+        self.assertEqual(sys.path, ['a', 'b', 'c'])
+
+        # Leave sys.path unchanged if module_search_paths_set=0
+        self.set_config(module_search_paths_set=0,
+                        module_search_paths=['new_path'])
+        self.assertEqual(sys.path, ['a', 'b', 'c'])
+
+    def test_argv(self):
+        self.set_config(parse_argv=0,
+                        argv=['python_program', 'args'],
+                        orig_argv=['orig', 'orig_args'])
+        self.assertEqual(sys.argv, ['python_program', 'args'])
+        self.assertEqual(sys.orig_argv, ['orig', 'orig_args'])
+
+        self.set_config(parse_argv=0,
+                        argv=[],
+                        orig_argv=[])
+        self.assertEqual(sys.argv, [''])
+        self.assertEqual(sys.orig_argv, [])
+
+    def test_pycache_prefix(self):
+        self.check(pycache_prefix=None)
+        self.check(pycache_prefix="pycache_prefix")
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_array.py

@@ -393,7 +393,7 @@ class BaseTest:
         a.insert(0, self.outside)
         self.assertEqual(list(exhit), [])
         # The iterator index points past the 0th position so inserting
-        # an element in the beggining does not make it appear.
+        # an element in the beginning does not make it appear.
         self.assertEqual(list(empit), [])
         self.assertEqual(list(a), [self.outside] + list(self.example))
 

Lib/test/test_asyncio/test_futures2.py

@@ -0,0 +1,18 @@
+# IsolatedAsyncioTestCase based tests
+import asyncio
+import unittest
+
+
+class FutureTests(unittest.IsolatedAsyncioTestCase):
+    async def test_recursive_repr_for_pending_tasks(self):
+        # The call crashes if the guard for recursive call
+        # in base_futures:_future_repr_info is absent
+        # See Also: https://bugs.python.org/issue42183
+
+        async def func():
+            return asyncio.all_tasks()
+
+        # The repr() call should not raise RecursiveError at first.
+        # The check for returned string is not very reliable but
+        # exact comparison for the whole string is even weaker.
+        self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10)))

Lib/test/test_bytes.py

@@ -1036,6 +1036,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase):
             c_char_p)
 
         PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
+        PyBytes_FromFormat.argtypes = (c_char_p,)
         PyBytes_FromFormat.restype = py_object
 
         # basic tests

Lib/test/test_capi.py

@@ -405,6 +405,9 @@ class CAPITest(unittest.TestCase):
         self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc")
         self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)")
 
+    def test_null_type_doc(self):
+        self.assertEqual(_testcapi.NullTpDocType.__doc__, None)
+
     def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self):
         class HeapGcCTypeSubclass(_testcapi.HeapGcCType):
             def __init__(self):

Lib/test/test_contextlib_async.py

@@ -1,5 +1,7 @@
 import asyncio
-from contextlib import aclosing, asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack
+from contextlib import (
+    asynccontextmanager, AbstractAsyncContextManager,
+    AsyncExitStack, nullcontext, aclosing)
 import functools
 from test import support
 import unittest
@@ -278,6 +280,33 @@ class AsyncContextManagerTestCase(unittest.TestCase):
         async with woohoo(self=11, func=22, args=33, kwds=44) as target:
             self.assertEqual(target, (11, 22, 33, 44))
 
+    @_async_test
+    async def test_recursive(self):
+        depth = 0
+        ncols = 0
+
+        @asynccontextmanager
+        async def woohoo():
+            nonlocal ncols
+            ncols += 1
+
+            nonlocal depth
+            before = depth
+            depth += 1
+            yield
+            depth -= 1
+            self.assertEqual(depth, before)
+
+        @woohoo()
+        async def recursive():
+            if depth < 10:
+                await recursive()
+
+        await recursive()
+
+        self.assertEqual(ncols, 10)
+        self.assertEqual(depth, 0)
+
 
 class AclosingTestCase(unittest.TestCase):
 
@@ -510,5 +539,15 @@ class TestAsyncExitStack(TestBaseExitStack, unittest.TestCase):
         self.assertIsInstance(inner_exc.__context__, ZeroDivisionError)
 
 
+class TestAsyncNullcontext(unittest.TestCase):
+    @_async_test
+    async def test_async_nullcontext(self):
+        class C:
+            pass
+        c = C()
+        async with nullcontext(c) as c_in:
+            self.assertIs(c_in, c)
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_embed.py

@@ -30,6 +30,8 @@ API_PYTHON = 2
 # _PyCoreConfig_InitIsolatedConfig()
 API_ISOLATED = 3
 
+MAX_HASH_SEED = 4294967295
+
 
 def debug_build(program):
     program = os.path.basename(program)
@@ -382,6 +384,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'exec_prefix': GET_DEFAULT_CONFIG,
         'base_exec_prefix': GET_DEFAULT_CONFIG,
         'module_search_paths': GET_DEFAULT_CONFIG,
+        'module_search_paths_set': 1,
         'platlibdir': sys.platlibdir,
 
         'site_import': 1,
@@ -419,7 +422,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     CONFIG_PYTHON = dict(CONFIG_COMPAT,
         _config_init=API_PYTHON,
         configure_c_stdio=1,
-        parse_argv=1,
+        parse_argv=2,
     )
     CONFIG_ISOLATED = dict(CONFIG_COMPAT,
         _config_init=API_ISOLATED,
@@ -797,7 +800,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                           '-X', 'cmdline_xoption',
                           '-c', 'pass',
                           'arg2'],
-            'parse_argv': 1,
+            'parse_argv': 2,
             'xoptions': [
                 'config_xoption1=3',
                 'config_xoption2=',
@@ -1042,7 +1045,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'orig_argv': ['python3', '-c', code, 'arg2'],
             'program_name': './python3',
             'run_command': code + '\n',
-            'parse_argv': 1,
+            'parse_argv': 2,
         }
         self.check_all_configs("test_init_run_main", config, api=API_PYTHON)
 
@@ -1056,7 +1059,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                           'arg2'],
             'program_name': './python3',
             'run_command': code + '\n',
-            'parse_argv': 1,
+            'parse_argv': 2,
             '_init_main': 0,
         }
         self.check_all_configs("test_init_main", config,
@@ -1065,7 +1068,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
     def test_init_parse_argv(self):
         config = {
-            'parse_argv': 1,
+            'parse_argv': 2,
             'argv': ['-c', 'arg1', '-v', 'arg3'],
             'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
             'program_name': './argv0',
@@ -1394,11 +1397,31 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.check_all_configs("test_init_warnoptions", config, preconfig,
                                api=API_PYTHON)
 
+    def test_init_set_config(self):
+        config = {
+            '_init_main': 0,
+            'bytes_warning': 2,
+            'warnoptions': ['error::BytesWarning'],
+        }
+        self.check_all_configs("test_init_set_config", config,
+                               api=API_ISOLATED)
+
     def test_get_argc_argv(self):
         self.run_embedded_interpreter("test_get_argc_argv")
         # ignore output
 
 
+class SetConfigTests(unittest.TestCase):
+    def test_set_config(self):
+        # bpo-42260: Test _PyInterpreterState_SetConfig()
+        cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config']
+        proc = subprocess.run(cmd,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        self.assertEqual(proc.returncode, 0,
+                         (proc.returncode, proc.stdout, proc.stderr))
+
+
 class AuditingTests(EmbeddingTestsMixin, unittest.TestCase):
     def test_open_code_hook(self):
         self.run_embedded_interpreter("test_open_code_hook")

Lib/test/test_format.py

@@ -428,13 +428,16 @@ class FormatTest(unittest.TestCase):
             localeconv = locale.localeconv()
             sep = localeconv['thousands_sep']
             point = localeconv['decimal_point']
+            grouping = localeconv['grouping']
 
             text = format(123456789, "n")
-            self.assertIn(sep, text)
+            if grouping:
+                self.assertIn(sep, text)
             self.assertEqual(text.replace(sep, ''), '123456789')
 
             text = format(1234.5, "n")
-            self.assertIn(sep, text)
+            if grouping:
+                self.assertIn(sep, text)
             self.assertIn(point, text)
             self.assertEqual(text.replace(sep, ''), '1234' + point + '5')
         finally:

Lib/test/test_logging.py

@@ -4310,8 +4310,8 @@ class ModuleLevelMiscTest(BaseTest):
             logging.setLoggerClass(logging.Logger)
 
     def test_logging_at_shutdown(self):
-        # Issue #20037
-        code = """if 1:
+        # bpo-20037: Doing text I/O late at interpreter shutdown must not crash
+        code = textwrap.dedent("""
             import logging
 
             class A:
@@ -4321,22 +4321,55 @@ class ModuleLevelMiscTest(BaseTest):
                     except Exception:
                         logging.exception("exception in __del__")
 
-            a = A()"""
+            a = A()
+        """)
         rc, out, err = assert_python_ok("-c", code)
         err = err.decode()
         self.assertIn("exception in __del__", err)
         self.assertIn("ValueError: some error", err)
 
+    def test_logging_at_shutdown_open(self):
+        # bpo-26789: FileHandler keeps a reference to the builtin open()
+        # function to be able to open or reopen the file during Python
+        # finalization.
+        filename = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, filename)
+
+        code = textwrap.dedent(f"""
+            import builtins
+            import logging
+
+            class A:
+                def __del__(self):
+                    logging.error("log in __del__")
+
+            # basicConfig() opens the file, but logging.shutdown() closes
+            # it at Python exit. When A.__del__() is called,
+            # FileHandler._open() must be called again to re-open the file.
+            logging.basicConfig(filename={filename!r})
+
+            a = A()
+
+            # Simulate the Python finalization which removes the builtin
+            # open() function.
+            del builtins.open
+        """)
+        assert_python_ok("-c", code)
+
+        with open(filename) as fp:
+            self.assertEqual(fp.read().rstrip(), "ERROR:root:log in __del__")
+
     def test_recursion_error(self):
         # Issue 36272
-        code = """if 1:
+        code = textwrap.dedent("""
             import logging
 
             def rec():
                 logging.error("foo")
                 rec()
 
-            rec()"""
+            rec()
+        """)
         rc, out, err = assert_python_failure("-c", code)
         err = err.decode()
         self.assertNotIn("Cannot recover from stack overflow.", err)

Lib/test/test_opcache.py

@@ -0,0 +1,23 @@
+import unittest
+
+class TestLoadAttrCache(unittest.TestCase):
+    def test_descriptor_added_after_optimization(self):
+        class Descriptor:
+            pass
+
+        class C:
+            def __init__(self):
+                self.x = 1
+            x = Descriptor()
+
+        def f(o):
+            return o.x
+
+        o = C()
+        for i in range(1025):
+            assert f(o) == 1
+
+        Descriptor.__get__ = lambda self, instance, value: 2
+        Descriptor.__set__ = lambda *args: None
+
+        self.assertEqual(f(o), 2)

Lib/test/test_peg_generator/test_c_parser.py

@@ -1,3 +1,4 @@
+import sysconfig
 import textwrap
 import unittest
 from distutils.tests.support import TempdirManager
@@ -8,6 +9,11 @@ from test import support
 from test.support import os_helper
 from test.support.script_helper import assert_python_ok
 
+_py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
+_pgo_flag = sysconfig.get_config_var('PGO_PROF_USE_FLAG')
+if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist:
+    raise unittest.SkipTest("peg_generator test disabled under PGO build")
+
 test_tools.skip_if_missing("peg_generator")
 with test_tools.imports_under_tool("peg_generator"):
     from pegen.grammar_parser import GeneratedParser as GrammarParser

Lib/test/test_platform.py

@@ -246,7 +246,7 @@ class PlatformTest(unittest.TestCase):
             self.assertEqual(res[1], ('', '', ''))
 
             if sys.byteorder == 'little':
-                self.assertIn(res[2], ('i386', 'x86_64'))
+                self.assertIn(res[2], ('i386', 'x86_64', 'arm64'))
             else:
                 self.assertEqual(res[2], 'PowerPC')
 

Lib/test/test_plistlib.py

@@ -2,6 +2,7 @@
 import copy
 import operator
 import pickle
+import struct
 import unittest
 import plistlib
 import os
@@ -119,6 +120,285 @@ XML_PLIST_WITH_ENTITY=b'''\
 </plist>
 '''
 
+INVALID_BINARY_PLISTS = [
+    ('too short data',
+        b''
+    ),
+    ('too large offset_table_offset and offset_size = 1',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x2a'
+    ),
+    ('too large offset_table_offset and nonstandard offset_size',
+        b'\x00\x00\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x03\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x2c'
+    ),
+    ('integer overflow in offset_table_offset',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\xff\xff\xff\xff\xff\xff\xff\xff'
+    ),
+    ('too large top_object',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('integer overflow in top_object',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\xff\xff\xff\xff\xff\xff\xff\xff'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('too large num_objects and offset_size = 1',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\xff'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('too large num_objects and nonstandard offset_size',
+        b'\x00\x00\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x03\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\xff'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('extremally large num_objects (32 bit)',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x7f\xff\xff\xff'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('extremally large num_objects (64 bit)',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\xff\xff\xff\xff\xff'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('integer overflow in num_objects',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\xff\xff\xff\xff\xff\xff\xff\xff'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('offset_size = 0',
+        b'\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('ref_size = 0',
+        b'\xa1\x01\x00\x08\x0a'
+        b'\x00\x00\x00\x00\x00\x00\x01\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0b'
+    ),
+    ('too large offset',
+        b'\x00\x2a'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('integer overflow in offset',
+        b'\x00\xff\xff\xff\xff\xff\xff\xff\xff'
+        b'\x00\x00\x00\x00\x00\x00\x08\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x09'
+    ),
+    ('too large array size',
+        b'\xaf\x00\x01\xff\x00\x08\x0c'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0d'
+    ),
+    ('extremally large array size (32-bit)',
+        b'\xaf\x02\x7f\xff\xff\xff\x01\x00\x08\x0f'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x10'
+    ),
+    ('extremally large array size (64-bit)',
+        b'\xaf\x03\x00\x00\x00\xff\xff\xff\xff\xff\x01\x00\x08\x13'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x14'
+    ),
+    ('integer overflow in array size',
+        b'\xaf\x03\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x08\x13'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x14'
+    ),
+    ('too large reference index',
+        b'\xa1\x02\x00\x08\x0a'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0b'
+    ),
+    ('integer overflow in reference index',
+        b'\xa1\xff\xff\xff\xff\xff\xff\xff\xff\x00\x08\x11'
+        b'\x00\x00\x00\x00\x00\x00\x01\x08'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x12'
+    ),
+    ('too large bytes size',
+        b'\x4f\x00\x23\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0c'
+    ),
+    ('extremally large bytes size (32-bit)',
+        b'\x4f\x02\x7f\xff\xff\xff\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0f'
+    ),
+    ('extremally large bytes size (64-bit)',
+        b'\x4f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x13'
+    ),
+    ('integer overflow in bytes size',
+        b'\x4f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x13'
+    ),
+    ('too large ASCII size',
+        b'\x5f\x00\x23\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0c'
+    ),
+    ('extremally large ASCII size (32-bit)',
+        b'\x5f\x02\x7f\xff\xff\xff\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0f'
+    ),
+    ('extremally large ASCII size (64-bit)',
+        b'\x5f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x13'
+    ),
+    ('integer overflow in ASCII size',
+        b'\x5f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x13'
+    ),
+    ('invalid ASCII',
+        b'\x51\xff\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0a'
+    ),
+    ('too large UTF-16 size',
+        b'\x6f\x00\x13\x20\xac\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0e'
+    ),
+    ('extremally large UTF-16 size (32-bit)',
+        b'\x6f\x02\x4f\xff\xff\xff\x20\xac\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x11'
+    ),
+    ('extremally large UTF-16 size (64-bit)',
+        b'\x6f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x20\xac\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x15'
+    ),
+    ('integer overflow in UTF-16 size',
+        b'\x6f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x20\xac\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x15'
+    ),
+    ('invalid UTF-16',
+        b'\x61\xd8\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0b'
+    ),
+    ('non-hashable key',
+        b'\xd1\x01\x01\xa0\x08\x0b'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x02'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x0c'
+    ),
+    ('too large datetime (datetime overflow)',
+        b'\x33\x42\x50\x00\x00\x00\x00\x00\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x11'
+    ),
+    ('too large datetime (timedelta overflow)',
+        b'\x33\x42\xe0\x00\x00\x00\x00\x00\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x11'
+    ),
+    ('invalid datetime (Infinity)',
+        b'\x33\x7f\xf0\x00\x00\x00\x00\x00\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x11'
+    ),
+    ('invalid datetime (NaN)',
+        b'\x33\x7f\xf8\x00\x00\x00\x00\x00\x00\x08'
+        b'\x00\x00\x00\x00\x00\x00\x01\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x01'
+        b'\x00\x00\x00\x00\x00\x00\x00\x00'
+        b'\x00\x00\x00\x00\x00\x00\x00\x11'
+    ),
+]
+
 
 class TestPlistlib(unittest.TestCase):
 
@@ -558,6 +838,21 @@ class TestPlistlib(unittest.TestCase):
 
 class TestBinaryPlistlib(unittest.TestCase):
 
+    @staticmethod
+    def decode(*objects, offset_size=1, ref_size=1):
+        data = [b'bplist00']
+        offset = 8
+        offsets = []
+        for x in objects:
+            offsets.append(offset.to_bytes(offset_size, 'big'))
+            data.append(x)
+            offset += len(x)
+        tail = struct.pack('>6xBBQQQ', offset_size, ref_size,
+                           len(objects), 0, offset)
+        data.extend(offsets)
+        data.append(tail)
+        return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY)
+
     def test_nonstandard_refs_size(self):
         # Issue #21538: Refs and offsets are 24-bit integers
         data = (b'bplist00'
@@ -572,7 +867,7 @@ class TestBinaryPlistlib(unittest.TestCase):
 
     def test_dump_duplicates(self):
         # Test effectiveness of saving duplicated objects
-        for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde',
+        for x in (None, False, True, 12345, 123.45, 'abcde', 'абвгд', b'abcde',
                   datetime.datetime(2004, 10, 26, 10, 33, 33),
                   bytearray(b'abcde'), [12, 345], (12, 345), {'12': 345}):
             with self.subTest(x=x):
@@ -609,6 +904,20 @@ class TestBinaryPlistlib(unittest.TestCase):
         b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY))
         self.assertIs(b['x'], b)
 
+    def test_deep_nesting(self):
+        for N in [300, 100000]:
+            chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)]
+            try:
+                result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4)
+            except RecursionError:
+                pass
+            else:
+                for i in range(N):
+                    self.assertIsInstance(result, list)
+                    self.assertEqual(len(result), 1)
+                    result = result[0]
+                self.assertEqual(result, 'seed')
+
     def test_large_timestamp(self):
         # Issue #26709: 32-bit timestamp out of range
         for ts in -2**31-1, 2**31:
@@ -618,55 +927,37 @@ class TestBinaryPlistlib(unittest.TestCase):
                 data = plistlib.dumps(d, fmt=plistlib.FMT_BINARY)
                 self.assertEqual(plistlib.loads(data), d)
 
+    def test_load_singletons(self):
+        self.assertIs(self.decode(b'\x00'), None)
+        self.assertIs(self.decode(b'\x08'), False)
+        self.assertIs(self.decode(b'\x09'), True)
+        self.assertEqual(self.decode(b'\x0f'), b'')
+
+    def test_load_int(self):
+        self.assertEqual(self.decode(b'\x10\x00'), 0)
+        self.assertEqual(self.decode(b'\x10\xfe'), 0xfe)
+        self.assertEqual(self.decode(b'\x11\xfe\xdc'), 0xfedc)
+        self.assertEqual(self.decode(b'\x12\xfe\xdc\xba\x98'), 0xfedcba98)
+        self.assertEqual(self.decode(b'\x13\x01\x23\x45\x67\x89\xab\xcd\xef'),
+                         0x0123456789abcdef)
+        self.assertEqual(self.decode(b'\x13\xfe\xdc\xba\x98\x76\x54\x32\x10'),
+                         -0x123456789abcdf0)
+
+    def test_unsupported(self):
+        unsupported = [*range(1, 8), *range(10, 15),
+                       0x20, 0x21, *range(0x24, 0x33), *range(0x34, 0x40)]
+        for i in [0x70, 0x90, 0xb0, 0xc0, 0xe0, 0xf0]:
+            unsupported.extend(i + j for j in range(16))
+        for token in unsupported:
+            with self.subTest(f'token {token:02x}'):
+                with self.assertRaises(plistlib.InvalidFileException):
+                    self.decode(bytes([token]) + b'\x00'*16)
+
     def test_invalid_binary(self):
-        for data in [
-                # too short data
-                b'',
-                # too large offset_table_offset and nonstandard offset_size
-                b'\x00\x08'
-                b'\x00\x00\x00\x00\x00\x00\x03\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x2a',
-                # integer overflow in offset_table_offset
-                b'\x00\x08'
-                b'\x00\x00\x00\x00\x00\x00\x01\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\xff\xff\xff\xff\xff\xff\xff\xff',
-                # offset_size = 0
-                b'\x00\x08'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x09',
-                # ref_size = 0
-                b'\xa1\x01\x00\x08\x0a'
-                b'\x00\x00\x00\x00\x00\x00\x01\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x02'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x0b',
-                # integer overflow in offset
-                b'\x00\xff\xff\xff\xff\xff\xff\xff\xff'
-                b'\x00\x00\x00\x00\x00\x00\x08\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x09',
-                # invalid ASCII
-                b'\x51\xff\x08'
-                b'\x00\x00\x00\x00\x00\x00\x01\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x0a',
-                # invalid UTF-16
-                b'\x61\xd8\x00\x08'
-                b'\x00\x00\x00\x00\x00\x00\x01\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x01'
-                b'\x00\x00\x00\x00\x00\x00\x00\x00'
-                b'\x00\x00\x00\x00\x00\x00\x00\x0b',
-                ]:
-            with self.assertRaises(plistlib.InvalidFileException):
-                plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
+        for name, data in INVALID_BINARY_PLISTS:
+            with self.subTest(name):
+                with self.assertRaises(plistlib.InvalidFileException):
+                    plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY)
 
 
 class TestKeyedArchive(unittest.TestCase):

Lib/test/test_posix.py

@@ -1925,6 +1925,233 @@ class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin):
         assert_python_ok(*args, PATH=path)
 
 
+@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS")
+class TestPosixWeaklinking(unittest.TestCase):
+    # These test cases verify that weak linking support on macOS works
+    # as expected. These cases only test new behaviour introduced by weak linking,
+    # regular behaviour is tested by the normal test cases. 
+    #
+    # See the section on Weak Linking in Mac/README.txt for more information.
+    def setUp(self):
+        import sysconfig
+        import platform
+
+        config_vars = sysconfig.get_config_vars()
+        self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] }
+        self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split("."))
+
+    def _verify_available(self, name):
+        if name not in self.available:
+            raise unittest.SkipTest(f"{name} not weak-linked")
+
+    def test_pwritev(self):
+        self._verify_available("HAVE_PWRITEV")
+        if self.mac_ver >= (10, 16):
+            self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available")
+            self.assertTrue(hasattr(os, "preadv"), "os.readv is not available")
+
+        else:
+            self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available")
+            self.assertFalse(hasattr(os, "preadv"), "os.readv is available")
+
+    def test_stat(self):
+        self._verify_available("HAVE_FSTATAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_FSTATAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_FSTATAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.stat("file", dir_fd=0)
+
+    def test_access(self):
+        self._verify_available("HAVE_FACCESSAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_FACCESSAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_FACCESSAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.access("file", os.R_OK, dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"):
+                os.access("file", os.R_OK, follow_symlinks=False)
+
+            with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"):
+                os.access("file", os.R_OK, effective_ids=True)
+
+    def test_chmod(self):
+        self._verify_available("HAVE_FCHMODAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_FCHMODAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_FCHMODAT", posix._have_functions)
+            self.assertIn("HAVE_LCHMOD", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.chmod("file", 0o644, dir_fd=0)
+
+    def test_chown(self):
+        self._verify_available("HAVE_FCHOWNAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_FCHOWNAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions)
+            self.assertIn("HAVE_LCHOWN", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.chown("file", 0, 0, dir_fd=0)
+
+    def test_link(self):
+        self._verify_available("HAVE_LINKAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_LINKAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_LINKAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"):
+                os.link("source", "target",  src_dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"):
+                os.link("source", "target",  dst_dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"):
+                os.link("source", "target",  src_dir_fd=0, dst_dir_fd=0)
+
+            # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag
+            with os_helper.temp_dir() as base_path:
+                link_path = os.path.join(base_path, "link")
+                target_path = os.path.join(base_path, "target")
+                source_path = os.path.join(base_path, "source")
+
+                with open(source_path, "w") as fp:
+                    fp.write("data")
+
+                os.symlink("target", link_path)
+
+                # Calling os.link should fail in the link(2) call, and
+                # should not reject *follow_symlinks* (to match the
+                # behaviour you'd get when building on a platform without
+                # linkat)
+                with self.assertRaises(FileExistsError):
+                    os.link(source_path, link_path, follow_symlinks=True)
+
+                with self.assertRaises(FileExistsError):
+                    os.link(source_path, link_path, follow_symlinks=False)
+
+
+    def test_listdir_scandir(self):
+        self._verify_available("HAVE_FDOPENDIR")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_FDOPENDIR", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions)
+
+            with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"):
+                os.listdir(0)
+
+            with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"):
+                os.scandir(0)
+
+    def test_mkdir(self):
+        self._verify_available("HAVE_MKDIRAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_MKDIRAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_MKDIRAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.mkdir("dir", dir_fd=0)
+
+    def test_rename_replace(self):
+        self._verify_available("HAVE_RENAMEAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_RENAMEAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_RENAMEAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+                os.rename("a", "b", src_dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+                os.rename("a", "b", dst_dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+                os.replace("a", "b", src_dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"):
+                os.replace("a", "b", dst_dir_fd=0)
+
+    def test_unlink_rmdir(self):
+        self._verify_available("HAVE_UNLINKAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_UNLINKAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_UNLINKAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.unlink("path", dir_fd=0)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.rmdir("path", dir_fd=0)
+
+    def test_open(self):
+        self._verify_available("HAVE_OPENAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_OPENAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_OPENAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.open("path", os.O_RDONLY, dir_fd=0)
+
+    def test_readlink(self):
+        self._verify_available("HAVE_READLINKAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_READLINKAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_READLINKAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.readlink("path",  dir_fd=0)
+
+    def test_symlink(self):
+        self._verify_available("HAVE_SYMLINKAT")
+        if self.mac_ver >= (10, 10):
+            self.assertIn("HAVE_SYMLINKAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.symlink("a", "b",  dir_fd=0)
+
+    def test_utime(self):
+        self._verify_available("HAVE_FUTIMENS")
+        self._verify_available("HAVE_UTIMENSAT")
+        if self.mac_ver >= (10, 13):
+            self.assertIn("HAVE_FUTIMENS", posix._have_functions)
+            self.assertIn("HAVE_UTIMENSAT", posix._have_functions)
+
+        else:
+            self.assertNotIn("HAVE_FUTIMENS", posix._have_functions)
+            self.assertNotIn("HAVE_UTIMENSAT", posix._have_functions)
+
+            with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"):
+                os.utime("path", dir_fd=0)
+
+
 def test_main():
     try:
         support.run_unittest(
@@ -1932,6 +2159,7 @@ def test_main():
             PosixGroupsTester,
             TestPosixSpawn,
             TestPosixSpawnP,
+            TestPosixWeaklinking
         )
     finally:
         support.reap_children()

Lib/test/test_socketserver.py

@@ -277,13 +277,6 @@ class SocketServerTest(unittest.TestCase):
             t.join()
             s.server_close()
 
-    def test_close_immediately(self):
-        class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
-            pass
-
-        server = MyServer((HOST, 0), lambda: None)
-        server.server_close()
-
     def test_tcpserver_bind_leak(self):
         # Issue #22435: the server socket wouldn't be closed if bind()/listen()
         # failed.
@@ -498,23 +491,6 @@ class MiscTestCase(unittest.TestCase):
         self.assertEqual(server.shutdown_called, 1)
         server.server_close()
 
-    def test_threads_reaped(self):
-        """
-        In #37193, users reported a memory leak
-        due to the saving of every request thread. Ensure that the
-        threads are cleaned up after the requests complete.
-        """
-        class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
-            pass
-
-        server = MyServer((HOST, 0), socketserver.StreamRequestHandler)
-        for n in range(10):
-            with socket.create_connection(server.server_address):
-                server.handle_request()
-        [thread.join() for thread in server._threads]
-        self.assertEqual(len(server._threads), 0)
-        server.server_close()
-
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_sys.py

@@ -432,6 +432,73 @@ class SysModuleTest(unittest.TestCase):
         leave_g.set()
         t.join()
 
+    @threading_helper.reap_threads
+    def test_current_exceptions(self):
+        import threading
+        import traceback
+
+        # Spawn a thread that blocks at a known place.  Then the main
+        # thread does sys._current_frames(), and verifies that the frames
+        # returned make sense.
+        entered_g = threading.Event()
+        leave_g = threading.Event()
+        thread_info = []  # the thread's id
+
+        def f123():
+            g456()
+
+        def g456():
+            thread_info.append(threading.get_ident())
+            entered_g.set()
+            while True:
+                try:
+                    raise ValueError("oops")
+                except ValueError:
+                    if leave_g.wait(timeout=support.LONG_TIMEOUT):
+                        break
+
+        t = threading.Thread(target=f123)
+        t.start()
+        entered_g.wait()
+
+        # At this point, t has finished its entered_g.set(), although it's
+        # impossible to guess whether it's still on that line or has moved on
+        # to its leave_g.wait().
+        self.assertEqual(len(thread_info), 1)
+        thread_id = thread_info[0]
+
+        d = sys._current_exceptions()
+        for tid in d:
+            self.assertIsInstance(tid, int)
+            self.assertGreater(tid, 0)
+
+        main_id = threading.get_ident()
+        self.assertIn(main_id, d)
+        self.assertIn(thread_id, d)
+        self.assertEqual((None, None, None), d.pop(main_id))
+
+        # Verify that the captured thread frame is blocked in g456, called
+        # from f123.  This is a litte tricky, since various bits of
+        # threading.py are also in the thread's call stack.
+        exc_type, exc_value, exc_tb = d.pop(thread_id)
+        stack = traceback.extract_stack(exc_tb.tb_frame)
+        for i, (filename, lineno, funcname, sourceline) in enumerate(stack):
+            if funcname == "f123":
+                break
+        else:
+            self.fail("didn't find f123() on thread's call stack")
+
+        self.assertEqual(sourceline, "g456()")
+
+        # And the next record must be for g456().
+        filename, lineno, funcname, sourceline = stack[i+1]
+        self.assertEqual(funcname, "g456")
+        self.assertTrue(sourceline.startswith("if leave_g.wait("))
+
+        # Reap the spawned thread.
+        leave_g.set()
+        t.join()
+
     def test_attributes(self):
         self.assertIsInstance(sys.api_version, int)
         self.assertIsInstance(sys.argv, list)

Lib/test/test_threading.py

@@ -765,6 +765,27 @@ class ThreadTests(BaseTestCase):
         finally:
             sys.settrace(old_trace)
 
+    def test_gettrace(self):
+        def noop_trace(frame, event, arg):
+            # no operation
+            return noop_trace
+        old_trace = threading.gettrace()
+        try:
+            threading.settrace(noop_trace)
+            trace_func = threading.gettrace()
+            self.assertEqual(noop_trace,trace_func)
+        finally:
+            threading.settrace(old_trace)
+
+    def test_getprofile(self):
+        def fn(*args): pass
+        old_profile = threading.getprofile()
+        try:
+            threading.setprofile(fn)
+            self.assertEqual(fn, threading.getprofile())
+        finally:
+            threading.setprofile(old_profile)
+
     @cpython_only
     def test_shutdown_locks(self):
         for daemon in (False, True):

Lib/test/test_time.py

@@ -1041,6 +1041,36 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase):
             with self.assertRaises(ValueError):
                 pytime_object_to_timespec(float('nan'), time_rnd)
 
+@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS")
+class TestTimeWeaklinking(unittest.TestCase):
+    # These test cases verify that weak linking support on macOS works
+    # as expected. These cases only test new behaviour introduced by weak linking,
+    # regular behaviour is tested by the normal test cases.
+    #
+    # See the section on Weak Linking in Mac/README.txt for more information.
+    def test_clock_functions(self):
+        import sysconfig
+        import platform
+
+        config_vars = sysconfig.get_config_vars()
+        var_name = "HAVE_CLOCK_GETTIME"
+        if var_name not in config_vars or not config_vars[var_name]:
+            raise unittest.SkipTest(f"{var_name} is not available")
+
+        mac_ver = tuple(int(x) for x in platform.mac_ver()[0].split("."))
+
+        clock_names = [
+            "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime",
+            "clock_settime_ns", "clock_getres"]
+        
+        if mac_ver >= (10, 12):
+            for name in clock_names:
+                self.assertTrue(hasattr(time, name), f"time.{name} is not available")
+
+        else:
+            for name in clock_names:
+                self.assertFalse(hasattr(time, name), f"time.{name} is available")
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_tools/test_i18n.py

@@ -220,6 +220,76 @@ class Test_pygettext(unittest.TestCase):
         '''))
         self.assertIn('doc', msgids)
 
+    def test_calls_in_fstrings(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_raw(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        rf"{_('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_nested(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"""{f'{_("foo bar")}'}"""
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_attribute(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{obj._('foo bar')}"
+        '''))
+        self.assertIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_with_call_on_call(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{type(str)('foo bar')}"
+        '''))
+        self.assertNotIn('foo bar', msgids)
+
+    def test_calls_in_fstrings_with_format(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo {bar}').format(bar='baz')}"
+        '''))
+        self.assertIn('foo {bar}', msgids)
+
+    def test_calls_in_fstrings_with_wrong_input_1(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(f'foo {bar}')}"
+        '''))
+        self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid])
+
+    def test_calls_in_fstrings_with_wrong_input_2(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(1)}"
+        '''))
+        self.assertNotIn(1, msgids)
+
+    def test_calls_in_fstring_with_multiple_args(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo', 'bar')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertNotIn('bar', msgids)
+
+    def test_calls_in_fstring_with_keyword_args(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_('foo', bar='baz')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertNotIn('bar', msgids)
+        self.assertNotIn('baz', msgids)
+
+    def test_calls_in_fstring_with_partially_wrong_expression(self):
+        msgids = self.extract_docstrings_from_str(dedent('''\
+        f"{_(f'foo') + _('bar')}"
+        '''))
+        self.assertNotIn('foo', msgids)
+        self.assertIn('bar', msgids)
+
     def test_files_list(self):
         """Make sure the directories are inspected for source files
            bpo-31920

Lib/test/test_traceback.py

@@ -212,6 +212,26 @@ class TracebackCases(unittest.TestCase):
         )
         self.assertEqual(output.getvalue(), "Exception: projector\n")
 
+    def test_print_exception_exc(self):
+        output = StringIO()
+        traceback.print_exception(Exception("projector"), file=output)
+        self.assertEqual(output.getvalue(), "Exception: projector\n")
+
+    def test_format_exception_exc(self):
+        e = Exception("projector")
+        output = traceback.format_exception(e)
+        self.assertEqual(output, ["Exception: projector\n"])
+        with self.assertRaisesRegex(ValueError, 'Both or neither'):
+            traceback.format_exception(e.__class__, e)
+        with self.assertRaisesRegex(ValueError, 'Both or neither'):
+            traceback.format_exception(e.__class__, tb=e.__traceback__)
+        with self.assertRaisesRegex(TypeError, 'positional-only'):
+            traceback.format_exception(exc=e)
+
+    def test_format_exception_only_exc(self):
+        output = traceback.format_exception_only(Exception("projector"))
+        self.assertEqual(output, ["Exception: projector\n"])
+
 
 class TracebackFormatTests(unittest.TestCase):
 

Lib/test/test_types.py

@@ -713,6 +713,28 @@ class TypesTests(unittest.TestCase):
         assert repr(int | None) == "int | None"
         assert repr(int | typing.GenericAlias(list, int)) == "int | list[int]"
 
+    def test_or_type_operator_with_genericalias(self):
+        a = list[int]
+        b = list[str]
+        c = dict[float, str]
+        # equivalence with typing.Union
+        self.assertEqual(a | b | c, typing.Union[a, b, c])
+        # de-duplicate
+        self.assertEqual(a | c | b | b | a | c, a | b | c)
+        # order shouldn't matter
+        self.assertEqual(a | b, b | a)
+        self.assertEqual(repr(a | b | c),
+                         "list[int] | list[str] | dict[float, str]")
+
+        class BadType(type):
+            def __eq__(self, other):
+                return 1 / 0
+
+        bt = BadType('bt', (), {})
+        # Comparison should fail and errors should propagate out for bad types.
+        with self.assertRaises(ZeroDivisionError):
+            list[int] | list[bt]
+
     def test_ellipsis_type(self):
         self.assertIsInstance(Ellipsis, types.EllipsisType)
 

Lib/test/test_unicode.py

@@ -2516,11 +2516,13 @@ class CAPITest(unittest.TestCase):
     def test_from_format(self):
         import_helper.import_module('ctypes')
         from ctypes import (
+            c_char_p,
             pythonapi, py_object, sizeof,
             c_int, c_long, c_longlong, c_ssize_t,
             c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
         name = "PyUnicode_FromFormat"
         _PyUnicode_FromFormat = getattr(pythonapi, name)
+        _PyUnicode_FromFormat.argtypes = (c_char_p,)
         _PyUnicode_FromFormat.restype = py_object
 
         def PyUnicode_FromFormat(format, *args):

Lib/test/test_utf8_mode.py

@@ -3,11 +3,13 @@ Test the implementation of the PEP 540: the UTF-8 Mode.
 """
 
 import locale
+import subprocess
 import sys
 import textwrap
 import unittest
 from test import support
 from test.support.script_helper import assert_python_ok, assert_python_failure
+from test.support import os_helper
 
 
 MS_WINDOWS = (sys.platform == 'win32')
@@ -250,6 +252,31 @@ class UTF8ModeTests(unittest.TestCase):
         out = self.get_output('-X', 'utf8', '-E', '-c', code)
         self.assertEqual(out, '1')
 
+    @unittest.skipIf(MS_WINDOWS,
+                     "os.device_encoding() doesn't implement "
+                     "the UTF-8 Mode on Windows")
+    def test_device_encoding(self):
+        # Use stdout as TTY
+        if not sys.stdout.isatty():
+            self.skipTest("sys.stdout is not a TTY")
+
+        filename = 'out.txt'
+        self.addCleanup(os_helper.unlink, filename)
+
+        code = (f'import os, sys; fd = sys.stdout.fileno(); '
+                f'out = open({filename!r}, "w", encoding="utf-8"); '
+                f'print(os.isatty(fd), os.device_encoding(fd), file=out); '
+                f'out.close()')
+        cmd = [sys.executable, '-X', 'utf8', '-c', code]
+        # The stdout TTY is inherited to the child process
+        proc = subprocess.run(cmd, text=True)
+        self.assertEqual(proc.returncode, 0, proc)
+
+        # In UTF-8 Mode, device_encoding(fd) returns "UTF-8" if fd is a TTY
+        with open(filename, encoding="utf8") as fp:
+            out = fp.read().rstrip()
+        self.assertEqual(out, 'True UTF-8')
+
 
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_xml_etree.py

@@ -2852,8 +2852,12 @@ class ElementFindTest(unittest.TestCase):
             ['tag'] * 3)
         self.assertEqual(summarize_list(e.findall('.//tag[@class="a"]')),
             ['tag'])
+        self.assertEqual(summarize_list(e.findall('.//tag[@class!="a"]')),
+            ['tag'] * 2)
         self.assertEqual(summarize_list(e.findall('.//tag[@class="b"]')),
             ['tag'] * 2)
+        self.assertEqual(summarize_list(e.findall('.//tag[@class!="b"]')),
+            ['tag'])
         self.assertEqual(summarize_list(e.findall('.//tag[@id]')),
             ['tag'])
         self.assertEqual(summarize_list(e.findall('.//section[tag]')),
@@ -2875,6 +2879,19 @@ class ElementFindTest(unittest.TestCase):
         self.assertEqual(summarize_list(e.findall(".//section[ tag = 'subtext' ]")),
             ['section'])
 
+        # Negations of above tests. They match nothing because the sole section
+        # tag has subtext.
+        self.assertEqual(summarize_list(e.findall(".//section[tag!='subtext']")),
+            [])
+        self.assertEqual(summarize_list(e.findall(".//section[tag !='subtext']")),
+            [])
+        self.assertEqual(summarize_list(e.findall(".//section[tag!= 'subtext']")),
+            [])
+        self.assertEqual(summarize_list(e.findall(".//section[tag != 'subtext']")),
+            [])
+        self.assertEqual(summarize_list(e.findall(".//section[ tag != 'subtext' ]")),
+            [])
+
         self.assertEqual(summarize_list(e.findall(".//tag[.='subtext']")),
                          ['tag'])
         self.assertEqual(summarize_list(e.findall(".//tag[. ='subtext']")),
@@ -2890,6 +2907,24 @@ class ElementFindTest(unittest.TestCase):
         self.assertEqual(summarize_list(e.findall(".//tag[.= ' subtext']")),
                          [])
 
+        # Negations of above tests.
+        #   Matches everything but the tag containing subtext
+        self.assertEqual(summarize_list(e.findall(".//tag[.!='subtext']")),
+                         ['tag'] * 3)
+        self.assertEqual(summarize_list(e.findall(".//tag[. !='subtext']")),
+                         ['tag'] * 3)
+        self.assertEqual(summarize_list(e.findall('.//tag[.!= "subtext"]')),
+                         ['tag'] * 3)
+        self.assertEqual(summarize_list(e.findall('.//tag[ . != "subtext" ]')),
+                         ['tag'] * 3)
+        self.assertEqual(summarize_list(e.findall(".//tag[. != 'subtext']")),
+                         ['tag'] * 3)
+        # Matches all tags.
+        self.assertEqual(summarize_list(e.findall(".//tag[. != 'subtext ']")),
+                         ['tag'] * 4)
+        self.assertEqual(summarize_list(e.findall(".//tag[.!= ' subtext']")),
+                         ['tag'] * 4)
+
         # duplicate section => 2x tag matches
         e[1] = e[2]
         self.assertEqual(summarize_list(e.findall(".//section[tag = 'subtext']")),

Lib/threading.py

@@ -28,7 +28,7 @@ __all__ = ['get_ident', 'active_count', 'Condition', 'current_thread',
            'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread',
            'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError',
            'setprofile', 'settrace', 'local', 'stack_size',
-           'excepthook', 'ExceptHookArgs']
+           'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile']
 
 # Rename some stuff so "from threading import *" is safe
 _start_new_thread = _thread.start_new_thread
@@ -65,6 +65,10 @@ def setprofile(func):
     global _profile_hook
     _profile_hook = func
 
+def getprofile():
+    """Get the profiler function as set by threading.setprofile()."""
+    return _profile_hook
+
 def settrace(func):
     """Set a trace function for all threads started from the threading module.
 
@@ -75,6 +79,10 @@ def settrace(func):
     global _trace_hook
     _trace_hook = func
 
+def gettrace():
+    """Get the trace function as set by threading.settrace()."""
+    return _trace_hook
+
 # Synchronization classes
 
 Lock = _allocate_lock

Lib/traceback.py

@@ -84,7 +84,19 @@ _context_message = (
     "another exception occurred:\n\n")
 
 
-def print_exception(etype, value, tb, limit=None, file=None, chain=True):
+_sentinel = object()
+
+
+def _parse_value_tb(exc, value, tb):
+    if (value is _sentinel) != (tb is _sentinel):
+        raise ValueError("Both or neither of value and tb must be given")
+    if value is tb is _sentinel:
+        return exc, exc.__traceback__
+    return value, tb
+
+
+def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
+                    file=None, chain=True):
     """Print exception up to 'limit' stack trace entries from 'tb' to 'file'.
 
     This differs from print_tb() in the following ways: (1) if
@@ -95,9 +107,7 @@ def print_exception(etype, value, tb, limit=None, file=None, chain=True):
     occurred with a caret on the next line indicating the approximate
     position of the error.
     """
-    # format_exception has ignored etype for some time, and code such as cgitb
-    # passes in bogus values as a result. For compatibility with such code we
-    # ignore it here (rather than in the new TracebackException API).
+    value, tb = _parse_value_tb(exc, value, tb)
     if file is None:
         file = sys.stderr
     for line in TracebackException(
@@ -105,7 +115,8 @@ def print_exception(etype, value, tb, limit=None, file=None, chain=True):
         print(line, file=file, end="")
 
 
-def format_exception(etype, value, tb, limit=None, chain=True):
+def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
+                     chain=True):
     """Format a stack trace and the exception information.
 
     The arguments have the same meaning as the corresponding arguments
@@ -114,19 +125,15 @@ def format_exception(etype, value, tb, limit=None, chain=True):
     these lines are concatenated and printed, exactly the same text is
     printed as does print_exception().
     """
-    # format_exception has ignored etype for some time, and code such as cgitb
-    # passes in bogus values as a result. For compatibility with such code we
-    # ignore it here (rather than in the new TracebackException API).
+    value, tb = _parse_value_tb(exc, value, tb)
     return list(TracebackException(
         type(value), value, tb, limit=limit).format(chain=chain))
 
 
-def format_exception_only(etype, value):
+def format_exception_only(exc, /, value=_sentinel):
     """Format the exception part of a traceback.
 
-    The arguments are the exception type and value such as given by
-    sys.last_type and sys.last_value. The return value is a list of
-    strings, each ending in a newline.
+    The return value is a list of strings, each ending in a newline.
 
     Normally, the list contains a single string; however, for
     SyntaxError exceptions, it contains several lines that (when
@@ -137,7 +144,10 @@ def format_exception_only(etype, value):
     string in the list.
 
     """
-    return list(TracebackException(etype, value, None).format_exception_only())
+    if value is _sentinel:
+        value = exc
+    return list(TracebackException(
+        type(value), value, None).format_exception_only())
 
 
 # -- not official API but folk probably use these two functions.

Lib/unittest/_log.py

@@ -47,6 +47,7 @@ class _AssertLogsContext(_BaseTestCaseContext):
             logger = self.logger = logging.getLogger(self.logger_name)
         formatter = logging.Formatter(self.LOGGING_FORMAT)
         handler = _CapturingHandler()
+        handler.setLevel(self.level)
         handler.setFormatter(formatter)
         self.watcher = handler.watcher
         self.old_handlers = logger.handlers[:]