bpo-29102: Add a unique ID to PyInterpreterState. (#1639)

Doc/c-api/init.rst

@@ -821,6 +821,14 @@ been created.
    :c:func:`PyThreadState_Clear`.
 
 
+.. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp)
+
+   Return the interpreter's unique ID.  If there was any error in doing
+   so then -1 is returned and an error is set.
+
+   .. versionadded:: 3.7
+
+
 .. c:function:: PyObject* PyThreadState_GetDict()
 
    Return a dictionary in which extensions can store thread-specific state

Include/pystate.h

@@ -28,6 +28,8 @@ typedef struct _is {
     struct _is *next;
     struct _ts *tstate_head;
 
+    int64_t id;
+
     PyObject *modules;
     PyObject *modules_by_index;
     PyObject *sysdict;
@@ -154,9 +156,16 @@ typedef struct _ts {
 #endif
 
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyInterpreterState_Init(void);
+#endif /* !Py_LIMITED_API */
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void);
 PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *);
 PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *);
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
+/* New in 3.7 */
+PyAPI_FUNC(int64_t) PyInterpreterState_GetID(PyInterpreterState *);
+#endif
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*);
 #endif /* !Py_LIMITED_API */

Lib/test/test_capi.py

@@ -1,8 +1,10 @@
 # Run the _testcapi module tests (tests for the Python/C API):  by defn,
 # these are all functions _testcapi exports whose name begins with 'test_'.
 
+from collections import namedtuple
 import os
 import pickle
+import platform
 import random
 import re
 import subprocess
@@ -384,12 +386,91 @@ class EmbeddingTests(unittest.TestCase):
         return out, err
 
     def test_subinterps(self):
-        # This is just a "don't crash" test
         out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters")
-        if support.verbose:
+        self.assertEqual(err, "")
-            print()
+
-            print(out)
+        # The output from _testembed looks like this:
-            print(err)
+        # --- Pass 0 ---
+        # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
+        # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784
+        # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368
+        # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200
+        # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728
+        # --- Pass 1 ---
+        # ...
+
+        interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, "
+                      r"thread state <(0x[\dA-F]+)>: "
+                      r"id\(modules\) = ([\d]+)$")
+        Interp = namedtuple("Interp", "id interp tstate modules")
+
+        main = None
+        lastmain = None
+        numinner = None
+        numloops = 0
+        for line in out.splitlines():
+            if line == "--- Pass {} ---".format(numloops):
+                if numinner is not None:
+                    self.assertEqual(numinner, 5)
+                if support.verbose:
+                    print(line)
+                lastmain = main
+                main = None
+                mainid = 0
+                numloops += 1
+                numinner = 0
+                continue
+            numinner += 1
+
+            self.assertLessEqual(numinner, 5)
+            match = re.match(interp_pat, line)
+            if match is None:
+                self.assertRegex(line, interp_pat)
+
+            # The last line in the loop should be the same as the first.
+            if numinner == 5:
+                self.assertEqual(match.groups(), main)
+                continue
+
+            # Parse the line from the loop.  The first line is the main
+            # interpreter and the 3 afterward are subinterpreters.
+            interp = Interp(*match.groups())
+            if support.verbose:
+                print(interp)
+            if numinner == 1:
+                main = interp
+                id = str(mainid)
+            else:
+                subid = mainid + numinner - 1
+                id = str(subid)
+
+            # Validate the loop line for each interpreter.
+            self.assertEqual(interp.id, id)
+            self.assertTrue(interp.interp)
+            self.assertTrue(interp.tstate)
+            self.assertTrue(interp.modules)
+            if platform.system() == 'Windows':
+                # XXX Fix on Windows: something is going on with the
+                # pointers in Programs/_testembed.c.  interp.interp
+                # is 0x0 and # interp.modules is the same between
+                # interpreters.
+                continue
+            if interp is main:
+                if lastmain is not None:
+                    # A new main interpreter may have the same interp
+                    # and/or tstate pointer as an earlier finalized/
+                    # destroyed one.  So we do not check interp or
+                    # tstate here.
+                    self.assertNotEqual(interp.modules, lastmain.modules)
+            else:
+                # A new subinterpreter may have the same
+                # PyInterpreterState pointer as a previous one if
+                # the earlier one has already been destroyed.  So
+                # we compare with the main interpreter.  The same
+                # applies to tstate.
+                self.assertNotEqual(interp.interp, main.interp)
+                self.assertNotEqual(interp.tstate, main.tstate)
+                self.assertNotEqual(interp.modules, main.modules)
 
     @staticmethod
     def _get_default_pipe_encoding():

Misc/NEWS

@@ -53,6 +53,9 @@ Core and Builtins
 - bpo-24821: Fixed the slowing down to 25 times in the searching of some
   unlucky Unicode characters.
 
+- bpo-29102: Add a unique ID to PyInterpreterState.  This makes it easier
+  to identify each subinterpreter.
+
 - bpo-29894: The deprecation warning is emitted if __complex__ returns an
   instance of a strict subclass of complex.  In a future versions of Python
   this can be an error.

PC/python3.def

@@ -538,6 +538,7 @@ EXPORTS
   PySlice_Type=python37.PySlice_Type DATA
   PySlice_Unpack=python37.PySlice_Unpack
   PySortWrapper_Type=python37.PySortWrapper_Type DATA
+  PyInterpreterState_GetID=python37.PyInterpreterState_GetID
   PyState_AddModule=python37.PyState_AddModule
   PyState_FindModule=python37.PyState_FindModule
   PyState_RemoveModule=python37.PyState_RemoveModule

Programs/_testembed.c

@@ -1,4 +1,5 @@
 #include <Python.h>
+#include <inttypes.h>
 #include <stdio.h>
 
 /*********************************************************
@@ -22,9 +23,13 @@ static void _testembed_Py_Initialize(void)
 
 static void print_subinterp(void)
 {
-    /* Just output some debug stuff */
+    /* Output information about the interpreter in the format
+       expected in Lib/test/test_capi.py (test_subinterps). */
     PyThreadState *ts = PyThreadState_Get();
-    printf("interp %p, thread state %p: ", ts->interp, ts);
+    PyInterpreterState *interp = ts->interp;
+    int64_t id = PyInterpreterState_GetID(interp);
+    printf("interp %lu <0x%" PRIXPTR ">, thread state <0x%" PRIXPTR ">: ",
+            id, (uintptr_t)interp, (uintptr_t)ts);
     fflush(stdout);
     PyRun_SimpleString(
         "import sys;"

Python/pylifecycle.c

@@ -344,6 +344,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
 
     _PyRandom_Init();
 
+    _PyInterpreterState_Init();
     interp = PyInterpreterState_New();
     if (interp == NULL)
         Py_FatalError("Py_Initialize: can't make first interpreter");

Python/pystate.c

@@ -65,6 +65,23 @@ PyThreadFrameGetter _PyThreadState_GetFrame = NULL;
 static void _PyGILState_NoteThreadState(PyThreadState* tstate);
 #endif
 
+/* _next_interp_id is an auto-numbered sequence of small integers.
+   It gets initialized in _PyInterpreterState_Init(), which is called
+   in Py_Initialize(), and used in PyInterpreterState_New().  A negative
+   interpreter ID indicates an error occurred.  The main interpreter
+   will always have an ID of 0.  Overflow results in a RuntimeError.
+   If that becomes a problem later then we can adjust, e.g. by using
+   a Python int.
+
+   We initialize this to -1 so that the pre-Py_Initialize() value
+   results in an error. */
+static int64_t _next_interp_id = -1;
+
+void
+_PyInterpreterState_Init(void)
+{
+    _next_interp_id = 0;
+}
 
 PyInterpreterState *
 PyInterpreterState_New(void)
@@ -103,6 +120,15 @@ PyInterpreterState_New(void)
         HEAD_LOCK();
         interp->next = interp_head;
         interp_head = interp;
+        if (_next_interp_id < 0) {
+            /* overflow or Py_Initialize() not called! */
+            PyErr_SetString(PyExc_RuntimeError,
+                            "failed to get an interpreter ID");
+            interp = NULL;
+        } else {
+            interp->id = _next_interp_id;
+            _next_interp_id += 1;
+        }
         HEAD_UNLOCK();
     }
 
@@ -170,6 +196,17 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
 }
 
 
+int64_t
+PyInterpreterState_GetID(PyInterpreterState *interp)
+{
+    if (interp == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "no interpreter provided");
+        return -1;
+    }
+    return interp->id;
+}
+
+
 /* Default implementation for _PyThreadState_GetFrame */
 static struct _frame *
 threadstate_getframe(PyThreadState *self)