Add the co_extra field and accompanying APIs to code objects.

This completes PEP 523.

Doc/whatsnew/3.6.rst

@@ -127,7 +127,8 @@ evaluation, etc.
 
 This API is not part of the limited C API and is marked as private to
 signal that usage of this API is expected to be limited and only
-applicable to very select, low-level use-cases.
+applicable to very select, low-level use-cases. Semantics of the
+API will change with Python as necessary.
 
 .. seealso::
 

Include/ceval.h

@@ -187,6 +187,10 @@ PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
 PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void);
 #endif
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc);
+#endif
+
 #define Py_BEGIN_ALLOW_THREADS { \
                         PyThreadState *_save; \
                         _save = PyEval_SaveThread();

Include/code.h

@@ -7,6 +7,14 @@
 extern "C" {
 #endif
 
+
+/* Holder for co_extra information */
+typedef struct {
+    Py_ssize_t ce_size;
+    void **ce_extras;
+} _PyCodeObjectExtra;
+
+
 /* Bytecode object */
 typedef struct {
     PyObject_HEAD
@@ -15,6 +23,7 @@ typedef struct {
     int co_nlocals;		/* #local variables */
     int co_stacksize;		/* #entries needed for evaluation stack */
     int co_flags;		/* CO_..., see below */
+    int co_firstlineno;   /* first source line number */
     PyObject *co_code;		/* instruction opcodes */
     PyObject *co_consts;	/* list (constants used) */
     PyObject *co_names;		/* list of strings (names used) */
@@ -30,11 +39,12 @@ typedef struct {
     unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */
     PyObject *co_filename;	/* unicode (where it was loaded from) */
     PyObject *co_name;		/* unicode (name, for reference) */
-    int co_firstlineno;		/* first source line number */
     PyObject *co_lnotab;	/* string (encoding addr<->lineno mapping) See
 				   Objects/lnotab_notes.txt for details. */
     void *co_zombieframe;     /* for optimization only (see frameobject.c) */
     PyObject *co_weakreflist;   /* to support weakrefs to code objects */
+    /* Scratch space for extra data relating to the code object */
+    _PyCodeObjectExtra *co_extra;
 } PyCodeObject;
 
 /* Masks for co_flags above */
@@ -128,6 +138,14 @@ PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj);
 PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts,
                                       PyObject *names, PyObject *lnotab);
 
+
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index,
+                                 void **extra);
+PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
+                                 void *extra);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/pystate.h

@@ -8,6 +8,10 @@
 extern "C" {
 #endif
 
+/* This limitation is for performance and simplicity. If needed it can be
+removed (with effort). */
+#define MAX_CO_EXTRA_USERS 255
+
 /* State shared between threads */
 
 struct _ts; /* Forward */
@@ -141,6 +145,9 @@ typedef struct _ts {
     PyObject *coroutine_wrapper;
     int in_coroutine_wrapper;
 
+    Py_ssize_t co_extra_user_count;
+    freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
+
     /* XXX signal handlers should also be here */
 
 } PyThreadState;

Misc/NEWS

@@ -27,7 +27,7 @@ Core and Builtins
   the braces (where the expressions are). This is a breaking change
   from the 3.6 alpha releases.
 
-- Implement the frame evaluation part of PEP 523.
+- Implement PEP 523.
 
 - Issue #27870: A left shift of zero by a large integer no longer attempts
   to allocate large amounts of memory.

Objects/codeobject.c

@@ -152,6 +152,7 @@ PyCode_New(int argcount, int kwonlyargcount,
     co->co_lnotab = lnotab;
     co->co_zombieframe = NULL;
     co->co_weakreflist = NULL;
+    co->co_extra = NULL;
     return co;
 }
 
@@ -361,6 +362,20 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kw)
 static void
 code_dealloc(PyCodeObject *co)
 {
+    if (co->co_extra != NULL) {
+        PyThreadState *tstate = PyThreadState_Get();
+
+        for (Py_ssize_t i = 0; i < co->co_extra->ce_size; i++) {
+            freefunc free_extra = tstate->co_extra_freefuncs[i];
+
+            if (free_extra != NULL) {
+                free_extra(co->co_extra->ce_extras[i]);
+            }
+        }
+
+        PyMem_FREE(co->co_extra);
+    }
+
     Py_XDECREF(co->co_code);
     Py_XDECREF(co->co_consts);
     Py_XDECREF(co->co_names);
@@ -752,3 +767,79 @@ _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds)
 
     return line;
 }
+
+
+int
+_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+{
+    PyCodeObject *o;
+
+    assert(*extra == NULL);
+
+    if (!PyCode_Check(code)) {
+        PyErr_BadInternalCall();
+        return 1;
+    }
+
+    o = (PyCodeObject*) code;
+
+    if (o->co_extra == NULL || o->co_extra->ce_size <= index) {
+        return 0;
+    }
+
+    *extra = o->co_extra->ce_extras[index];
+    return 0;
+}
+
+
+int
+_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+{
+    PyCodeObject *o;
+    PyThreadState *tstate = PyThreadState_Get();
+
+    if (!PyCode_Check(code) || index < 0 ||
+            index >= tstate->co_extra_user_count) {
+        PyErr_BadInternalCall();
+        return 1;
+    }
+
+    o = (PyCodeObject*) code;
+
+    if (o->co_extra == NULL) {
+        o->co_extra = (_PyCodeObjectExtra*) PyMem_Malloc(
+            sizeof(_PyCodeObjectExtra));
+        if (o->co_extra == NULL) {
+            return 1;
+        }
+
+        o->co_extra->ce_extras = PyMem_Malloc(
+            tstate->co_extra_user_count * sizeof(void*));
+        if (o->co_extra->ce_extras == NULL) {
+            return 1;
+        }
+
+        o->co_extra->ce_size = tstate->co_extra_user_count;
+
+        for (Py_ssize_t i = 0; i < o->co_extra->ce_size; i++) {
+            o->co_extra->ce_extras[i] = NULL;
+        }
+    }
+    else if (o->co_extra->ce_size <= index) {
+        o->co_extra->ce_extras = PyMem_Realloc(
+            o->co_extra->ce_extras, tstate->co_extra_user_count * sizeof(void*));
+
+        if (o->co_extra->ce_extras == NULL) {
+            return 1;
+        }
+
+        o->co_extra->ce_size = tstate->co_extra_user_count;
+
+        for (Py_ssize_t i = o->co_extra->ce_size; i < o->co_extra->ce_size; i++) {
+            o->co_extra->ce_extras[i] = NULL;
+        }
+    }
+
+    o->co_extra->ce_extras[index] = extra;
+    return 0;
+}

Python/ceval.c

@@ -5608,3 +5608,17 @@ _Py_GetDXProfile(PyObject *self, PyObject *args)
 }
 
 #endif
+
+Py_ssize_t
+_PyEval_RequestCodeExtraIndex(freefunc free)
+{
+    PyThreadState *tstate = PyThreadState_Get();
+    Py_ssize_t new_index;
+
+    if (tstate->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) {
+        return -1;
+    }
+    new_index = tstate->co_extra_user_count++;
+    tstate->co_extra_freefuncs[new_index] = free;
+    return new_index;
+}

Python/pystate.c

@@ -227,6 +227,7 @@ new_threadstate(PyInterpreterState *interp, int init)
 
         tstate->coroutine_wrapper = NULL;
         tstate->in_coroutine_wrapper = 0;
+        tstate->co_extra_user_count = 0;
 
         if (init)
             _PyThreadState_Init(tstate);