Add Garbage Collection support to new-style classes (not yet to their

instances).

Also added GC support to various auxiliary types: super, property,
descriptors, wrappers, dictproxy.  (Only type objects have a tp_clear
field; the other types are.)

One change was necessary to the GC infrastructure.  We have statically
allocated type objects that don't have a GC header (and can't easily
be given one) and heap-allocated type objects that do have a GC
header.  Giving these different metatypes would be really ugly: I
tried, and I had to modify pickle.py, cPickle.c, copy.py, add a new
invent a new name for the new metatype and make it a built-in, change
affected tests...  In short, a mess.  So instead, we add a new type
slot tp_is_gc, which is a simple Boolean function that determines
whether a particular instance has GC headers or not.  This slot is
only relevant for types that have the (new) GC flag bit set.  If the
tp_is_gc slot is NULL (by far the most common case), all instances of
the type are deemed to have GC headers.  This slot is called by the
PyObject_IS_GC() macro (which is only used twice, both times in
gcmodule.c).

I also changed the extern declarations for a bunch of GC-related
functions (_PyObject_GC_Del etc.): these always exist but objimpl.h
only declared them when WITH_CYCLE_GC was defined, but I needed to be
able to reference them without #ifdefs.  (When WITH_CYCLE_GC is not
defined, they do the same as their non-GC counterparts anyway.)

Include/object.h

@@ -285,6 +285,7 @@ typedef struct _typeobject {
 	allocfunc tp_alloc;
 	newfunc tp_new;
 	destructor tp_free; /* Low-level free-memory routine */
+	inquiry tp_is_gc; /* For PyObject_IS_GC */
 	PyObject *tp_bases;
 	PyObject *tp_mro; /* method resolution order */
 	PyObject *tp_defined;

Include/objimpl.h

@@ -217,13 +217,18 @@ extern DL_IMPORT(void) _PyObject_Del(PyObject *);
 /*
  * Garbage Collection Support
  * ==========================
+ *
+ * Some of the functions and macros below are always defined; when
+ * WITH_CYCLE_GC is undefined, they simply don't do anything different
+ * than their non-GC counterparts.
  */
 
 /* Test if a type has a GC head */
 #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
 
 /* Test if an object has a GC head */
-#define PyObject_IS_GC(o) PyType_IS_GC((o)->ob_type)
+#define PyObject_IS_GC(o) (PyType_IS_GC((o)->ob_type) && \
+	((o)->ob_type->tp_is_gc == NULL || (o)->ob_type->tp_is_gc(o)))
 
 extern DL_IMPORT(PyObject *) _PyObject_GC_Malloc(PyTypeObject *, int);
 extern DL_IMPORT(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, int);
@@ -231,14 +236,14 @@ extern DL_IMPORT(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, int);
 #define PyObject_GC_Resize(type, op, n) \
 		( (type *) _PyObject_GC_Resize((PyVarObject *)(op), (n)) )
 
-#ifdef WITH_CYCLE_GC
-
 extern DL_IMPORT(PyObject *) _PyObject_GC_New(PyTypeObject *);
 extern DL_IMPORT(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, int);
 extern DL_IMPORT(void) _PyObject_GC_Del(PyObject *);
 extern DL_IMPORT(void) _PyObject_GC_Track(PyObject *);
 extern DL_IMPORT(void) _PyObject_GC_UnTrack(PyObject *);
 
+#ifdef WITH_CYCLE_GC
+
 /* GC information is stored BEFORE the object structure */
 typedef struct _gc_head {
 	struct _gc_head *gc_next; /* not NULL if object is tracked */

Lib/test/test_gc.py

@@ -7,9 +7,9 @@ def expect(actual, expected, name):
         raise TestFailed, "test_%s: actual %d, expected %d" % (
             name, actual, expected)
 
-def expect_not(actual, expected, name):
+def expect_nonzero(actual, name):
-    if actual == expected:
+    if actual == 0:
-        raise TestFailed, "test_%s: actual %d unexpected" % (name, actual)
+        raise TestFailed, "test_%s: unexpected zero" % name
 
 def run_test(name, thunk):
     if verbose:
@@ -48,7 +48,21 @@ def test_class():
     A.a = A
     gc.collect()
     del A
-    expect_not(gc.collect(), 0, "class")
+    expect_nonzero(gc.collect(), "class")
+
+def test_staticclass():
+    class A(object):
+        __dynamic__ = 0
+    gc.collect()
+    del A
+    expect_nonzero(gc.collect(), "staticclass")
+
+def test_dynamicclass():
+    class A(object):
+        __dynamic__ = 1
+    gc.collect()
+    del A
+    expect_nonzero(gc.collect(), "dynamicclass")
 
 def test_instance():
     class A:
@@ -57,7 +71,7 @@ def test_instance():
     a.a = a
     gc.collect()
     del a
-    expect_not(gc.collect(), 0, "instance")
+    expect_nonzero(gc.collect(), "instance")
 
 def test_method():
     # Tricky: self.__init__ is a bound method, it references the instance.
@@ -67,7 +81,7 @@ def test_method():
     a = A()
     gc.collect()
     del a
-    expect_not(gc.collect(), 0, "method")
+    expect_nonzero(gc.collect(), "method")
 
 def test_finalizer():
     # A() is uncollectable if it is part of a cycle, make sure it shows up
@@ -84,7 +98,7 @@ def test_finalizer():
     gc.collect()
     del a
     del b
-    expect_not(gc.collect(), 0, "finalizer")
+    expect_nonzero(gc.collect(), "finalizer")
     for obj in gc.garbage:
         if id(obj) == id_a:
             del obj.a
@@ -153,6 +167,8 @@ def test_all():
     run_test("dicts", test_dict)
     run_test("tuples", test_tuple)
     run_test("classes", test_class)
+    run_test("static classes", test_staticclass)
+    run_test("dynamic classes", test_dynamicclass)
     run_test("instances", test_instance)
     run_test("methods", test_method)
     run_test("functions", test_function)

Objects/descrobject.c

@@ -38,9 +38,10 @@ typedef struct {
 static void
 descr_dealloc(PyDescrObject *descr)
 {
+	_PyObject_GC_UNTRACK(descr);
 	Py_XDECREF(descr->d_type);
 	Py_XDECREF(descr->d_name);
-	PyObject_DEL(descr);
+	PyObject_GC_Del(descr);
 }
 
 static char *
@@ -352,6 +353,20 @@ static PyGetSetDef wrapper_getset[] = {
 	{0}
 };
 
+static int
+descr_traverse(PyObject *self, visitproc visit, void *arg)
+{
+	PyDescrObject *descr = (PyDescrObject *)self;
+	int err;
+
+	if (descr->d_type) {
+		err = visit((PyObject *)(descr->d_type), arg);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
 static PyTypeObject PyMethodDescr_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,
@@ -373,9 +388,9 @@ static PyTypeObject PyMethodDescr_Type = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
 	0,					/* tp_doc */
-	0,					/* tp_traverse */
+	descr_traverse,				/* tp_traverse */
 	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -411,9 +426,9 @@ static PyTypeObject PyMemberDescr_Type = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
 	0,					/* tp_doc */
-	0,					/* tp_traverse */
+	descr_traverse,				/* tp_traverse */
 	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -449,9 +464,9 @@ static PyTypeObject PyGetSetDescr_Type = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
 	0,					/* tp_doc */
-	0,					/* tp_traverse */
+	descr_traverse,				/* tp_traverse */
 	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -487,9 +502,9 @@ static PyTypeObject PyWrapperDescr_Type = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
 	0,					/* tp_doc */
-	0,					/* tp_traverse */
+	descr_traverse,				/* tp_traverse */
 	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -679,8 +694,9 @@ static PyMethodDef proxy_methods[] = {
 static void
 proxy_dealloc(proxyobject *pp)
 {
+	_PyObject_GC_UNTRACK(pp);
 	Py_DECREF(pp->dict);
-	PyObject_DEL(pp);
+	PyObject_GC_Del(pp);
 }
 
 static PyObject *
@@ -695,6 +711,20 @@ proxy_str(proxyobject *pp)
 	return PyObject_Str(pp->dict);
 }
 
+static int
+proxy_traverse(PyObject *self, visitproc visit, void *arg)
+{
+	proxyobject *pp = (proxyobject *)self;
+	int err;
+
+	if (pp->dict) {
+		err = visit(pp->dict, arg);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
 PyTypeObject proxytype = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -717,9 +747,9 @@ PyTypeObject proxytype = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
  	0,					/* tp_doc */
- 	0,					/* tp_traverse */
+	proxy_traverse,				/* tp_traverse */
  	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -739,10 +769,11 @@ PyDictProxy_New(PyObject *dict)
 {
 	proxyobject *pp;
 
-	pp = PyObject_NEW(proxyobject, &proxytype);
+	pp = PyObject_GC_New(proxyobject, &proxytype);
 	if (pp != NULL) {
 		Py_INCREF(dict);
 		pp->dict = dict;
+		_PyObject_GC_TRACK(pp);
 	}
 	return (PyObject *)pp;
 }
@@ -762,9 +793,10 @@ typedef struct {
 static void
 wrapper_dealloc(wrapperobject *wp)
 {
+	_PyObject_GC_UNTRACK(wp);
 	Py_XDECREF(wp->descr);
 	Py_XDECREF(wp->self);
-	PyObject_DEL(wp);
+	PyObject_GC_Del(wp);
 }
 
 static PyMethodDef wrapper_methods[] = {
@@ -808,6 +840,25 @@ wrapper_call(wrapperobject *wp, PyObject *args, PyObject *kwds)
 	return (*wrapper)(self, args, wp->descr->d_wrapped);
 }
 
+static int
+wrapper_traverse(PyObject *self, visitproc visit, void *arg)
+{
+	wrapperobject *wp = (wrapperobject *)self;
+	int err;
+
+	if (wp->descr) {
+		err = visit((PyObject *)(wp->descr), arg);
+		if (err)
+			return err;
+	}
+	if (wp->self) {
+		err = visit(wp->self, arg);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
 PyTypeObject wrappertype = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -830,9 +881,9 @@ PyTypeObject wrappertype = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,			/* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */
  	0,					/* tp_doc */
- 	0,					/* tp_traverse */
+	wrapper_traverse,			/* tp_traverse */
  	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -857,12 +908,13 @@ PyWrapper_New(PyObject *d, PyObject *self)
 	descr = (PyWrapperDescrObject *)d;
 	assert(PyObject_IsInstance(self, (PyObject *)(descr->d_type)));
 
-	wp = PyObject_NEW(wrapperobject, &wrappertype);
+	wp = PyObject_GC_New(wrapperobject, &wrappertype);
 	if (wp != NULL) {
 		Py_INCREF(descr);
 		wp->descr = descr;
 		Py_INCREF(self);
 		wp->self = self;
+		_PyObject_GC_TRACK(wp);
 	}
 	return (PyObject *)wp;
 }
@@ -919,6 +971,7 @@ property_dealloc(PyObject *self)
 {
 	propertyobject *gs = (propertyobject *)self;
 
+	_PyObject_GC_UNTRACK(self);
 	Py_XDECREF(gs->prop_get);
 	Py_XDECREF(gs->prop_set);
 	Py_XDECREF(gs->prop_del);
@@ -1012,6 +1065,26 @@ static char property_doc[] =
 "    def delx(self): del self.__x\n"
 "    x = property(getx, setx, delx, \"I'm the 'x' property.\")";
 
+static int
+property_traverse(PyObject *self, visitproc visit, void *arg)
+{
+	propertyobject *pp = (propertyobject *)self;
+	int err;
+
+#define VISIT(SLOT) \
+	if (pp->SLOT) { \
+		err = visit((PyObject *)(pp->SLOT), arg); \
+		if (err) \
+			return err; \
+	}
+
+	VISIT(prop_get);
+	VISIT(prop_set);
+	VISIT(prop_del);
+
+	return 0;
+}
+
 PyTypeObject PyProperty_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -1034,9 +1107,10 @@ PyTypeObject PyProperty_Type = {
 	PyObject_GenericGetAttr,		/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+		Py_TPFLAGS_BASETYPE,		/* tp_flags */
  	property_doc,				/* tp_doc */
- 	0,					/* tp_traverse */
+	property_traverse,			/* tp_traverse */
  	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -1053,5 +1127,5 @@ PyTypeObject PyProperty_Type = {
 	property_init,				/* tp_init */
 	PyType_GenericAlloc,			/* tp_alloc */
 	PyType_GenericNew,			/* tp_new */
-	_PyObject_Del,				/* tp_free */
+	_PyObject_GC_Del,			/* tp_free */
 };

Objects/typeobject.c

@@ -265,7 +265,7 @@ subtype_dealloc(PyObject *self)
 
 	/* Finalize GC if the base doesn't do GC and we do */
 	if (PyType_IS_GC(type) && !PyType_IS_GC(base))
-		PyObject_GC_Fini(self);
+		_PyObject_GC_UNTRACK(self);
 
 	/* Call the base tp_dealloc() */
 	assert(f);
@@ -864,6 +864,8 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
 		Py_TPFLAGS_BASETYPE;
 	if (dynamic)
 		type->tp_flags |= Py_TPFLAGS_DYNAMICTYPE;
+	if (base->tp_flags & Py_TPFLAGS_HAVE_GC)
+		type->tp_flags |= Py_TPFLAGS_HAVE_GC;
 
 	/* It's a new-style number unless it specifically inherits any
 	   old-style numeric behavior */
@@ -934,7 +936,8 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
 	else {
 		if (add_dict) {
 			if (base->tp_itemsize)
-				type->tp_dictoffset = -(long)sizeof(PyObject *);
+				type->tp_dictoffset =
+					-(long)sizeof(PyObject *);
 			else
 				type->tp_dictoffset = slotoffset;
 			slotoffset += sizeof(PyObject *);
@@ -966,7 +969,13 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
 
 	/* Always override allocation strategy to use regular heap */
 	type->tp_alloc = PyType_GenericAlloc;
-	type->tp_free = _PyObject_Del;
+	if (type->tp_flags & Py_TPFLAGS_HAVE_GC) {
+		type->tp_free = _PyObject_GC_Del;
+		type->tp_traverse = base->tp_traverse;
+		type->tp_clear = base->tp_clear;
+	}
+	else
+		type->tp_free = _PyObject_Del;
 
 	/* Initialize the rest */
 	if (PyType_Ready(type) < 0) {
@@ -1080,6 +1089,7 @@ type_dealloc(PyTypeObject *type)
 
 	/* Assert this is a heap-allocated type object */
 	assert(type->tp_flags & Py_TPFLAGS_HEAPTYPE);
+	_PyObject_GC_UNTRACK(type);
 	et = (etype *)type;
 	Py_XDECREF(type->tp_base);
 	Py_XDECREF(type->tp_dict);
@@ -1102,6 +1112,72 @@ static char type_doc[] =
 "type(object) -> the object's type\n"
 "type(name, bases, dict) -> a new type";
 
+static int
+type_traverse(PyTypeObject *type, visitproc visit, void *arg)
+{
+	etype *et;
+	int err;
+
+	if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE))
+		return 0;
+
+	et = (etype *)type;
+
+#define VISIT(SLOT) \
+	if (SLOT) { \
+		err = visit((PyObject *)(SLOT), arg); \
+		if (err) \
+			return err; \
+	}
+
+	VISIT(type->tp_dict);
+	VISIT(type->tp_defined);
+	VISIT(type->tp_mro);
+	VISIT(type->tp_bases);
+	VISIT(type->tp_base);
+	VISIT(et->slots);
+
+#undef VISIT
+
+	return 0;
+}
+
+static int
+type_clear(PyTypeObject *type)
+{
+	etype *et;
+	PyObject *tmp;
+
+	if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE))
+		return 0;
+
+	et = (etype *)type;
+
+#define CLEAR(SLOT) \
+	if (SLOT) { \
+		tmp = (PyObject *)(SLOT); \
+		SLOT = NULL; \
+		Py_DECREF(tmp); \
+	}
+
+	CLEAR(type->tp_dict);
+	CLEAR(type->tp_defined);
+	CLEAR(type->tp_mro);
+	CLEAR(type->tp_bases);
+	CLEAR(type->tp_base);
+	CLEAR(et->slots);
+
+#undef CLEAR
+
+	return 0;
+}
+
+static int
+type_is_gc(PyTypeObject *type)
+{
+	return type->tp_flags & Py_TPFLAGS_HEAPTYPE;
+}
+
 PyTypeObject PyType_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -1123,10 +1199,11 @@ PyTypeObject PyType_Type = {
 	(getattrofunc)type_getattro,		/* tp_getattro */
 	(setattrofunc)type_setattro,		/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+		Py_TPFLAGS_BASETYPE,		/* tp_flags */
 	type_doc,				/* tp_doc */
-	0,					/* tp_traverse */
+	(traverseproc)type_traverse,		/* tp_traverse */
-	0,					/* tp_clear */
+	(inquiry)type_clear,			/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
 	0,					/* tp_iter */
@@ -1142,6 +1219,8 @@ PyTypeObject PyType_Type = {
 	0,					/* tp_init */
 	0,					/* tp_alloc */
 	type_new,				/* tp_new */
+	_PyObject_GC_Del,			/* tp_free */
+	(inquiry)type_is_gc,			/* tp_is_gc */
 };
 
 
@@ -3531,6 +3610,7 @@ super_dealloc(PyObject *self)
 {
 	superobject *su = (superobject *)self;
 
+	_PyObject_GC_UNTRACK(self);
 	Py_XDECREF(su->obj);
 	Py_XDECREF(su->type);
 	self->ob_type->tp_free(self);
@@ -3666,6 +3746,27 @@ static char super_doc[] =
 "    def meth(self, arg):\n"
 "        super(C, self).meth(arg)";
 
+static int
+super_traverse(PyObject *self, visitproc visit, void *arg)
+{
+	superobject *su = (superobject *)self;
+	int err;
+
+#define VISIT(SLOT) \
+	if (SLOT) { \
+		err = visit((PyObject *)(SLOT), arg); \
+		if (err) \
+			return err; \
+	}
+
+	VISIT(su->obj);
+	VISIT(su->type);
+
+#undef VISIT
+
+	return 0;
+}
+
 PyTypeObject PySuper_Type = {
 	PyObject_HEAD_INIT(&PyType_Type)
 	0,					/* ob_size */
@@ -3688,9 +3789,10 @@ PyTypeObject PySuper_Type = {
 	super_getattro,				/* tp_getattro */
 	0,					/* tp_setattro */
 	0,					/* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
+	Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
+		Py_TPFLAGS_BASETYPE,		/* tp_flags */
  	super_doc,				/* tp_doc */
- 	0,					/* tp_traverse */
+ 	super_traverse,				/* tp_traverse */
  	0,					/* tp_clear */
 	0,					/* tp_richcompare */
 	0,					/* tp_weaklistoffset */
@@ -3707,5 +3809,5 @@ PyTypeObject PySuper_Type = {
 	super_init,				/* tp_init */
 	PyType_GenericAlloc,			/* tp_alloc */
 	PyType_GenericNew,			/* tp_new */
-	_PyObject_Del,				/* tp_free */
+	_PyObject_GC_Del,			/* tp_free */
 };