traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

SF patch 576101, by Oren Tirosh: alternative implementation of 

interning.  I modified Oren's patch significantly, but the basic idea
and most of the implementation is unchanged.  Interned strings created
with PyString_InternInPlace() are now mortal, and you must keep a
reference to the resulting string around; use the new function
PyString_InternImmortal() to create immortal interned strings.
This commit is contained in:
Guido van Rossum 2002-08-19 21:43:18 +00:00
parent d8dbf847b6
commit 45ec02aed1
7 changed files with 171 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Doc/lib/libfuncs.tex
View File

@ -518,8 +518,10 @@ def my_import(name):
be done by a pointer compare instead of a string compare. Normally,
the names used in Python programs are automatically interned, and
the dictionaries used to hold module, class or instance attributes
have interned keys. Interned strings are immortal (never get
garbage collected).
have interned keys. \versionchanged[Interned strings are not
immortal (like they used to be in Python 2.2 and before);
you must keep a reference to the return value of \function{intern()}
around to benefit from it]{2.3}
\end{funcdesc}
\begin{funcdesc}{isinstance}{object, classinfo}

7
Include/modsupport.h
View File

@ -23,8 +23,8 @@ PyAPI_FUNC(int) PyModule_AddObject(PyObject *, char *, PyObject *);
PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, char *, long);
PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, char *, char *);
#define PYTHON_API_VERSION 1011
#define PYTHON_API_STRING "1011"
#define PYTHON_API_VERSION 1012
#define PYTHON_API_STRING "1012"
/* The API version is maintained (independently from the Python version)
so we can detect mismatches between the interpreter and dynamically
loaded modules. These are diagnosed by an error message but
@ -38,6 +38,9 @@ PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, char *, char *);
Please add a line or two to the top of this log for each API
version change:
19-Aug-2002 GvR 1012 Changes to string object struct for
interning changes, saving 3 bytes.
17-Jul-2001 GvR 1011 Descr-branch, just to be on the safe side
25-Jan-2001 FLD 1010 Parameters added to PyCode_New() and

12
Include/stringobject.h
View File

@ -25,7 +25,7 @@ functions should be applied to nil objects.
*/
/* Caching the hash (ob_shash) saves recalculation of a string's hash value.
Interning strings (ob_sinterned) tries to ensure that only one string
Interning strings (ob_sstate) tries to ensure that only one string
object with a given value exists, so equality tests can be one pointer
comparison. This is generally restricted to strings that "look like"
Python identifiers, although the intern() builtin can be used to force
@ -35,10 +35,14 @@ functions should be applied to nil objects.
typedef struct {
PyObject_VAR_HEAD
long ob_shash;
PyObject *ob_sinterned;
int ob_sstate;
char ob_sval[1];
} PyStringObject;
#define SSTATE_NOT_INTERNED 0
#define SSTATE_INTERNED_MORTAL 1
#define SSTATE_INTERNED_IMMORTAL 2
PyAPI_DATA(PyTypeObject) PyBaseString_Type;
PyAPI_DATA(PyTypeObject) PyString_Type;
@ -66,9 +70,13 @@ extern DL_IMPORT(PyObject *) PyString_DecodeEscape(const char *, int,
const char *);
PyAPI_FUNC(void) PyString_InternInPlace(PyObject **);
PyAPI_FUNC(void) PyString_InternImmortal(PyObject **);
PyAPI_FUNC(PyObject *) PyString_InternFromString(const char *);
PyAPI_FUNC(void) _Py_ReleaseInternedStrings(void);
/* Use only if you know it's a string */
#define PyString_CHECK_INTERNED(op) (((PyStringObject *)(op))->ob_sstate)
/* Macro, trading safety for speed */
#define PyString_AS_STRING(op) (((PyStringObject *)(op))->ob_sval)
#define PyString_GET_SIZE(op) (((PyStringObject *)(op))->ob_size)

17
Misc/NEWS
View File

@ -57,6 +57,10 @@ Type/class unification and new-style classes
Core and builtins
- A subtle change to the semantics of the built-in function intern():
interned strings are no longer immortal. You must keep a reference
to the return value intern() around to get the benefit.
- Use of 'None' as a variable, argument or attribute name now
issues a SyntaxWarning. In the future, None may become a keyword.
@ -514,6 +518,19 @@ Build
C API
- The string object's layout has changed: the pointer member
ob_sinterned has been replaced by an int member ob_sstate. On some
platforms (e.g. most 64-bit systems) this may change the offset of
the ob_sval member, so as a precaution the API_VERSION has been
incremented. The apparently unused feature of "indirect interned
strings", supported by the ob_sinterned member, is gone. Interned
strings are now usually mortal; theres a new API,
PyString_InternImmortal() that creates immortal interned strings.
(The ob_sstate member can only take three values; however, while
making it a char saves a few bytes per string object on average, in
it also slowed things down a bit because ob_sval was no longer
aligned.)
- The Py_InitModule*() functions now accept NULL for the 'methods'
argument. Modules without global functions are becoming more common
now that factories can be types rather than functions.

44
Objects/classobject.c
View File

@ -2300,37 +2300,38 @@ instancemethod_traverse(PyMethodObject *im, visitproc visit, void *arg)
return 0;
}
static char *
getclassname(PyObject *class)
static void
getclassname(PyObject *class, char *buf, int bufsize)
{
PyObject *name;
assert(bufsize > 1);
strcpy(buf, "?"); /* Default outcome */
if (class == NULL)
name = NULL;
else
name = PyObject_GetAttrString(class, "__name__");
return;
name = PyObject_GetAttrString(class, "__name__");
if (name == NULL) {
/* This function cannot return an exception */
PyErr_Clear();
return "?";
return;
}
if (!PyString_Check(name)) {
Py_DECREF(name);
return "?";
if (PyString_Check(name)) {
strncpy(buf, PyString_AS_STRING(name), bufsize);
buf[bufsize-1] = '\0';
}
PyString_InternInPlace(&name);
Py_DECREF(name);
return PyString_AS_STRING(name);
}
static char *
getinstclassname(PyObject *inst)
static void
getinstclassname(PyObject *inst, char *buf, int bufsize)
{
PyObject *class;
char *name;
if (inst == NULL)
return "nothing";
if (inst == NULL) {
assert(bufsize > strlen("nothing"));
strcpy(buf, "nothing");
return;
}
class = PyObject_GetAttrString(inst, "__class__");
if (class == NULL) {
@ -2339,9 +2340,8 @@ getinstclassname(PyObject *inst)
class = (PyObject *)(inst->ob_type);
Py_INCREF(class);
}
name = getclassname(class);
getclassname(class, buf, bufsize);
Py_XDECREF(class);
return name;
}
static PyObject *
@ -2366,14 +2366,18 @@ instancemethod_call(PyObject *func, PyObject *arg, PyObject *kw)
return NULL;
}
if (!ok) {
char clsbuf[256];
char instbuf[256];
getclassname(class, clsbuf, sizeof(clsbuf));
getinstclassname(self, instbuf, sizeof(instbuf));
PyErr_Format(PyExc_TypeError,
"unbound method %s%s must be called with "
"%s instance as first argument "
"(got %s%s instead)",
PyEval_GetFuncName(func),
PyEval_GetFuncDesc(func),
getclassname(class),
getinstclassname(self),
clsbuf,
instbuf,
self == NULL ? "" : " instance");
return NULL;
}

12
Objects/dictobject.c
View File

@ -511,15 +511,9 @@ PyDict_SetItem(register PyObject *op, PyObject *key, PyObject *value)
}
mp = (dictobject *)op;
if (PyString_CheckExact(key)) {
if (((PyStringObject *)key)->ob_sinterned != NULL) {
key = ((PyStringObject *)key)->ob_sinterned;
hash = ((PyStringObject *)key)->ob_shash;
}
else {
hash = ((PyStringObject *)key)->ob_shash;
if (hash == -1)
hash = PyObject_Hash(key);
}
hash = ((PyStringObject *)key)->ob_shash;
if (hash == -1)
hash = PyObject_Hash(key);
}
else {
hash = PyObject_Hash(key);

179
Objects/stringobject.c
View File

@ -15,6 +15,17 @@ int null_strings, one_strings;
static PyStringObject *characters[UCHAR_MAX + 1];
static PyStringObject *nullstring;
/* This dictionary holds all interned strings. Note that references to
strings in this dictionary are *not* counted in the string's ob_refcnt.
When the interned string reaches a refcnt of 0 the string deallocation
function will delete the reference from this dictionary.
Another way to look at this is that to say that the actual reference
count of a string is: s->ob_refcnt + (s->ob_sstate?2:0)
*/
static PyObject *interned;
/*
For both PyString_FromString() and PyString_FromStringAndSize(), the
parameter `size' denotes number of characters to allocate, not counting any
@ -69,7 +80,7 @@ PyString_FromStringAndSize(const char *str, int size)
return PyErr_NoMemory();
PyObject_INIT_VAR(op, &PyString_Type, size);
op->ob_shash = -1;
op->ob_sinterned = NULL;
op->ob_sstate = SSTATE_NOT_INTERNED;
if (str != NULL)
memcpy(op->ob_sval, str, size);
op->ob_sval[size] = '\0';
@ -125,7 +136,7 @@ PyString_FromString(const char *str)
return PyErr_NoMemory();
PyObject_INIT_VAR(op, &PyString_Type, size);
op->ob_shash = -1;
op->ob_sinterned = NULL;
op->ob_sstate = SSTATE_NOT_INTERNED;
memcpy(op->ob_sval, str, size+1);
/* share short strings */
if (size == 0) {
@ -486,6 +497,24 @@ PyObject *PyString_AsEncodedString(PyObject *str,
static void
string_dealloc(PyObject *op)
{
switch (PyString_CHECK_INTERNED(op)) {
case SSTATE_NOT_INTERNED:
break;
case SSTATE_INTERNED_MORTAL:
/* revive dead object temporarily for DelItem */
op->ob_refcnt = 3;
if (PyDict_DelItem(interned, op) != 0)
Py_FatalError(
"deletion of interned string failed");
break;
case SSTATE_INTERNED_IMMORTAL:
Py_FatalError("Immortal interned string died.");
default:
Py_FatalError("Inconsistent interned string state.");
}
op->ob_type->tp_free(op);
}
@ -885,7 +914,7 @@ string_concat(register PyStringObject *a, register PyObject *bb)
return PyErr_NoMemory();
PyObject_INIT_VAR(op, &PyString_Type, size);
op->ob_shash = -1;
op->ob_sinterned = NULL;
op->ob_sstate = SSTATE_NOT_INTERNED;
memcpy(op->ob_sval, a->ob_sval, (int) a->ob_size);
memcpy(op->ob_sval + a->ob_size, b->ob_sval, (int) b->ob_size);
op->ob_sval[size] = '\0';
@ -928,7 +957,7 @@ string_repeat(register PyStringObject *a, register int n)
return PyErr_NoMemory();
PyObject_INIT_VAR(op, &PyString_Type, size);
op->ob_shash = -1;
op->ob_sinterned = NULL;
op->ob_sstate = SSTATE_NOT_INTERNED;
for (i = 0; i < size; i += a->ob_size)
memcpy(op->ob_sval+i, a->ob_sval, (int) a->ob_size);
op->ob_sval[size] = '\0';
@ -1093,9 +1122,6 @@ string_hash(PyStringObject *a)
if (a->ob_shash != -1)
return a->ob_shash;
if (a->ob_sinterned != NULL)
return (a->ob_shash =
((PyStringObject *)(a->ob_sinterned))->ob_shash);
len = a->ob_size;
p = (unsigned char *) a->ob_sval;
x = *p << 7;
@ -3067,8 +3093,7 @@ str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
memcpy(PyString_AS_STRING(pnew), PyString_AS_STRING(tmp), n+1);
((PyStringObject *)pnew)->ob_shash =
((PyStringObject *)tmp)->ob_shash;
((PyStringObject *)pnew)->ob_sinterned =
((PyStringObject *)tmp)->ob_sinterned;
((PyStringObject *)pnew)->ob_sstate = SSTATE_NOT_INTERNED;
}
Py_DECREF(tmp);
return pnew;
@ -3983,22 +4008,6 @@ PyString_Format(PyObject *format, PyObject *args)
return NULL;
}
/* This dictionary will leak at PyString_Fini() time. That's acceptable
* because PyString_Fini() specifically frees interned strings that are
* only referenced by this dictionary. The CVS log entry for revision 2.45
* says:
*
* Change the Fini function to only remove otherwise unreferenced
* strings from the interned table. There are references in
* hard-to-find static variables all over the interpreter, and it's not
* worth trying to get rid of all those; but "uninterning" isn't fair
* either and may cause subtle failures later -- so we have to keep them
* in the interned table.
*/
static PyObject *interned;
void
PyString_InternInPlace(PyObject **p)
{
@ -4006,49 +4015,57 @@ PyString_InternInPlace(PyObject **p)
PyObject *t;
if (s == NULL || !PyString_Check(s))
Py_FatalError("PyString_InternInPlace: strings only please!");
if ((t = s->ob_sinterned) != NULL) {
if (t == (PyObject *)s)
return;
Py_INCREF(t);
*p = t;
Py_DECREF(s);
if (PyString_CHECK_INTERNED(s))
return;
}
if (interned == NULL) {
interned = PyDict_New();
if (interned == NULL)
if (interned == NULL) {
PyErr_Clear(); /* Don't leave an exception */
return;
}
}
if ((t = PyDict_GetItem(interned, (PyObject *)s)) != NULL) {
Py_INCREF(t);
*p = s->ob_sinterned = t;
Py_DECREF(s);
Py_DECREF(*p);
*p = t;
return;
}
/* Ensure that only true string objects appear in the intern dict,
and as the value of ob_sinterned. */
if (PyString_CheckExact(s)) {
t = (PyObject *)s;
if (PyDict_SetItem(interned, t, t) == 0) {
s->ob_sinterned = t;
return;
}
}
else {
/* Ensure that only true string objects appear in the intern dict */
if (!PyString_CheckExact(s)) {
t = PyString_FromStringAndSize(PyString_AS_STRING(s),
PyString_GET_SIZE(s));
if (t != NULL) {
if (PyDict_SetItem(interned, t, t) == 0) {
*p = s->ob_sinterned = t;
Py_DECREF(s);
return;
}
Py_DECREF(t);
if (t == NULL) {
PyErr_Clear();
return;
}
} else {
t = (PyObject*) s;
Py_INCREF(t);
}
if (PyDict_SetItem(interned, t, t) == 0) {
/* The two references in interned are not counted by
refcnt. The string deallocator will take care of this */
((PyObject *)t)->ob_refcnt-=2;
PyString_CHECK_INTERNED(t) = SSTATE_INTERNED_MORTAL;
Py_DECREF(*p);
*p = t;
return;
}
Py_DECREF(t);
PyErr_Clear();
}
void
PyString_InternImmortal(PyObject **p)
{
PyString_InternInPlace(p);
if (PyString_CHECK_INTERNED(*p) != SSTATE_INTERNED_IMMORTAL) {
PyString_CHECK_INTERNED(*p) = SSTATE_INTERNED_IMMORTAL;
Py_INCREF(*p);
}
}
PyObject *
PyString_InternFromString(const char *cp)
@ -4070,28 +4087,48 @@ PyString_Fini(void)
}
Py_XDECREF(nullstring);
nullstring = NULL;
if (interned) {
int pos, changed;
PyObject *key, *value;
do {
changed = 0;
pos = 0;
while (PyDict_Next(interned, &pos, &key, &value)) {
if (key->ob_refcnt == 2 && key == value) {
PyDict_DelItem(interned, key);
changed = 1;
}
}
} while (changed);
}
}
void _Py_ReleaseInternedStrings(void)
{
if (interned) {
fprintf(stderr, "releasing interned strings\n");
PyDict_Clear(interned);
Py_DECREF(interned);
interned = NULL;
PyObject *keys;
PyStringObject *s;
int i, n;
if (interned == NULL || !PyDict_Check(interned))
return;
keys = PyDict_Keys(interned);
if (keys == NULL || !PyList_Check(keys)) {
PyErr_Clear();
return;
}
/* Since _Py_ReleaseInternedStrings() is intended to help a leak
detector, interned strings are not forcibly deallocated; rather, we
give them their stolen references back, and then clear and DECREF
the interned dict. */
fprintf(stderr, "releasing interned strings\n");
n = PyList_GET_SIZE(keys);
for (i = 0; i < n; i++) {
s = (PyStringObject *) PyList_GET_ITEM(keys, i);
switch (s->ob_sstate) {
case SSTATE_NOT_INTERNED:
/* XXX Shouldn't happen */
break;
case SSTATE_INTERNED_IMMORTAL:
s->ob_refcnt += 1;
break;
case SSTATE_INTERNED_MORTAL:
s->ob_refcnt += 2;
break;
default:
Py_FatalError("Inconsistent interned string state.");
}
s->ob_sstate = SSTATE_NOT_INTERNED;
}
Py_DECREF(keys);
PyDict_Clear(interned);
Py_DECREF(interned);
interned = NULL;
}