Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-113993: Allow interned strings to be mortal, and fix related issues (GH-120520) 

* Add an InternalDocs file describing how interning should work and how to use it.

* Add internal functions to *explicitly* request what kind of interning is done:
  - `_PyUnicode_InternMortal`
  - `_PyUnicode_InternImmortal`
  - `_PyUnicode_InternStatic`

* Switch uses of `PyUnicode_InternInPlace` to those.

* Disallow using `_Py_SetImmortal` on strings directly.
  You should use `_PyUnicode_InternImmortal` instead:
  - Strings should be interned before immortalization, otherwise you're possibly
    interning a immortalizing copy.
  - `_Py_SetImmortal` doesn't handle the `SSTATE_INTERNED_MORTAL` to
    `SSTATE_INTERNED_IMMORTAL` update, and those flags can't be changed in
    backports, as they are now part of public API and version-specific ABI.

* Add private `_only_immortal` argument for `sys.getunicodeinternedsize`, used in refleak test machinery.

* Make sure the statically allocated string singletons are unique. This means these sets are now disjoint:
  - `_Py_ID`
  - `_Py_STR` (including the empty string)
  - one-character latin-1 singletons

  Now, when you intern a singleton, that exact singleton will be interned.

* Add a `_Py_LATIN1_CHR` macro, use it instead of `_Py_ID`/`_Py_STR` for one-character latin-1 singletons everywhere (including Clinic).

* Intern `_Py_STR` singletons at startup.

* For free-threaded builds, intern `_Py_LATIN1_CHR` singletons at startup.

* Beef up the tests. Cover internal details (marked with `@cpython_only`).

* Add lots of assertions

Co-Authored-By: Eric Snow <ericsnowcurrently@gmail.com>
This commit is contained in:
Petr Viktorin 2024-06-21 17:19:31 +02:00 committed by GitHub
parent 7595e6743a
commit 6f1d448bc1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
42 changed files with 2464 additions and 1140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Include/internal/pycore_global_objects_fini_generated.h generated
View File

@ -551,12 +551,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_setcomp));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_string));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(anon_unknown));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(close_br));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_close_br));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_open_br));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dbl_percent));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(defaults));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format));
@ -564,9 +562,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(newline));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(open_br));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(percent));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(CANCELLED));
@ -578,7 +573,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__));
@ -772,6 +766,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_only_immortal));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_showwarnmsg));
@ -784,7 +779,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_uninitialized_submodules));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_warn_unawaited_coroutine));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_xoptions));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(a));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(abs_tol));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(access));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(aclose));
@ -808,7 +802,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(b));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(base));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before));
@ -826,7 +819,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byteorder));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes_per_sep));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_call));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_exception));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return));
@ -882,7 +874,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(count));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(covariant));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cwd));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(d));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day));
@ -911,7 +902,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory));
@ -935,7 +925,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(f));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false));
@ -968,7 +957,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fset));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(g));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(genexpr));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get));
@ -982,7 +970,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(globals));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groupindex));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle_seq));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(has_location));
@ -1093,7 +1080,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_fields));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_sequence_fields));
@ -1139,7 +1125,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(outgoing));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(overlapped));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(owner));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(p));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(password));
@ -1167,7 +1152,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ps2));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(query));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(quotetabs));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(r));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(raw));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(read1));
@ -1191,7 +1175,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(return));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reverse));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reversed));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(s));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler));
@ -1298,7 +1281,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(writable));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(write_through));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(x));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(year));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(zdict));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_SINGLETON(strings).ascii[0]);

24
Include/internal/pycore_global_strings.h
View File

@ -37,12 +37,10 @@ struct _Py_global_strings {
STRUCT_FOR_STR(anon_setcomp, "<setcomp>")
STRUCT_FOR_STR(anon_string, "<string>")
STRUCT_FOR_STR(anon_unknown, "<unknown>")
STRUCT_FOR_STR(close_br, "}")
STRUCT_FOR_STR(dbl_close_br, "}}")
STRUCT_FOR_STR(dbl_open_br, "{{")
STRUCT_FOR_STR(dbl_percent, "%%")
STRUCT_FOR_STR(defaults, ".defaults")
STRUCT_FOR_STR(dot, ".")
STRUCT_FOR_STR(dot_locals, ".<locals>")
STRUCT_FOR_STR(empty, "")
STRUCT_FOR_STR(format, ".format")
@ -50,9 +48,6 @@ struct _Py_global_strings {
STRUCT_FOR_STR(json_decoder, "json.decoder")
STRUCT_FOR_STR(kwdefaults, ".kwdefaults")
STRUCT_FOR_STR(list_err, "list index out of range")
STRUCT_FOR_STR(newline, "\n")
STRUCT_FOR_STR(open_br, "{")
STRUCT_FOR_STR(percent, "%")
STRUCT_FOR_STR(type_params, ".type_params")
STRUCT_FOR_STR(utf_8, "utf-8")
} literals;
@ -67,7 +62,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(TextIOWrapper)
STRUCT_FOR_ID(True)
STRUCT_FOR_ID(WarningMessage)
STRUCT_FOR_ID(_)
STRUCT_FOR_ID(_WindowsConsoleIO)
STRUCT_FOR_ID(__IOBase_closed)
STRUCT_FOR_ID(__abc_tpflags__)
@ -261,6 +255,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(_lock_unlock_module)
STRUCT_FOR_ID(_loop)
STRUCT_FOR_ID(_needs_com_addref_)
STRUCT_FOR_ID(_only_immortal)
STRUCT_FOR_ID(_pack_)
STRUCT_FOR_ID(_restype_)
STRUCT_FOR_ID(_showwarnmsg)
@ -273,7 +268,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(_uninitialized_submodules)
STRUCT_FOR_ID(_warn_unawaited_coroutine)
STRUCT_FOR_ID(_xoptions)
STRUCT_FOR_ID(a)
STRUCT_FOR_ID(abs_tol)
STRUCT_FOR_ID(access)
STRUCT_FOR_ID(aclose)
@ -297,7 +291,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(attribute)
STRUCT_FOR_ID(authorizer_callback)
STRUCT_FOR_ID(autocommit)
STRUCT_FOR_ID(b)
STRUCT_FOR_ID(backtick)
STRUCT_FOR_ID(base)
STRUCT_FOR_ID(before)
@ -315,7 +308,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(byteorder)
STRUCT_FOR_ID(bytes)
STRUCT_FOR_ID(bytes_per_sep)
STRUCT_FOR_ID(c)
STRUCT_FOR_ID(c_call)
STRUCT_FOR_ID(c_exception)
STRUCT_FOR_ID(c_return)
@ -371,7 +363,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(count)
STRUCT_FOR_ID(covariant)
STRUCT_FOR_ID(cwd)
STRUCT_FOR_ID(d)
STRUCT_FOR_ID(data)
STRUCT_FOR_ID(database)
STRUCT_FOR_ID(day)
@ -400,7 +391,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(dont_inherit)
STRUCT_FOR_ID(dst)
STRUCT_FOR_ID(dst_dir_fd)
STRUCT_FOR_ID(e)
STRUCT_FOR_ID(eager_start)
STRUCT_FOR_ID(effective_ids)
STRUCT_FOR_ID(element_factory)
@ -424,7 +414,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(exp)
STRUCT_FOR_ID(extend)
STRUCT_FOR_ID(extra_tokens)
STRUCT_FOR_ID(f)
STRUCT_FOR_ID(facility)
STRUCT_FOR_ID(factory)
STRUCT_FOR_ID(false)
@ -457,7 +446,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(fset)
STRUCT_FOR_ID(func)
STRUCT_FOR_ID(future)
STRUCT_FOR_ID(g)
STRUCT_FOR_ID(generation)
STRUCT_FOR_ID(genexpr)
STRUCT_FOR_ID(get)
@ -471,7 +459,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(globals)
STRUCT_FOR_ID(groupindex)
STRUCT_FOR_ID(groups)
STRUCT_FOR_ID(h)
STRUCT_FOR_ID(handle)
STRUCT_FOR_ID(handle_seq)
STRUCT_FOR_ID(has_location)
@ -582,7 +569,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(msg)
STRUCT_FOR_ID(mutex)
STRUCT_FOR_ID(mycmp)
STRUCT_FOR_ID(n)
STRUCT_FOR_ID(n_arg)
STRUCT_FOR_ID(n_fields)
STRUCT_FOR_ID(n_sequence_fields)
@ -628,7 +614,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(outgoing)
STRUCT_FOR_ID(overlapped)
STRUCT_FOR_ID(owner)
STRUCT_FOR_ID(p)
STRUCT_FOR_ID(pages)
STRUCT_FOR_ID(parent)
STRUCT_FOR_ID(password)
@ -656,7 +641,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(ps2)
STRUCT_FOR_ID(query)
STRUCT_FOR_ID(quotetabs)
STRUCT_FOR_ID(r)
STRUCT_FOR_ID(raw)
STRUCT_FOR_ID(read)
STRUCT_FOR_ID(read1)
@ -680,7 +664,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(return)
STRUCT_FOR_ID(reverse)
STRUCT_FOR_ID(reversed)
STRUCT_FOR_ID(s)
STRUCT_FOR_ID(salt)
STRUCT_FOR_ID(sched_priority)
STRUCT_FOR_ID(scheduler)
@ -787,7 +770,6 @@ struct _Py_global_strings {
STRUCT_FOR_ID(writable)
STRUCT_FOR_ID(write)
STRUCT_FOR_ID(write_through)
STRUCT_FOR_ID(x)
STRUCT_FOR_ID(year)
STRUCT_FOR_ID(zdict)
} identifiers;
@ -810,6 +792,10 @@ struct _Py_global_strings {
(_Py_SINGLETON(strings.identifiers._py_ ## NAME._ascii.ob_base))
#define _Py_STR(NAME) \
(_Py_SINGLETON(strings.literals._py_ ## NAME._ascii.ob_base))
#define _Py_LATIN1_CHR(CH) \
((CH) < 128 \
? (PyObject*)&_Py_SINGLETON(strings).ascii[(CH)] \
: (PyObject*)&_Py_SINGLETON(strings).latin1[(CH) - 128])
/* _Py_DECLARE_STR() should precede all uses of _Py_STR() in a function.

20
Include/internal/pycore_runtime_init_generated.h generated
View File

@ -546,12 +546,10 @@ extern "C" {
INIT_STR(anon_setcomp, "<setcomp>"), \
INIT_STR(anon_string, "<string>"), \
INIT_STR(anon_unknown, "<unknown>"), \
INIT_STR(close_br, "}"), \
INIT_STR(dbl_close_br, "}}"), \
INIT_STR(dbl_open_br, "{{"), \
INIT_STR(dbl_percent, "%%"), \
INIT_STR(defaults, ".defaults"), \
INIT_STR(dot, "."), \
INIT_STR(dot_locals, ".<locals>"), \
INIT_STR(empty, ""), \
INIT_STR(format, ".format"), \
@ -559,9 +557,6 @@ extern "C" {
INIT_STR(json_decoder, "json.decoder"), \
INIT_STR(kwdefaults, ".kwdefaults"), \
INIT_STR(list_err, "list index out of range"), \
INIT_STR(newline, "\n"), \
INIT_STR(open_br, "{"), \
INIT_STR(percent, "%"), \
INIT_STR(type_params, ".type_params"), \
INIT_STR(utf_8, "utf-8"), \
}
@ -576,7 +571,6 @@ extern "C" {
INIT_ID(TextIOWrapper), \
INIT_ID(True), \
INIT_ID(WarningMessage), \
INIT_ID(_), \
INIT_ID(_WindowsConsoleIO), \
INIT_ID(__IOBase_closed), \
INIT_ID(__abc_tpflags__), \
@ -770,6 +764,7 @@ extern "C" {
INIT_ID(_lock_unlock_module), \
INIT_ID(_loop), \
INIT_ID(_needs_com_addref_), \
INIT_ID(_only_immortal), \
INIT_ID(_pack_), \
INIT_ID(_restype_), \
INIT_ID(_showwarnmsg), \
@ -782,7 +777,6 @@ extern "C" {
INIT_ID(_uninitialized_submodules), \
INIT_ID(_warn_unawaited_coroutine), \
INIT_ID(_xoptions), \
INIT_ID(a), \
INIT_ID(abs_tol), \
INIT_ID(access), \
INIT_ID(aclose), \
@ -806,7 +800,6 @@ extern "C" {
INIT_ID(attribute), \
INIT_ID(authorizer_callback), \
INIT_ID(autocommit), \
INIT_ID(b), \
INIT_ID(backtick), \
INIT_ID(base), \
INIT_ID(before), \
@ -824,7 +817,6 @@ extern "C" {
INIT_ID(byteorder), \
INIT_ID(bytes), \
INIT_ID(bytes_per_sep), \
INIT_ID(c), \
INIT_ID(c_call), \
INIT_ID(c_exception), \
INIT_ID(c_return), \
@ -880,7 +872,6 @@ extern "C" {
INIT_ID(count), \
INIT_ID(covariant), \
INIT_ID(cwd), \
INIT_ID(d), \
INIT_ID(data), \
INIT_ID(database), \
INIT_ID(day), \
@ -909,7 +900,6 @@ extern "C" {
INIT_ID(dont_inherit), \
INIT_ID(dst), \
INIT_ID(dst_dir_fd), \
INIT_ID(e), \
INIT_ID(eager_start), \
INIT_ID(effective_ids), \
INIT_ID(element_factory), \
@ -933,7 +923,6 @@ extern "C" {
INIT_ID(exp), \
INIT_ID(extend), \
INIT_ID(extra_tokens), \
INIT_ID(f), \
INIT_ID(facility), \
INIT_ID(factory), \
INIT_ID(false), \
@ -966,7 +955,6 @@ extern "C" {
INIT_ID(fset), \
INIT_ID(func), \
INIT_ID(future), \
INIT_ID(g), \
INIT_ID(generation), \
INIT_ID(genexpr), \
INIT_ID(get), \
@ -980,7 +968,6 @@ extern "C" {
INIT_ID(globals), \
INIT_ID(groupindex), \
INIT_ID(groups), \
INIT_ID(h), \
INIT_ID(handle), \
INIT_ID(handle_seq), \
INIT_ID(has_location), \
@ -1091,7 +1078,6 @@ extern "C" {
INIT_ID(msg), \
INIT_ID(mutex), \
INIT_ID(mycmp), \
INIT_ID(n), \
INIT_ID(n_arg), \
INIT_ID(n_fields), \
INIT_ID(n_sequence_fields), \
@ -1137,7 +1123,6 @@ extern "C" {
INIT_ID(outgoing), \
INIT_ID(overlapped), \
INIT_ID(owner), \
INIT_ID(p), \
INIT_ID(pages), \
INIT_ID(parent), \
INIT_ID(password), \
@ -1165,7 +1150,6 @@ extern "C" {
INIT_ID(ps2), \
INIT_ID(query), \
INIT_ID(quotetabs), \
INIT_ID(r), \
INIT_ID(raw), \
INIT_ID(read), \
INIT_ID(read1), \
@ -1189,7 +1173,6 @@ extern "C" {
INIT_ID(return), \
INIT_ID(reverse), \
INIT_ID(reversed), \
INIT_ID(s), \
INIT_ID(salt), \
INIT_ID(sched_priority), \
INIT_ID(scheduler), \
@ -1296,7 +1279,6 @@ extern "C" {
INIT_ID(writable), \
INIT_ID(write), \
INIT_ID(write_through), \
INIT_ID(x), \
INIT_ID(year), \
INIT_ID(zdict), \
}

15
Include/internal/pycore_unicodeobject.h
View File

@ -12,6 +12,7 @@ extern "C" {
#include "pycore_fileutils.h" // _Py_error_handler
#include "pycore_identifier.h" // _Py_Identifier
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
#include "pycore_global_objects.h" // _Py_SINGLETON
/* --- Characters Type APIs ----------------------------------------------- */
@ -33,6 +34,7 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
PyAPI_FUNC(void) _PyUnicode_ExactDealloc(PyObject *op);
extern Py_ssize_t _PyUnicode_InternedSize(void);
extern Py_ssize_t _PyUnicode_InternedSize_Immortal(void);
// Get a copy of a Unicode string.
// Export for '_datetime' shared extension.
@ -275,6 +277,18 @@ extern void _PyUnicode_FiniTypes(PyInterpreterState *);
extern PyTypeObject _PyUnicodeASCIIIter_Type;
/* --- Interning ---------------------------------------------------------- */
// All these are "ref-neutral", like the public PyUnicode_InternInPlace.
// Explicit interning routines:
PyAPI_FUNC(void) _PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **);
PyAPI_FUNC(void) _PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **);
// Left here to help backporting:
PyAPI_FUNC(void) _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p);
// Only for singletons in the _PyRuntime struct:
extern void _PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **);
/* --- Other API ---------------------------------------------------------- */
struct _Py_unicode_runtime_ids {
@ -311,7 +325,6 @@ struct _Py_unicode_state {
struct _Py_unicode_ids ids;
};
extern void _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p);
extern void _PyUnicode_ClearInterned(PyInterpreterState *interp);
// Like PyUnicode_AsUTF8(), but check for embedded null characters.

2288
Include/internal/pycore_unicodeobject_generated.h generated
View File

File diff suppressed because it is too large Load Diff

122
InternalDocs/string_interning.md Normal file
View File

@ -0,0 +1,122 @@
# String interning
*Interned* strings are conceptually part of an interpreter-global
*set* of interned strings, meaning that:
- no two interned strings have the same content (across an interpreter);
- two interned strings can be safely compared using pointer equality
(Python `is`).
This is used to optimize dict and attribute lookups, among other things.
Python uses three different mechanisms to intern strings:
- Singleton strings marked in C source with `_Py_STR` and `_Py_ID` macros.
These are statically allocated, and collected using `make regen-global-objects`
(`Tools/build/generate_global_objects.py`), which generates code
for declaration, initialization and finalization.
The difference between the two kinds is not important. (A `_Py_ID` string is
a valid C name, with which we can refer to it; a `_Py_STR` may e.g. contain
non-identifier characters, so it needs a separate C-compatible name.)
The empty string is in this category (as `_Py_STR(empty)`).
These singletons are interned in a runtime-global lookup table,
`_PyRuntime.cached_objects.interned_strings` (`INTERNED_STRINGS`),
at runtime initialization.
- The 256 possible one-character latin-1 strings are singletons,
which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in runtime-global
arrays, `_PyRuntime.static_objects.strings.ascii` and
`_PyRuntime.static_objects.strings.latin1`.
These are NOT interned at startup in the normal build.
In the free-threaded build, they are; this avoids modifying the
global lookup table after threads are started.
Interning a one-char latin-1 string will always intern the corresponding
singleton.
- All other strings are allocated dynamically, and have their
`_PyUnicode_STATE(s).statically_allocated` flag set to zero.
When interned, such strings are added to an interpreter-wide dict,
`PyInterpreterState.cached_objects.interned_strings`.
The key and value of each entry in this dict reference the same object.
The three sets of singletons (`_Py_STR`, `_Py_ID`, `_Py_LATIN1_CHR`)
are disjoint.
If you have such a singleton, it (and no other copy) will be interned.
## Immortality and reference counting
Invariant: Every immortal string is interned, *except* the one-char latin-1
singletons (which might but might not be interned).
In practice, this means that you must not use `_Py_SetImmortal` on
a string. (If you know it's already immortal, don't immortalize it;
if you know it's not interned you might be immortalizing a redundant copy;
if it's interned and mortal it needs extra processing in
`_PyUnicode_InternImmortal`.)
The converse is not true: interned strings can be mortal.
For mortal interned strings:
- the 2 references from the interned dict (key & value) are excluded from
their refcount
- the deallocator (`unicode_dealloc`) removes the string from the interned dict
- at shutdown, when the interned dict is cleared, the references are added back
As with any type, you should only immortalize strings that will live until
interpreter shutdown.
We currently also immortalize strings contained in code objects and similar,
specifically in the compiler and in `marshal`.
These are “close enough” to immortal: even in use cases like hot reloading
or `eval`-ing user input, the number of distinct identifiers and string
constants expected to stay low.
## Internal API
We have the following *internal* API for interning:
- `_PyUnicode_InternMortal`: just intern the string
- `_PyUnicode_InternImmortal`: intern, and immortalize the result
- `_PyUnicode_InternStatic`: intern a static singleton (`_Py_STR`, `_Py_ID`
or one-byte). Not for general use.
All take an interpreter state, and a pointer to a `PyObject*` which they
modify in place.
The functions take ownership of (“steal”) the reference to their argument,
and update the argument with a *new* reference.
This means:
- They're “reference neutral”.
- They must not be called with a borrowed reference.
## State
The intern state (retrieved by `PyUnicode_CHECK_INTERNED(s)`;
stored in `_PyUnicode_STATE(s).interned`) can be:
- `SSTATE_NOT_INTERNED` (defined as 0, which is useful in a boolean context)
- `SSTATE_INTERNED_MORTAL` (1)
- `SSTATE_INTERNED_IMMORTAL` (2)
- `SSTATE_INTERNED_IMMORTAL_STATIC` (3)
The valid transitions between these states are:
- For dynamically allocated strings:
- 0 -> 1 (`_PyUnicode_InternMortal`)
- 1 -> 2 or 0 -> 2 (`_PyUnicode_InternImmortal`)
Using `_PyUnicode_InternStatic` on these is an error; the other cases
don't change the state.
- One-char latin-1 singletons can be interned (0 -> 3) using any interning
function; after that the functions don't change the state.
- Other statically allocated strings are interned (0 -> 3) at runtime init;
after that all interning functions don't change the state.

104
Lib/test/clinic.test.c
View File

@ -2179,7 +2179,7 @@ test_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2213,7 +2213,7 @@ exit:
static PyObject *
test_keywords_impl(PyObject *module, PyObject *a, PyObject *b)
/*[clinic end generated code: output=73d46a9ae3320f96 input=0d3484844749c05b]*/
/*[clinic end generated code: output=13ba007e1c842a37 input=0d3484844749c05b]*/
/*[clinic input]
@ -2249,7 +2249,7 @@ test_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2283,7 +2283,7 @@ exit:
static PyObject *
test_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b)
/*[clinic end generated code: output=c9f02a41f425897d input=384adc78bfa0bff7]*/
/*[clinic end generated code: output=789799a6d2d6eb4d input=384adc78bfa0bff7]*/
/*[clinic input]
@ -2320,7 +2320,7 @@ test_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2367,7 +2367,7 @@ exit:
static PyObject *
test_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c)
/*[clinic end generated code: output=b35d4e66f7283e46 input=eda7964f784f4607]*/
/*[clinic end generated code: output=42430dd8ea5afde6 input=eda7964f784f4607]*/
/*[clinic input]
@ -2406,7 +2406,7 @@ test_keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nar
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2464,7 +2464,7 @@ exit:
static PyObject *
test_keywords_opt_kwonly_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c, PyObject *d)
/*[clinic end generated code: output=ede7e6e65106bf2b input=209387a4815e5082]*/
/*[clinic end generated code: output=f312c35c380d2bf9 input=209387a4815e5082]*/
/*[clinic input]
@ -2502,7 +2502,7 @@ test_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2549,7 +2549,7 @@ exit:
static PyObject *
test_keywords_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c)
/*[clinic end generated code: output=36d4df939a4c3eef input=18393cc64fa000f4]*/
/*[clinic end generated code: output=3937da2a8233ebe0 input=18393cc64fa000f4]*/
/*[clinic input]
@ -2585,7 +2585,7 @@ test_posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2619,7 +2619,7 @@ exit:
static PyObject *
test_posonly_keywords_impl(PyObject *module, PyObject *a, PyObject *b)
/*[clinic end generated code: output=4835f4b6cf386c28 input=1767b0ebdf06060e]*/
/*[clinic end generated code: output=6b4f6dd5f4db3877 input=1767b0ebdf06060e]*/
/*[clinic input]
@ -2656,7 +2656,7 @@ test_posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2690,7 +2690,7 @@ exit:
static PyObject *
test_posonly_kwonly_impl(PyObject *module, PyObject *a, PyObject *c)
/*[clinic end generated code: output=2570ea156a8d3cb5 input=9042f2818f664839]*/
/*[clinic end generated code: output=8bef2a8198e70b26 input=9042f2818f664839]*/
/*[clinic input]
@ -2729,7 +2729,7 @@ test_posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2766,7 +2766,7 @@ exit:
static PyObject *
test_posonly_keywords_kwonly_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c)
/*[clinic end generated code: output=aaa0e6b5ce02900d input=29546ebdca492fea]*/
/*[clinic end generated code: output=a44b8ae8300955e1 input=29546ebdca492fea]*/
/*[clinic input]
@ -2805,7 +2805,7 @@ test_posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t na
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2854,7 +2854,7 @@ exit:
static PyObject *
test_posonly_keywords_opt_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c, PyObject *d)
/*[clinic end generated code: output=1d9f2d8420d0a85f input=cdf5a9625e554e9b]*/
/*[clinic end generated code: output=cae6647c9e8e0238 input=cdf5a9625e554e9b]*/
/*[clinic input]
@ -2892,7 +2892,7 @@ test_posonly_keywords_opt2(PyObject *module, PyObject *const *args, Py_ssize_t n
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2939,7 +2939,7 @@ exit:
static PyObject *
test_posonly_keywords_opt2_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c)
/*[clinic end generated code: output=a83caa0505b296cf input=1581299d21d16f14]*/
/*[clinic end generated code: output=6526fd08aafa2149 input=1581299d21d16f14]*/
/*[clinic input]
@ -2978,7 +2978,7 @@ test_posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3032,7 +3032,7 @@ exit:
static PyObject *
test_posonly_opt_keywords_opt_impl(PyObject *module, PyObject *a,
PyObject *b, PyObject *c, PyObject *d)
/*[clinic end generated code: output=0b24fba3dc04d26b input=408798ec3d42949f]*/
/*[clinic end generated code: output=b8d01e98443738c2 input=408798ec3d42949f]*/
/*[clinic input]
@ -3072,7 +3072,7 @@ test_posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3121,7 +3121,7 @@ exit:
static PyObject *
test_posonly_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c, PyObject *d)
/*[clinic end generated code: output=592b217bca2f7bcc input=8d8e5643bbbc2309]*/
/*[clinic end generated code: output=81d71c288f13d4dc input=8d8e5643bbbc2309]*/
/*[clinic input]
@ -3160,7 +3160,7 @@ test_posonly_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ssize_t nar
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3207,7 +3207,7 @@ exit:
static PyObject *
test_posonly_kwonly_opt2_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c)
/*[clinic end generated code: output=b8b00420826bc11f input=f7e5eed94f75fff0]*/
/*[clinic end generated code: output=a717d2a1a3310289 input=f7e5eed94f75fff0]*/
/*[clinic input]
@ -3247,7 +3247,7 @@ test_posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3301,7 +3301,7 @@ exit:
static PyObject *
test_posonly_opt_kwonly_opt_impl(PyObject *module, PyObject *a, PyObject *b,
PyObject *c, PyObject *d)
/*[clinic end generated code: output=3b9ee879ebee285a input=1e557dc979d120fd]*/
/*[clinic end generated code: output=0f50b4b8d45cf2de input=1e557dc979d120fd]*/
/*[clinic input]
@ -3343,7 +3343,7 @@ test_posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssi
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3395,7 +3395,7 @@ static PyObject *
test_posonly_keywords_kwonly_opt_impl(PyObject *module, PyObject *a,
PyObject *b, PyObject *c, PyObject *d,
PyObject *e)
/*[clinic end generated code: output=d380f84f81cc0e45 input=c3884a4f956fdc89]*/
/*[clinic end generated code: output=8dac8d2a4e6105fa input=c3884a4f956fdc89]*/
/*[clinic input]
@ -3435,7 +3435,7 @@ test_posonly_keywords_kwonly_opt2(PyObject *module, PyObject *const *args, Py_ss
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3484,7 +3484,7 @@ exit:
static PyObject *
test_posonly_keywords_kwonly_opt2_impl(PyObject *module, PyObject *a,
PyObject *b, PyObject *c, PyObject *d)
/*[clinic end generated code: output=ee629e962cb06992 input=68d01d7c0f6dafb0]*/
/*[clinic end generated code: output=5a96d521e6414f5d input=68d01d7c0f6dafb0]*/
/*[clinic input]
@ -3527,7 +3527,7 @@ test_posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3588,7 +3588,7 @@ static PyObject *
test_posonly_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a,
PyObject *b, PyObject *c,
PyObject *d, PyObject *e)
/*[clinic end generated code: output=a2721babb42ecfd1 input=d0883d45876f186c]*/
/*[clinic end generated code: output=d5a474dcd5dc3e9f input=d0883d45876f186c]*/
/*[clinic input]
@ -3631,7 +3631,7 @@ test_posonly_keywords_opt2_kwonly_opt(PyObject *module, PyObject *const *args, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3697,7 +3697,7 @@ static PyObject *
test_posonly_keywords_opt2_kwonly_opt_impl(PyObject *module, PyObject *a,
PyObject *b, PyObject *c,
PyObject *d, PyObject *e)
/*[clinic end generated code: output=0626203eedb6e7e8 input=c95e2e1ec93035ad]*/
/*[clinic end generated code: output=ac239c5ee8a74408 input=c95e2e1ec93035ad]*/
/*[clinic input]
@ -3742,7 +3742,7 @@ test_posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), },
.ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3816,7 +3816,7 @@ test_posonly_opt_keywords_opt_kwonly_opt_impl(PyObject *module, PyObject *a,
PyObject *b, PyObject *c,
PyObject *d, PyObject *e,
PyObject *f)
/*[clinic end generated code: output=07d8acc04558a5a0 input=9914857713c5bbf8]*/
/*[clinic end generated code: output=638bbd0005639342 input=9914857713c5bbf8]*/
/*[clinic input]
test_keyword_only_parameter
@ -4212,7 +4212,7 @@ test_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -4247,7 +4247,7 @@ exit:
static PyObject *
test_vararg_impl(PyObject *module, PyObject *a, PyObject *args)
/*[clinic end generated code: output=880365c61ae205d7 input=81d33815ad1bae6e]*/
/*[clinic end generated code: output=1411e464f358a7ba input=81d33815ad1bae6e]*/
/*[clinic input]
test_vararg_with_default
@ -4284,7 +4284,7 @@ test_vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nar
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -4330,7 +4330,7 @@ exit:
static PyObject *
test_vararg_with_default_impl(PyObject *module, PyObject *a, PyObject *args,
int b)
/*[clinic end generated code: output=291e9a5a09831128 input=6e110b54acd9b22d]*/
/*[clinic end generated code: output=f09d4b917063ca41 input=6e110b54acd9b22d]*/
/*[clinic input]
test_vararg_with_only_defaults
@ -4367,7 +4367,7 @@ test_vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -4418,7 +4418,7 @@ exit:
static PyObject *
test_vararg_with_only_defaults_impl(PyObject *module, PyObject *args, int b,
PyObject *c)
/*[clinic end generated code: output=dd21b28f0db26a4b input=fa56a709a035666e]*/
/*[clinic end generated code: output=cc6590b8805d5433 input=fa56a709a035666e]*/
/*[clinic input]
test_paramname_module
@ -4685,7 +4685,7 @@ Test_cls_with_param(TestObj *self, PyTypeObject *cls, PyObject *const *args, Py_
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -4720,7 +4720,7 @@ exit:
static PyObject *
Test_cls_with_param_impl(TestObj *self, PyTypeObject *cls, int a)
/*[clinic end generated code: output=d89b99e83d442be0 input=af158077bd237ef9]*/
/*[clinic end generated code: output=a3a968137b0f320a input=af158077bd237ef9]*/
/*[clinic input]
@ -5033,7 +5033,7 @@ Test___init__(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -5067,7 +5067,7 @@ exit:
static int
Test___init___impl(TestObj *self, PyObject *a)
/*[clinic end generated code: output=0b9ca79638ab3ecb input=a8f9222a6ab35c59]*/
/*[clinic end generated code: output=0e1239b9bc247bc1 input=a8f9222a6ab35c59]*/
/*[clinic input]
@ -5301,7 +5301,7 @@ mangled_c_keyword_identifier(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(i), },
.ob_item = { _Py_LATIN1_CHR('i'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -5336,7 +5336,7 @@ exit:
static PyObject *
mangled_c_keyword_identifier_impl(PyObject *module, int int_value)
/*[clinic end generated code: output=f24b37e0368e0eb8 input=060876448ab567a2]*/
/*[clinic end generated code: output=01a8088b57632916 input=060876448ab567a2]*/
/*[clinic input]
@ -5648,7 +5648,7 @@ docstr_fallback_to_converter_default(PyObject *module, PyObject *const *args, Py
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -5682,7 +5682,7 @@ exit:
static PyObject *
docstr_fallback_to_converter_default_impl(PyObject *module, str a)
/*[clinic end generated code: output=ae24a9c6f60ee8a6 input=0cbe6a4d24bc2274]*/
/*[clinic end generated code: output=3fff7b702f0065cb input=0cbe6a4d24bc2274]*/
/*[clinic input]

12
Lib/test/libregrtest/refleak.py
View File

@ -110,7 +110,7 @@ def runtest_refleak(test_name, test_func,
getunicodeinternedsize = sys.getunicodeinternedsize
fd_count = os_helper.fd_count
# initialize variables to make pyflakes quiet
rc_before = alloc_before = fd_before = interned_before = 0
rc_before = alloc_before = fd_before = interned_immortal_before = 0
if not quiet:
print("beginning", repcount, "repetitions. Showing number of leaks "
@ -141,9 +141,11 @@ def runtest_refleak(test_name, test_func,
# Also, readjust the reference counts and alloc blocks by ignoring
# any strings that might have been interned during test_func. These
# strings will be deallocated at runtime shutdown
interned_after = getunicodeinternedsize()
alloc_after = getallocatedblocks() - interned_after
rc_after = gettotalrefcount() - interned_after * 2
interned_immortal_after = getunicodeinternedsize(
# Use an internal-only keyword argument that mypy doesn't know yet
_only_immortal=True) # type: ignore[call-arg]
alloc_after = getallocatedblocks() - interned_immortal_after
rc_after = gettotalrefcount() - interned_immortal_after * 2
fd_after = fd_count()
rc_deltas[i] = get_pooled_int(rc_after - rc_before)
@ -170,7 +172,7 @@ def runtest_refleak(test_name, test_func,
alloc_before = alloc_after
rc_before = rc_after
fd_before = fd_after
interned_before = interned_after
interned_immortal_before = interned_immortal_after
restore_support_xml(xml_filename)

2
Lib/test/test_clinic.py
View File

@ -2624,7 +2624,7 @@ class ClinicExternalTest(TestCase):
# Verify by checking the checksum.
checksum = (
"/*[clinic end generated code: "
"output=c16447c01510dfb3 input=9543a8d2da235301]*/\n"
"output=0acbef4794cb933e input=9543a8d2da235301]*/\n"
)
with open(fn, encoding='utf-8') as f:
generated = f.read()

55
Lib/test/test_sys.py
View File

@ -731,8 +731,11 @@ class SysModuleTest(unittest.TestCase):
if has_is_interned:
self.assertIs(sys._is_interned(S("abc")), False)
@support.cpython_only
@requires_subinterpreters
def test_subinterp_intern_dynamically_allocated(self):
# Implementation detail: Dynamically allocated strings
# are distinct between interpreters
s = "never interned before" + str(random.randrange(0, 10**9))
t = sys.intern(s)
self.assertIs(t, s)
@ -740,24 +743,58 @@ class SysModuleTest(unittest.TestCase):
interp = interpreters.create()
interp.exec(textwrap.dedent(f'''
import sys
t = sys.intern({s!r})
# set `s`, avoid parser interning & constant folding
s = str({s.encode()!r}, 'utf-8')
t = sys.intern(s)
assert id(t) != {id(s)}, (id(t), {id(s)})
assert id(t) != {id(t)}, (id(t), {id(t)})
'''))
@support.cpython_only
@requires_subinterpreters
def test_subinterp_intern_statically_allocated(self):
# Implementation detail: Statically allocated strings are shared
# between interpreters.
# See Tools/build/generate_global_objects.py for the list
# of strings that are always statically allocated.
s = '__init__'
t = sys.intern(s)
for s in ('__init__', 'CANCELLED', '<module>', 'utf-8',
'{{', '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff',
):
with self.subTest(s=s):
t = sys.intern(s)
interp = interpreters.create()
interp.exec(textwrap.dedent(f'''
import sys
t = sys.intern({s!r})
assert id(t) == {id(t)}, (id(t), {id(t)})
'''))
interp = interpreters.create()
interp.exec(textwrap.dedent(f'''
import sys
# set `s`, avoid parser interning & constant folding
s = str({s.encode()!r}, 'utf-8')
t = sys.intern(s)
assert id(t) == {id(t)}, (id(t), {id(t)})
'''))
@support.cpython_only
@requires_subinterpreters
def test_subinterp_intern_singleton(self):
# Implementation detail: singletons are used for 0- and 1-character
# latin1 strings.
for s in '', '\n', '_', 'x', '\0', '\N{CEDILLA}', '\xff':
with self.subTest(s=s):
interp = interpreters.create()
interp.exec(textwrap.dedent(f'''
import sys
# set `s`, avoid parser interning & constant folding
s = str({s.encode()!r}, 'utf-8')
assert id(s) == {id(s)}
t = sys.intern(s)
'''))
self.assertTrue(sys._is_interned(s))
def test_sys_flags(self):
self.assertTrue(sys.flags)

5
Misc/NEWS.d/next/Core and Builtins/2024-06-14-22-02-25.gh-issue-113993.MiA0vX.rst Normal file
View File

@ -0,0 +1,5 @@
Strings interned with :func:`sys.intern` are again garbage-collected when no
longer used, as per the documentation. Strings interned with the C function
:c:func:`PyUnicode_InternInPlace` are still immortal. Internals of the
string interning mechanism have been changed. This may affect performance
and identities of :class:`str` objects.

2
Modules/_io/stringio.c
View File

@ -196,7 +196,7 @@ write_str(stringio *self, PyObject *obj)
}
if (self->writenl) {
PyObject *translated = PyUnicode_Replace(
decoded, &_Py_STR(newline), self->writenl, -1);
decoded, _Py_LATIN1_CHR('\n'), self->writenl, -1);
Py_SETREF(decoded, translated);
}
if (decoded == NULL)

11
Modules/_operator.c
View File

@ -2,6 +2,7 @@
#include "pycore_modsupport.h" // _PyArg_NoKwnames()
#include "pycore_moduleobject.h" // _PyModule_GetState()
#include "pycore_runtime.h" // _Py_ID()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "clinic/_operator.c.h"
@ -1236,6 +1237,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
return NULL;
/* prepare attr while checking args */
PyInterpreterState *interp = _PyInterpreterState_GET();
for (idx = 0; idx < nattrs; ++idx) {
PyObject *item = PyTuple_GET_ITEM(args, idx);
int dot_count;
@ -1259,7 +1261,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (dot_count == 0) {
Py_INCREF(item);
PyUnicode_InternInPlace(&item);
_PyUnicode_InternMortal(interp, &item);
PyTuple_SET_ITEM(attr, idx, item);
} else { /* make it a tuple of non-dotted attrnames */
PyObject *attr_chain = PyTuple_New(dot_count + 1);
@ -1285,7 +1287,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
Py_DECREF(attr);
return NULL;
}
PyUnicode_InternInPlace(&attr_chain_item);
_PyUnicode_InternMortal(interp, &attr_chain_item);
PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item);
++attr_chain_idx;
unibuff_till = unibuff_from = unibuff_till + 1;
@ -1299,7 +1301,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
Py_DECREF(attr);
return NULL;
}
PyUnicode_InternInPlace(&attr_chain_item);
_PyUnicode_InternMortal(interp, &attr_chain_item);
PyTuple_SET_ITEM(attr_chain, attr_chain_idx, attr_chain_item);
PyTuple_SET_ITEM(attr, idx, attr_chain);
@ -1662,7 +1664,8 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
}
Py_INCREF(name);
PyUnicode_InternInPlace(&name);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternMortal(interp, &name);
mc->name = name;
mc->xargs = Py_XNewRef(args); // allows us to use borrowed references

9
Modules/_pickle.c
View File

@ -1807,8 +1807,7 @@ get_dotted_path(PyObject *obj, PyObject *name)
{
PyObject *dotted_path;
Py_ssize_t i, n;
_Py_DECLARE_STR(dot, ".");
dotted_path = PyUnicode_Split(name, &_Py_STR(dot), -1);
dotted_path = PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1);
if (dotted_path == NULL)
return NULL;
n = PyList_GET_SIZE(dotted_path);
@ -6605,8 +6604,10 @@ load_build(PickleState *st, UnpicklerObject *self)
/* normally the keys for instance attributes are
interned. we should try to do that here. */
Py_INCREF(d_key);
if (PyUnicode_CheckExact(d_key))
PyUnicode_InternInPlace(&d_key);
if (PyUnicode_CheckExact(d_key)) {
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternMortal(interp, &d_key);
}
if (PyObject_SetItem(dict, d_key, d_value) < 0) {
Py_DECREF(d_key);
goto error;

4
Modules/_sqlite/clinic/connection.c.h generated
View File

@ -844,7 +844,7 @@ pysqlite_connection_set_progress_handler(pysqlite_Connection *self, PyTypeObject
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(progress_handler), &_Py_ID(n), },
.ob_item = { &_Py_ID(progress_handler), _Py_LATIN1_CHR('n'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1866,4 +1866,4 @@ exit:
#ifndef DESERIALIZE_METHODDEF
#define DESERIALIZE_METHODDEF
#endif /* !defined(DESERIALIZE_METHODDEF) */
/*[clinic end generated code: output=7d41a178b7b2b683 input=a9049054013a1b77]*/
/*[clinic end generated code: output=5d4d7e4abe17b164 input=a9049054013a1b77]*/

10
Modules/clinic/_bisectmodule.c.h generated
View File

@ -44,7 +44,7 @@ _bisect_bisect_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -152,7 +152,7 @@ _bisect_insort_right(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -257,7 +257,7 @@ _bisect_bisect_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -365,7 +365,7 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(x), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('x'), &_Py_ID(lo), &_Py_ID(hi), &_Py_ID(key), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -434,4 +434,4 @@ skip_optional_kwonly:
exit:
return return_value;
}
/*[clinic end generated code: output=4af5bd405149bf5f input=a9049054013a1b77]*/
/*[clinic end generated code: output=0a8d5a32dd0a3f04 input=a9049054013a1b77]*/

4
Modules/clinic/_hashopenssl.c.h generated
View File

@ -1347,7 +1347,7 @@ _hashlib_scrypt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(password), &_Py_ID(salt), &_Py_ID(n), &_Py_ID(r), &_Py_ID(p), &_Py_ID(maxmem), &_Py_ID(dklen), },
.ob_item = { &_Py_ID(password), &_Py_ID(salt), _Py_LATIN1_CHR('n'), _Py_LATIN1_CHR('r'), _Py_LATIN1_CHR('p'), &_Py_ID(maxmem), &_Py_ID(dklen), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1824,4 +1824,4 @@ exit:
#ifndef _HASHLIB_SCRYPT_METHODDEF
#define _HASHLIB_SCRYPT_METHODDEF
#endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */
/*[clinic end generated code: output=b7eddeb3d6ccdeec input=a9049054013a1b77]*/
/*[clinic end generated code: output=fef43fd9f4dbea49 input=a9049054013a1b77]*/

42
Modules/clinic/_testclinic.c.h generated
View File

@ -1458,7 +1458,7 @@ keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1514,7 +1514,7 @@ keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1570,7 +1570,7 @@ keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1639,7 +1639,7 @@ keywords_opt_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1719,7 +1719,7 @@ keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1787,7 +1787,7 @@ posonly_keywords(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1843,7 +1843,7 @@ posonly_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1900,7 +1900,7 @@ posonly_keywords_kwonly(PyObject *module, PyObject *const *args, Py_ssize_t narg
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1959,7 +1959,7 @@ posonly_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs,
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2030,7 +2030,7 @@ posonly_opt_keywords_opt(PyObject *module, PyObject *const *args, Py_ssize_t nar
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2106,7 +2106,7 @@ posonly_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2177,7 +2177,7 @@ posonly_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t nargs
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2253,7 +2253,7 @@ posonly_keywords_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2328,7 +2328,7 @@ posonly_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_ssiz
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2412,7 +2412,7 @@ posonly_opt_keywords_opt_kwonly_opt(PyObject *module, PyObject *const *args, Py_
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2491,7 +2491,7 @@ keyword_only_parameter(PyObject *module, PyObject *const *args, Py_ssize_t nargs
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2546,7 +2546,7 @@ posonly_vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2641,7 +2641,7 @@ vararg(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwna
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2699,7 +2699,7 @@ vararg_with_default(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2766,7 +2766,7 @@ vararg_with_only_defaults(PyObject *module, PyObject *const *args, Py_ssize_t na
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -3418,4 +3418,4 @@ _testclinic_TestClass_get_defining_class_arg(PyObject *self, PyTypeObject *cls,
exit:
return return_value;
}
/*[clinic end generated code: output=545d409a47f1826d input=a9049054013a1b77]*/
/*[clinic end generated code: output=0d0ceed6c46547bb input=a9049054013a1b77]*/

62
Modules/clinic/_testclinic_depr.c.h generated
View File

@ -48,7 +48,7 @@ depr_star_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -133,7 +133,7 @@ depr_star_new_clone(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyO
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -215,7 +215,7 @@ depr_star_init(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -300,7 +300,7 @@ depr_star_init_clone(PyObject *self, PyObject *const *args, Py_ssize_t nargs, Py
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -372,7 +372,7 @@ depr_star_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -451,7 +451,7 @@ depr_kwd_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -534,7 +534,7 @@ depr_kwd_init(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -608,7 +608,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -635,7 +635,7 @@ depr_kwd_init_noinline(PyObject *self, PyObject *args, PyObject *kwargs)
&a, &b, &c, &d, &d_length)) {
goto exit;
}
if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, &_Py_ID(c))))) {
if (kwargs && PyDict_GET_SIZE(kwargs) && ((nargs < 2) || (nargs < 3 && PyDict_Contains(kwargs, _Py_LATIN1_CHR('c'))))) {
if (PyErr_Occurred()) { // PyDict_Contains() above can fail
goto exit;
}
@ -692,7 +692,7 @@ depr_star_pos0_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), },
.ob_item = { _Py_LATIN1_CHR('a'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -770,7 +770,7 @@ depr_star_pos0_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -851,7 +851,7 @@ depr_star_pos0_len3_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -935,7 +935,7 @@ depr_star_pos1_len1_opt(PyObject *module, PyObject *const *args, Py_ssize_t narg
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1020,7 +1020,7 @@ depr_star_pos1_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1101,7 +1101,7 @@ depr_star_pos1_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1186,7 +1186,7 @@ depr_star_pos2_len1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1269,7 +1269,7 @@ depr_star_pos2_len2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1354,7 +1354,7 @@ depr_star_pos2_len2_with_kwd(PyObject *module, PyObject *const *args, Py_ssize_t
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1441,7 +1441,7 @@ depr_star_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1525,7 +1525,7 @@ depr_star_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1618,7 +1618,7 @@ depr_kwd_required_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1699,7 +1699,7 @@ depr_kwd_required_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1780,7 +1780,7 @@ depr_kwd_optional_1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), },
.ob_item = { _Py_LATIN1_CHR('b'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1866,7 +1866,7 @@ depr_kwd_optional_2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1959,7 +1959,7 @@ depr_kwd_optional_3(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2057,7 +2057,7 @@ depr_kwd_required_optional(PyObject *module, PyObject *const *args, Py_ssize_t n
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2145,7 +2145,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2171,7 +2171,7 @@ depr_kwd_noinline(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO
&a, &b, &c, &d, &d_length)) {
goto exit;
}
if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, &_Py_ID(c))))) {
if (kwnames && PyTuple_GET_SIZE(kwnames) && ((nargs < 2) || (nargs < 3 && PySequence_Contains(kwnames, _Py_LATIN1_CHR('c'))))) {
if (PyErr_Occurred()) { // PySequence_Contains() above can fail
goto exit;
}
@ -2232,7 +2232,7 @@ depr_kwd_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), &_Py_ID(h), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), _Py_LATIN1_CHR('h'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2333,7 +2333,7 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(b), &_Py_ID(c), &_Py_ID(d), &_Py_ID(e), &_Py_ID(f), &_Py_ID(g), },
.ob_item = { _Py_LATIN1_CHR('b'), _Py_LATIN1_CHR('c'), _Py_LATIN1_CHR('d'), _Py_LATIN1_CHR('e'), _Py_LATIN1_CHR('f'), _Py_LATIN1_CHR('g'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -2393,4 +2393,4 @@ depr_multi(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *
exit:
return return_value;
}
/*[clinic end generated code: output=2c19d1804ba6e53b input=a9049054013a1b77]*/
/*[clinic end generated code: output=ca6da2c7137554be input=a9049054013a1b77]*/

4
Modules/clinic/_testmultiphase.c.h generated
View File

@ -88,7 +88,7 @@ _testmultiphase_StateAccessType_increment_count_clinic(StateAccessTypeObject *se
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(n), &_Py_ID(twice), },
.ob_item = { _Py_LATIN1_CHR('n'), &_Py_ID(twice), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -162,4 +162,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj
}
return _testmultiphase_StateAccessType_get_count_impl(self, cls);
}
/*[clinic end generated code: output=2c199bad52e9cda7 input=a9049054013a1b77]*/
/*[clinic end generated code: output=59cb50dae2d11dc1 input=a9049054013a1b77]*/

4
Modules/clinic/cmathmodule.c.h generated
View File

@ -908,7 +908,7 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -982,4 +982,4 @@ skip_optional_kwonly:
exit:
return return_value;
}
/*[clinic end generated code: output=364093af55bfe53f input=a9049054013a1b77]*/
/*[clinic end generated code: output=454309b21cfa9bf6 input=a9049054013a1b77]*/

10
Modules/clinic/itertoolsmodule.c.h generated
View File

@ -47,7 +47,7 @@ batched_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(iterable), &_Py_ID(n), &_Py_ID(strict), },
.ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('n'), &_Py_ID(strict), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -509,7 +509,7 @@ itertools_combinations(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(iterable), &_Py_ID(r), },
.ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -580,7 +580,7 @@ itertools_combinations_with_replacement(PyTypeObject *type, PyObject *args, PyOb
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(iterable), &_Py_ID(r), },
.ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -650,7 +650,7 @@ itertools_permutations(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(iterable), &_Py_ID(r), },
.ob_item = { &_Py_ID(iterable), _Py_LATIN1_CHR('r'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -928,4 +928,4 @@ skip_optional_pos:
exit:
return return_value;
}
/*[clinic end generated code: output=c6a515f765da86b5 input=a9049054013a1b77]*/
/*[clinic end generated code: output=7b13be3075f2d6d3 input=a9049054013a1b77]*/

4
Modules/clinic/mathmodule.c.h generated
View File

@ -648,7 +648,7 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(a), &_Py_ID(b), &_Py_ID(rel_tol), &_Py_ID(abs_tol), },
.ob_item = { _Py_LATIN1_CHR('a'), _Py_LATIN1_CHR('b'), &_Py_ID(rel_tol), &_Py_ID(abs_tol), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -1011,4 +1011,4 @@ math_ulp(PyObject *module, PyObject *arg)
exit:
return return_value;
}
/*[clinic end generated code: output=9fe3f007f474e015 input=a9049054013a1b77]*/
/*[clinic end generated code: output=7d03f84f77342496 input=a9049054013a1b77]*/

4
Modules/clinic/posixmodule.c.h generated
View File

@ -2117,7 +2117,7 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(s), },
.ob_item = { _Py_LATIN1_CHR('s'), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
@ -12837,4 +12837,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored))
#ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
#define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF
#endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */
/*[clinic end generated code: output=5ae2e5ffcd9c8a84 input=a9049054013a1b77]*/
/*[clinic end generated code: output=31dc3bb3cba924d0 input=a9049054013a1b77]*/

14
Objects/codeobject.c
View File

@ -137,6 +137,7 @@ static PyObject *intern_one_constant(PyObject *op);
static int
intern_strings(PyObject *tuple)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
Py_ssize_t i;
for (i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
@ -146,7 +147,7 @@ intern_strings(PyObject *tuple)
"non-string found in code slot");
return -1;
}
PyUnicode_InternInPlace(&_PyTuple_ITEMS(tuple)[i]);
_PyUnicode_InternMortal(interp, &_PyTuple_ITEMS(tuple)[i]);
}
return 0;
}
@ -157,12 +158,13 @@ intern_strings(PyObject *tuple)
static int
intern_constants(PyObject *tuple, int *modified)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
PyObject *v = PyTuple_GET_ITEM(tuple, i);
if (PyUnicode_CheckExact(v)) {
if (should_intern_string(v)) {
PyObject *w = v;
PyUnicode_InternInPlace(&v);
_PyUnicode_InternMortal(interp, &v);
if (w != v) {
PyTuple_SET_ITEM(tuple, i, v);
if (modified) {
@ -458,12 +460,13 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
con->stacksize = 1;
}
PyInterpreterState *interp = _PyInterpreterState_GET();
co->co_filename = Py_NewRef(con->filename);
co->co_name = Py_NewRef(con->name);
co->co_qualname = Py_NewRef(con->qualname);
PyUnicode_InternInPlace(&co->co_filename);
PyUnicode_InternInPlace(&co->co_name);
PyUnicode_InternInPlace(&co->co_qualname);
_PyUnicode_InternMortal(interp, &co->co_filename);
_PyUnicode_InternMortal(interp, &co->co_name);
_PyUnicode_InternMortal(interp, &co->co_qualname);
co->co_flags = con->flags;
co->co_firstlineno = con->firstlineno;
@ -489,7 +492,6 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con)
co->co_framesize = nlocalsplus + con->stacksize + FRAME_SPECIALS_SIZE;
co->co_ncellvars = ncellvars;
co->co_nfreevars = nfreevars;
PyInterpreterState *interp = _PyInterpreterState_GET();
#ifdef Py_GIL_DISABLED
PyMutex_Lock(&interp->func_state.mutex);
#endif

3
Objects/dictobject.c
View File

@ -4923,7 +4923,8 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item)
kv = PyUnicode_FromString(key);
if (kv == NULL)
return -1;
PyUnicode_InternInPlace(&kv); /* XXX Should we really? */
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &kv); /* XXX Should we really? */
err = PyDict_SetItem(v, kv, item);
Py_DECREF(kv);
return err;

10
Objects/object.c
View File

@ -1326,7 +1326,8 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value)
}
Py_INCREF(name);
PyUnicode_InternInPlace(&name);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternMortal(interp, &name);
if (tp->tp_setattro != NULL) {
err = (*tp->tp_setattro)(v, name, value);
Py_DECREF(name);
@ -2405,6 +2406,13 @@ _Py_NewReferenceNoTotal(PyObject *op)
void
_Py_SetImmortalUntracked(PyObject *op)
{
#ifdef Py_DEBUG
// For strings, use _PyUnicode_InternImmortal instead.
if (PyUnicode_CheckExact(op)) {
assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL
|| PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC);
}
#endif
#ifdef Py_GIL_DISABLED
op->ob_tid = _Py_UNOWNED_TID;
op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL;

10
Objects/typeobject.c
View File

@ -1393,8 +1393,10 @@ type_module(PyTypeObject *type)
if (s != NULL) {
mod = PyUnicode_FromStringAndSize(
type->tp_name, (Py_ssize_t)(s - type->tp_name));
if (mod != NULL)
PyUnicode_InternInPlace(&mod);
if (mod != NULL) {
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternMortal(interp, &mod);
}
}
else {
mod = &_Py_ID(builtins);
@ -5692,9 +5694,9 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value)
if (name == NULL)
return -1;
}
/* bpo-40521: Interned strings are shared by all subinterpreters */
if (!PyUnicode_CHECK_INTERNED(name)) {
PyUnicode_InternInPlace(&name);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternMortal(interp, &name);
if (!PyUnicode_CHECK_INTERNED(name)) {
PyErr_SetString(PyExc_MemoryError,
"Out of memory interning an attribute name");

552
Objects/unicodeobject.c
View File

@ -177,10 +177,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently
*_to++ = (to_type) *_iter++; \
} while (0)
#define LATIN1(ch) \
(ch < 128 \
? (PyObject*)&_Py_SINGLETON(strings).ascii[ch] \
: (PyObject*)&_Py_SINGLETON(strings).latin1[ch - 128])
#define LATIN1 _Py_LATIN1_CHR
#ifdef MS_WINDOWS
/* On Windows, overallocate by 50% is the best factor */
@ -220,18 +217,20 @@ static inline PyObject* unicode_get_empty(void)
return &_Py_STR(empty);
}
/* This dictionary holds all interned unicode strings. Note that references
to strings in this dictionary are *not* counted in the string's ob_refcnt.
When the interned string reaches a refcnt of 0 the string deallocation
function will delete the reference from this dictionary.
*/
/* This dictionary holds per-interpreter interned strings.
* See InternalDocs/string_interning.md for details.
*/
static inline PyObject *get_interned_dict(PyInterpreterState *interp)
{
return _Py_INTERP_CACHED_OBJECT(interp, interned_strings);
}
/* This hashtable holds statically allocated interned strings.
* See InternalDocs/string_interning.md for details.
*/
#define INTERNED_STRINGS _PyRuntime.cached_objects.interned_strings
/* Get number of all interned strings for the current interpreter. */
Py_ssize_t
_PyUnicode_InternedSize(void)
{
@ -239,6 +238,27 @@ _PyUnicode_InternedSize(void)
return _Py_hashtable_len(INTERNED_STRINGS) + PyDict_GET_SIZE(dict);
}
/* Get number of immortal interned strings for the current interpreter. */
Py_ssize_t
_PyUnicode_InternedSize_Immortal(void)
{
PyObject *dict = get_interned_dict(_PyInterpreterState_GET());
PyObject *key, *value;
Py_ssize_t pos = 0;
Py_ssize_t count = 0;
// It's tempting to keep a count and avoid a loop here. But, this function
// is intended for refleak tests. It spends extra work to report the true
// value, to help detect bugs in optimizations.
while (PyDict_Next(dict, &pos, &key, &value)) {
if (_Py_IsImmortal(key)) {
count++;
}
}
return _Py_hashtable_len(INTERNED_STRINGS) + count;
}
static Py_hash_t unicode_hash(PyObject *);
static int unicode_compare_eq(PyObject *, PyObject *);
@ -264,20 +284,6 @@ hashtable_unicode_compare(const void *key1, const void *key2)
static int
init_interned_dict(PyInterpreterState *interp)
{
if (_Py_IsMainInterpreter(interp)) {
assert(INTERNED_STRINGS == NULL);
_Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree};
INTERNED_STRINGS = _Py_hashtable_new_full(
hashtable_unicode_hash,
hashtable_unicode_compare,
NULL,
NULL,
&hashtable_alloc
);
if (INTERNED_STRINGS == NULL) {
return -1;
}
}
assert(get_interned_dict(interp) == NULL);
PyObject *interned = interned = PyDict_New();
if (interned == NULL) {
@ -296,7 +302,57 @@ clear_interned_dict(PyInterpreterState *interp)
Py_DECREF(interned);
_Py_INTERP_CACHED_OBJECT(interp, interned_strings) = NULL;
}
if (_Py_IsMainInterpreter(interp) && INTERNED_STRINGS != NULL) {
}
static PyStatus
init_global_interned_strings(PyInterpreterState *interp)
{
assert(INTERNED_STRINGS == NULL);
_Py_hashtable_allocator_t hashtable_alloc = {PyMem_RawMalloc, PyMem_RawFree};
INTERNED_STRINGS = _Py_hashtable_new_full(
hashtable_unicode_hash,
hashtable_unicode_compare,
// Objects stored here are immortal and statically allocated,
// so we don't need key_destroy_func & value_destroy_func:
NULL,
NULL,
&hashtable_alloc
);
if (INTERNED_STRINGS == NULL) {
PyErr_Clear();
return _PyStatus_ERR("failed to create global interned dict");
}
/* Intern statically allocated string identifiers and deepfreeze strings.
* This must be done before any module initialization so that statically
* allocated string identifiers are used instead of heap allocated strings.
* Deepfreeze uses the interned identifiers if present to save space
* else generates them and they are interned to speed up dict lookups.
*/
_PyUnicode_InitStaticStrings(interp);
#ifdef Py_GIL_DISABLED
// In the free-threaded build, intern the 1-byte strings as well
for (int i = 0; i < 256; i++) {
PyObject *s = LATIN1(i);
_PyUnicode_InternStatic(interp, &s);
assert(s == LATIN1(i));
}
#endif
#ifdef Py_DEBUG
assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1));
for (int i = 0; i < 256; i++) {
assert(_PyUnicode_CheckConsistency(LATIN1(i), 1));
}
#endif
return _PyStatus_OK();
}
static void clear_global_interned_strings(void)
{
if (INTERNED_STRINGS != NULL) {
_Py_hashtable_destroy(INTERNED_STRINGS);
INTERNED_STRINGS = NULL;
}
@ -629,6 +685,39 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
}
CHECK(PyUnicode_READ(kind, data, ascii->length) == 0);
}
/* Check interning state */
#ifdef Py_DEBUG
switch (PyUnicode_CHECK_INTERNED(op)) {
case SSTATE_NOT_INTERNED:
if (ascii->state.statically_allocated) {
CHECK(_Py_IsImmortal(op));
// This state is for two exceptions:
// - strings are currently checked before they're interned
// - the 256 one-latin1-character strings
// are static but use SSTATE_NOT_INTERNED
}
else {
CHECK(!_Py_IsImmortal(op));
}
break;
case SSTATE_INTERNED_MORTAL:
CHECK(!ascii->state.statically_allocated);
CHECK(!_Py_IsImmortal(op));
break;
case SSTATE_INTERNED_IMMORTAL:
CHECK(!ascii->state.statically_allocated);
CHECK(_Py_IsImmortal(op));
break;
case SSTATE_INTERNED_IMMORTAL_STATIC:
CHECK(ascii->state.statically_allocated);
CHECK(_Py_IsImmortal(op));
break;
default:
Py_UNREACHABLE();
}
#endif
return 1;
#undef CHECK
@ -1588,16 +1677,74 @@ unicode_dealloc(PyObject *unicode)
_Py_FatalRefcountError("deallocating an Unicode singleton");
}
#endif
/* This should never get called, but we also don't want to SEGV if
* we accidentally decref an immortal string out of existence. Since
* the string is an immortal object, just re-set the reference count.
*/
if (PyUnicode_CHECK_INTERNED(unicode)
|| _PyUnicode_STATE(unicode).statically_allocated)
{
if (_PyUnicode_STATE(unicode).statically_allocated) {
/* This should never get called, but we also don't want to SEGV if
* we accidentally decref an immortal string out of existence. Since
* the string is an immortal object, just re-set the reference count.
*/
#ifdef Py_DEBUG
Py_UNREACHABLE();
#endif
_Py_SetImmortal(unicode);
return;
}
switch (_PyUnicode_STATE(unicode).interned) {
case SSTATE_NOT_INTERNED:
break;
case SSTATE_INTERNED_MORTAL:
/* Remove the object from the intern dict.
* Before doing so, we set the refcount to 2: the key and value
* in the interned_dict.
*/
assert(Py_REFCNT(unicode) == 0);
Py_SET_REFCNT(unicode, 2);
#ifdef Py_REF_DEBUG
/* let's be pedantic with the ref total */
_Py_IncRefTotal(_PyThreadState_GET());
_Py_IncRefTotal(_PyThreadState_GET());
#endif
PyInterpreterState *interp = _PyInterpreterState_GET();
PyObject *interned = get_interned_dict(interp);
assert(interned != NULL);
PyObject *popped;
int r = PyDict_Pop(interned, unicode, &popped);
if (r == -1) {
PyErr_WriteUnraisable(unicode);
// We don't know what happened to the string. It's probably
// best to leak it:
// - if it was popped, there are no more references to it
// so it can't cause trouble (except wasted memory)
// - if it wasn't popped, it'll remain interned
_Py_SetImmortal(unicode);
_PyUnicode_STATE(unicode).interned = SSTATE_INTERNED_IMMORTAL;
return;
}
if (r == 0) {
// The interned string was not found in the interned_dict.
#ifdef Py_DEBUG
Py_UNREACHABLE();
#endif
_Py_SetImmortal(unicode);
return;
}
// Successfully popped.
assert(popped == unicode);
// Only our `popped` reference should be left; remove it too.
assert(Py_REFCNT(unicode) == 1);
Py_SET_REFCNT(unicode, 0);
#ifdef Py_REF_DEBUG
/* let's be pedantic with the ref total */
_Py_DecRefTotal(_PyThreadState_GET());
#endif
break;
default:
// As with `statically_allocated` above.
#ifdef Py_REF_DEBUG
Py_UNREACHABLE();
#endif
_Py_SetImmortal(unicode);
return;
}
if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) {
PyMem_Free(_PyUnicode_UTF8(unicode));
}
@ -1942,7 +2089,7 @@ _PyUnicode_FromId(_Py_Identifier *id)
if (!obj) {
goto end;
}
PyUnicode_InternInPlace(&obj);
_PyUnicode_InternImmortal(interp, &obj);
if (index >= ids->size) {
// Overallocate to reduce the number of realloc
@ -10925,8 +11072,10 @@ _PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right)
if (left == right_uni)
return 1;
if (PyUnicode_CHECK_INTERNED(left))
assert(PyUnicode_CHECK_INTERNED(right_uni));
if (PyUnicode_CHECK_INTERNED(left)) {
return 0;
}
Py_hash_t right_hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyUnicode_HASH(right_uni));
assert(right_hash != -1);
@ -15017,30 +15166,19 @@ _PyUnicode_InitState(PyInterpreterState *interp)
PyStatus
_PyUnicode_InitGlobalObjects(PyInterpreterState *interp)
{
// Initialize the global interned dict
if (_Py_IsMainInterpreter(interp)) {
PyStatus status = init_global_interned_strings(interp);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
}
assert(INTERNED_STRINGS);
if (init_interned_dict(interp)) {
PyErr_Clear();
return _PyStatus_ERR("failed to create interned dict");
}
if (_Py_IsMainInterpreter(interp)) {
/* Intern statically allocated string identifiers and deepfreeze strings.
* This must be done before any module initialization so that statically
* allocated string identifiers are used instead of heap allocated strings.
* Deepfreeze uses the interned identifiers if present to save space
* else generates them and they are interned to speed up dict lookups.
*/
_PyUnicode_InitStaticStrings(interp);
#ifdef Py_DEBUG
assert(_PyUnicode_CheckConsistency(&_Py_STR(empty), 1));
for (int i = 0; i < 256; i++) {
assert(_PyUnicode_CheckConsistency(LATIN1(i), 1));
}
#endif
}
return _PyStatus_OK();
}
@ -15063,106 +15201,267 @@ error:
return _PyStatus_ERR("Can't initialize unicode types");
}
static /* non-null */ PyObject*
intern_static(PyInterpreterState *interp, PyObject *s /* stolen */)
{
// Note that this steals a reference to `s`, but in many cases that
// stolen ref is returned, requiring no decref/incref.
assert(s != NULL);
assert(_PyUnicode_CHECK(s));
assert(_PyUnicode_STATE(s).statically_allocated);
assert(_Py_IsImmortal(s));
switch (PyUnicode_CHECK_INTERNED(s)) {
case SSTATE_NOT_INTERNED:
break;
case SSTATE_INTERNED_IMMORTAL_STATIC:
return s;
default:
Py_FatalError("_PyUnicode_InternStatic called on wrong string");
}
#ifdef Py_DEBUG
/* We must not add process-global interned string if there's already a
* per-interpreter interned_dict, which might contain duplicates.
* Except "short string" singletons: those are special-cased. */
PyObject *interned = get_interned_dict(interp);
assert(interned == NULL || unicode_is_singleton(s));
#ifdef Py_GIL_DISABLED
// In the free-threaded build, don't allow even the short strings.
assert(interned == NULL);
#endif
#endif
/* Look in the global cache first. */
PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s);
/* We should only init each string once */
assert(r == NULL);
/* but just in case (for the non-debug build), handle this */
if (r != NULL && r != s) {
assert(_PyUnicode_STATE(r).interned == SSTATE_INTERNED_IMMORTAL_STATIC);
assert(_PyUnicode_CHECK(r));
Py_DECREF(s);
return Py_NewRef(r);
}
if (_Py_hashtable_set(INTERNED_STRINGS, s, s) < -1) {
Py_FatalError("failed to intern static string");
}
_PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL_STATIC;
return s;
}
void
_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p)
_PyUnicode_InternStatic(PyInterpreterState *interp, PyObject **p)
{
PyObject *s = *p;
// This should only be called as part of runtime initialization
assert(!Py_IsInitialized());
*p = intern_static(interp, *p);
assert(*p);
}
static void
immortalize_interned(PyObject *s)
{
assert(PyUnicode_CHECK_INTERNED(s) == SSTATE_INTERNED_MORTAL);
assert(!_Py_IsImmortal(s));
#ifdef Py_REF_DEBUG
/* The reference count value should be excluded from the RefTotal.
The decrements to these objects will not be registered so they
need to be accounted for in here. */
for (Py_ssize_t i = 0; i < Py_REFCNT(s); i++) {
_Py_DecRefTotal(_PyThreadState_GET());
}
#endif
_PyUnicode_STATE(s).interned = SSTATE_INTERNED_IMMORTAL;
_Py_SetImmortal(s);
}
static /* non-null */ PyObject*
intern_common(PyInterpreterState *interp, PyObject *s /* stolen */,
bool immortalize)
{
// Note that this steals a reference to `s`, but in many cases that
// stolen ref is returned, requiring no decref/incref.
#ifdef Py_DEBUG
assert(s != NULL);
assert(_PyUnicode_CHECK(s));
#else
if (s == NULL || !PyUnicode_Check(s)) {
return;
return s;
}
#endif
/* If it's a subclass, we don't really know what putting
it in the interned dict might do. */
if (!PyUnicode_CheckExact(s)) {
return;
}
if (PyUnicode_CHECK_INTERNED(s)) {
return;
}
/* Look in the global cache first. */
PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s);
if (r != NULL && r != s) {
Py_SETREF(*p, Py_NewRef(r));
return;
return s;
}
/* Handle statically allocated strings. */
if (_PyUnicode_STATE(s).statically_allocated) {
assert(_Py_IsImmortal(s));
if (_Py_hashtable_set(INTERNED_STRINGS, s, s) == 0) {
_PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC;
}
return;
return intern_static(interp, s);
}
/* Look in the per-interpreter cache. */
/* Is it already interned? */
switch (PyUnicode_CHECK_INTERNED(s)) {
case SSTATE_NOT_INTERNED:
// no, go on
break;
case SSTATE_INTERNED_MORTAL:
// yes but we might need to make it immortal
if (immortalize) {
immortalize_interned(s);
}
return s;
default:
// all done
return s;
}
#if Py_GIL_DISABLED
/* In the free-threaded build, all interned strings are immortal */
immortalize = 1;
#endif
/* If it's already immortal, intern it as such */
if (_Py_IsImmortal(s)) {
immortalize = 1;
}
/* if it's a short string, get the singleton -- and intern it */
if (PyUnicode_GET_LENGTH(s) == 1 &&
PyUnicode_KIND(s) == PyUnicode_1BYTE_KIND) {
PyObject *r = LATIN1(*(unsigned char*)PyUnicode_DATA(s));
if (!PyUnicode_CHECK_INTERNED(r)) {
r = intern_static(interp, r);
}
Py_DECREF(s);
return r;
}
#ifdef Py_DEBUG
assert(!unicode_is_singleton(s));
#endif
/* Look in the global cache now. */
{
PyObject *r = (PyObject *)_Py_hashtable_get(INTERNED_STRINGS, s);
if (r != NULL) {
assert(_Py_IsImmortal(r));
assert(r != s); // r must be statically_allocated; s is not
Py_DECREF(s);
return Py_NewRef(r);
}
}
/* Do a setdefault on the per-interpreter cache. */
PyObject *interned = get_interned_dict(interp);
assert(interned != NULL);
PyObject *t;
int res = PyDict_SetDefaultRef(interned, s, s, &t);
if (res < 0) {
PyErr_Clear();
return;
}
else if (res == 1) {
// value was already present (not inserted)
Py_SETREF(*p, t);
return;
}
Py_DECREF(t);
if (_Py_IsImmortal(s)) {
// XXX Restrict this to the main interpreter?
_PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC;
return;
{
int res = PyDict_SetDefaultRef(interned, s, s, &t);
if (res < 0) {
PyErr_Clear();
return s;
}
else if (res == 1) {
// value was already present (not inserted)
Py_DECREF(s);
if (immortalize &&
PyUnicode_CHECK_INTERNED(t) == SSTATE_INTERNED_MORTAL) {
immortalize_interned(t);
}
return t;
}
else {
// value was newly inserted
assert (s == t);
Py_DECREF(t);
}
}
/* NOT_INTERNED -> INTERNED_MORTAL */
assert(_PyUnicode_STATE(s).interned == SSTATE_NOT_INTERNED);
if (!_Py_IsImmortal(s)) {
/* The two references in interned dict (key and value) are not counted.
unicode_dealloc() and _PyUnicode_ClearInterned() take care of this. */
Py_SET_REFCNT(s, Py_REFCNT(s) - 2);
#ifdef Py_REF_DEBUG
/* The reference count value excluding the 2 references from the
interned dictionary should be excluded from the RefTotal. The
decrements to these objects will not be registered so they
need to be accounted for in here. */
for (Py_ssize_t i = 0; i < Py_REFCNT(s) - 2; i++) {
/* let's be pedantic with the ref total */
_Py_DecRefTotal(_PyThreadState_GET());
_Py_DecRefTotal(_PyThreadState_GET());
#endif
}
_PyUnicode_STATE(s).interned = SSTATE_INTERNED_MORTAL;
/* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */
#ifdef Py_DEBUG
if (_Py_IsImmortal(s)) {
assert(immortalize);
}
#endif
_Py_SetImmortal(s);
_PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL;
if (immortalize) {
immortalize_interned(s);
}
return s;
}
void
_PyUnicode_InternImmortal(PyInterpreterState *interp, PyObject **p)
{
*p = intern_common(interp, *p, 1);
assert(*p);
}
void
_PyUnicode_InternMortal(PyInterpreterState *interp, PyObject **p)
{
*p = intern_common(interp, *p, 0);
assert(*p);
}
void
_PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p)
{
_PyUnicode_InternImmortal(interp, p);
return;
}
void
PyUnicode_InternInPlace(PyObject **p)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternInPlace(interp, p);
_PyUnicode_InternImmortal(interp, p);
}
// Function kept for the stable ABI.
// Public-looking name kept for the stable ABI; user should not call this:
PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **);
void
PyUnicode_InternImmortal(PyObject **p)
{
PyUnicode_InternInPlace(p);
// Leak a reference on purpose
Py_INCREF(*p);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, p);
}
PyObject *
PyUnicode_InternFromString(const char *cp)
{
PyObject *s = PyUnicode_FromString(cp);
if (s == NULL)
if (s == NULL) {
return NULL;
PyUnicode_InternInPlace(&s);
}
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternMortal(interp, &s);
return s;
}
@ -15176,20 +15475,6 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp)
}
assert(PyDict_CheckExact(interned));
/* TODO:
* Currently, the runtime is not able to guarantee that it can exit without
* allocations that carry over to a future initialization of Python within
* the same process. i.e:
* ./python -X showrefcount -c 'import itertools'
* [237 refs, 237 blocks]
*
* Therefore, this should remain disabled for until there is a strict guarantee
* that no memory will be left after `Py_Finalize`.
*/
#ifdef Py_DEBUG
/* For all non-singleton interned strings, restore the two valid references
to that instance from within the intern string dictionary and let the
normal reference counting process clean up these instances. */
#ifdef INTERNED_STATS
fprintf(stderr, "releasing %zd interned strings\n",
PyDict_GET_SIZE(interned));
@ -15203,13 +15488,32 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp)
int shared = 0;
switch (PyUnicode_CHECK_INTERNED(s)) {
case SSTATE_INTERNED_IMMORTAL:
/* Make immortal interned strings mortal again.
*
* Currently, the runtime is not able to guarantee that it can exit
* without allocations that carry over to a future initialization
* of Python within the same process. i.e:
* ./python -X showrefcount -c 'import itertools'
* [237 refs, 237 blocks]
*
* This should remain disabled (`Py_DEBUG` only) until there is a
* strict guarantee that no memory will be left after
* `Py_Finalize`.
*/
#ifdef Py_DEBUG
// Skip the Immortal Instance check and restore
// the two references (key and value) ignored
// by PyUnicode_InternInPlace().
_Py_SetMortal(s, 2);
#ifdef Py_REF_DEBUG
/* let's be pedantic with the ref total */
_Py_IncRefTotal(_PyThreadState_GET());
_Py_IncRefTotal(_PyThreadState_GET());
#endif
#ifdef INTERNED_STATS
total_length += PyUnicode_GET_LENGTH(s);
#endif
#endif // Py_DEBUG
break;
case SSTATE_INTERNED_IMMORTAL_STATIC:
/* It is shared between interpreters, so we should unmark it
@ -15222,7 +15526,15 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp)
}
break;
case SSTATE_INTERNED_MORTAL:
/* fall through */
// Restore 2 references held by the interned dict; these will
// be decref'd by clear_interned_dict's PyDict_Clear.
Py_SET_REFCNT(s, Py_REFCNT(s) + 2);
#ifdef Py_REF_DEBUG
/* let's be pedantic with the ref total */
_Py_IncRefTotal(_PyThreadState_GET());
_Py_IncRefTotal(_PyThreadState_GET());
#endif
break;
case SSTATE_NOT_INTERNED:
/* fall through */
default:
@ -15243,8 +15555,10 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp)
for (Py_ssize_t i=0; i < ids->size; i++) {
Py_XINCREF(ids->array[i]);
}
#endif /* Py_DEBUG */
clear_interned_dict(interp);
if (_Py_IsMainInterpreter(interp)) {
clear_global_interned_strings();
}
}

4
Parser/action_helpers.c
View File

@ -3,6 +3,7 @@
#include "pegen.h"
#include "string_parser.h"
#include "pycore_runtime.h" // _PyRuntime
#include "pycore_pystate.h" // _PyInterpreterState_GET()
void *
_PyPegen_dummy_name(Parser *p, ...)
@ -123,7 +124,8 @@ _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
if (!uni) {
return NULL;
}
PyUnicode_InternInPlace(&uni);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &uni);
if (_PyArena_AddPyObject(p->arena, uni) < 0) {
Py_DECREF(uni);
return NULL;

3
Parser/pegen.c
View File

@ -527,7 +527,8 @@ _PyPegen_new_identifier(Parser *p, const char *n)
}
id = id2;
}
PyUnicode_InternInPlace(&id);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &id);
if (_PyArena_AddPyObject(p->arena, id) < 0)
{
Py_DECREF(id);

3
Python/ast_opt.c
View File

@ -273,10 +273,9 @@ parse_literal(PyObject *fmt, Py_ssize_t *ppos, PyArena *arena)
PyObject *str = PyUnicode_Substring(fmt, start, pos);
/* str = str.replace('%%', '%') */
if (str && has_percents) {
_Py_DECLARE_STR(percent, "%");
_Py_DECLARE_STR(dbl_percent, "%%");
Py_SETREF(str, PyUnicode_Replace(str, &_Py_STR(dbl_percent),
&_Py_STR(percent), -1));
_Py_LATIN1_CHR('%'), -1));
}
if (!str) {
return NULL;

10
Python/ast_unparse.c
View File

@ -10,9 +10,7 @@
* See ast.unparse for a full unparser (written in Python)
*/
_Py_DECLARE_STR(open_br, "{");
_Py_DECLARE_STR(dbl_open_br, "{{");
_Py_DECLARE_STR(close_br, "}");
_Py_DECLARE_STR(dbl_close_br, "}}");
/* We would statically initialize this if doing so were simple enough. */
@ -580,11 +578,13 @@ escape_braces(PyObject *orig)
{
PyObject *temp;
PyObject *result;
temp = PyUnicode_Replace(orig, &_Py_STR(open_br), &_Py_STR(dbl_open_br), -1);
temp = PyUnicode_Replace(orig, _Py_LATIN1_CHR('{'),
&_Py_STR(dbl_open_br), -1);
if (!temp) {
return NULL;
}
result = PyUnicode_Replace(temp, &_Py_STR(close_br), &_Py_STR(dbl_close_br), -1);
result = PyUnicode_Replace(temp, _Py_LATIN1_CHR('}'),
&_Py_STR(dbl_close_br), -1);
Py_DECREF(temp);
return result;
}
@ -678,7 +678,7 @@ append_formattedvalue(_PyUnicodeWriter *writer, expr_ty e)
if (!temp_fv_str) {
return -1;
}
if (PyUnicode_Find(temp_fv_str, &_Py_STR(open_br), 0, 1, 1) == 0) {
if (PyUnicode_Find(temp_fv_str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) {
/* Expression starts with a brace, split it with a space from the outer
one. */
outer_brace = "{ ";

52
Python/clinic/sysmodule.c.h generated
View File

@ -968,24 +968,64 @@ exit:
}
PyDoc_STRVAR(sys_getunicodeinternedsize__doc__,
"getunicodeinternedsize($module, /)\n"
"getunicodeinternedsize($module, /, *, _only_immortal=False)\n"
"--\n"
"\n"
"Return the number of elements of the unicode interned dictionary");
#define SYS_GETUNICODEINTERNEDSIZE_METHODDEF \
{"getunicodeinternedsize", (PyCFunction)sys_getunicodeinternedsize, METH_NOARGS, sys_getunicodeinternedsize__doc__},
{"getunicodeinternedsize", _PyCFunction_CAST(sys_getunicodeinternedsize), METH_FASTCALL|METH_KEYWORDS, sys_getunicodeinternedsize__doc__},
static Py_ssize_t
sys_getunicodeinternedsize_impl(PyObject *module);
sys_getunicodeinternedsize_impl(PyObject *module, int _only_immortal);
static PyObject *
sys_getunicodeinternedsize(PyObject *module, PyObject *Py_UNUSED(ignored))
sys_getunicodeinternedsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
#define NUM_KEYWORDS 1
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
.ob_item = { &_Py_ID(_only_immortal), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
#else // !Py_BUILD_CORE
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
static const char * const _keywords[] = {"_only_immortal", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "getunicodeinternedsize",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
PyObject *argsbuf[1];
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
int _only_immortal = 0;
Py_ssize_t _return_value;
_return_value = sys_getunicodeinternedsize_impl(module);
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
if (!args) {
goto exit;
}
if (!noptargs) {
goto skip_optional_kwonly;
}
_only_immortal = PyObject_IsTrue(args[0]);
if (_only_immortal < 0) {
goto exit;
}
skip_optional_kwonly:
_return_value = sys_getunicodeinternedsize_impl(module, _only_immortal);
if ((_return_value == -1) && PyErr_Occurred()) {
goto exit;
}
@ -1574,4 +1614,4 @@ exit:
#ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
#define SYS_GETANDROIDAPILEVEL_METHODDEF
#endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
/*[clinic end generated code: output=ef7c35945443d300 input=a9049054013a1b77]*/
/*[clinic end generated code: output=9cc9069aef1482bc input=a9049054013a1b77]*/

6
Python/codecs.c
View File

@ -147,7 +147,9 @@ PyObject *_PyCodec_Lookup(const char *encoding)
if (v == NULL) {
return NULL;
}
PyUnicode_InternInPlace(&v);
/* Intern the string. We'll make it immortal later if lookup succeeds. */
_PyUnicode_InternMortal(interp, &v);
/* First, try to lookup the name in the registry dictionary */
PyObject *result;
@ -200,6 +202,8 @@ PyObject *_PyCodec_Lookup(const char *encoding)
goto onError;
}
_PyUnicode_InternImmortal(interp, &v);
/* Cache and return the result */
if (PyDict_SetItem(interp->codecs.search_cache, v, result) < 0) {
Py_DECREF(result);

3
Python/compile.c
View File

@ -645,8 +645,7 @@ compiler_set_qualname(struct compiler *c)
}
if (base != NULL) {
_Py_DECLARE_STR(dot, ".");
name = PyUnicode_Concat(base, &_Py_STR(dot));
name = PyUnicode_Concat(base, _Py_LATIN1_CHR('.'));
Py_DECREF(base);
if (name == NULL) {
return ERROR;

3
Python/getargs.c
View File

@ -1934,7 +1934,8 @@ new_kwtuple(const char * const *keywords, int total, int pos)
Py_DECREF(kwtuple);
return NULL;
}
PyUnicode_InternInPlace(&str);
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &str);
PyTuple_SET_ITEM(kwtuple, i, str);
}
return kwtuple;

8
Python/import.c
View File

@ -1967,8 +1967,6 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0,
if (res.kind == _Py_ext_module_kind_SINGLEPHASE) {
/* Remember the filename as the __file__ attribute */
if (info->filename != NULL) {
// XXX There's a refleak somewhere with the filename.
// Until we can track it down, we intern it.
PyObject *filename = NULL;
if (switched) {
// The original filename may be allocated by subinterpreter's
@ -1980,7 +1978,11 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0,
} else {
filename = Py_NewRef(info->filename);
}
PyUnicode_InternInPlace(&filename);
// XXX There's a refleak somewhere with the filename.
// Until we can track it down, we immortalize it.
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &filename);
if (PyModule_AddObjectRef(mod, "__file__", filename) < 0) {
PyErr_Clear(); /* Not important enough to report */
}

17
Python/marshal.c
View File

@ -14,6 +14,7 @@
#include "pycore_long.h" // _PyLong_DigitCount
#include "pycore_setobject.h" // _PySet_NextEntry()
#include "marshal.h" // Py_MARSHAL_VERSION
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#ifdef __APPLE__
# include "TargetConditionals.h"
@ -1184,8 +1185,12 @@ r_object(RFILE *p)
v = PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, ptr, n);
if (v == NULL)
break;
if (is_interned)
PyUnicode_InternInPlace(&v);
if (is_interned) {
// marshal is meant to serialize .pyc files with code
// objects, and code-related strings are currently immortal.
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &v);
}
retval = v;
R_REF(retval);
break;
@ -1217,8 +1222,12 @@ r_object(RFILE *p)
}
if (v == NULL)
break;
if (is_interned)
PyUnicode_InternInPlace(&v);
if (is_interned) {
// marshal is meant to serialize .pyc files with code
// objects, and code-related strings are currently immortal.
PyInterpreterState *interp = _PyInterpreterState_GET();
_PyUnicode_InternImmortal(interp, &v);
}
retval = v;
R_REF(retval);
break;

28
Python/sysmodule.c
View File

@ -747,7 +747,7 @@ sys_displayhook(PyObject *module, PyObject *o)
if (o == Py_None) {
Py_RETURN_NONE;
}
if (PyObject_SetAttr(builtins, &_Py_ID(_), Py_None) != 0)
if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), Py_None) != 0)
return NULL;
outf = _PySys_GetAttr(tstate, &_Py_ID(stdout));
if (outf == NULL || outf == Py_None) {
@ -769,10 +769,9 @@ sys_displayhook(PyObject *module, PyObject *o)
return NULL;
}
}
_Py_DECLARE_STR(newline, "\n");
if (PyFile_WriteObject(&_Py_STR(newline), outf, Py_PRINT_RAW) != 0)
if (PyFile_WriteObject(_Py_LATIN1_CHR('\n'), outf, Py_PRINT_RAW) != 0)
return NULL;
if (PyObject_SetAttr(builtins, &_Py_ID(_), o) != 0)
if (PyObject_SetAttr(builtins, _Py_LATIN1_CHR('_'), o) != 0)
return NULL;
Py_RETURN_NONE;
}
@ -930,7 +929,7 @@ sys_getfilesystemencoding_impl(PyObject *module)
if (u == NULL) {
return NULL;
}
_PyUnicode_InternInPlace(interp, &u);
_PyUnicode_InternImmortal(interp, &u);
return u;
}
@ -950,7 +949,7 @@ sys_getfilesystemencodeerrors_impl(PyObject *module)
if (u == NULL) {
return NULL;
}
_PyUnicode_InternInPlace(interp, &u);
_PyUnicode_InternImmortal(interp, &u);
return u;
}
@ -972,8 +971,9 @@ sys_intern_impl(PyObject *module, PyObject *s)
/*[clinic end generated code: output=be680c24f5c9e5d6 input=849483c006924e2f]*/
{
if (PyUnicode_CheckExact(s)) {
PyInterpreterState *interp = _PyInterpreterState_GET();
Py_INCREF(s);
PyUnicode_InternInPlace(&s);
_PyUnicode_InternMortal(interp, &s);
return s;
}
else {
@ -2007,14 +2007,22 @@ sys_getallocatedblocks_impl(PyObject *module)
/*[clinic input]
sys.getunicodeinternedsize -> Py_ssize_t
*
_only_immortal: bool = False
Return the number of elements of the unicode interned dictionary
[clinic start generated code]*/
static Py_ssize_t
sys_getunicodeinternedsize_impl(PyObject *module)
/*[clinic end generated code: output=ad0e4c9738ed4129 input=726298eaa063347a]*/
sys_getunicodeinternedsize_impl(PyObject *module, int _only_immortal)
/*[clinic end generated code: output=29a6377a94a14f70 input=0330b3408dd5bcc6]*/
{
return _PyUnicode_InternedSize();
if (_only_immortal) {
return _PyUnicode_InternedSize_Immortal();
}
else {
return _PyUnicode_InternedSize();
}
}
/*[clinic input]

25
Tools/build/generate_global_objects.py
View File

@ -370,9 +370,14 @@ def generate_static_strings_initializer(identifiers, strings):
# This use of _Py_ID() is ignored by iter_global_strings()
# since iter_files() ignores .h files.
printer.write(f'string = &_Py_ID({i});')
printer.write(f'_PyUnicode_InternStatic(interp, &string);')
printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));')
printer.write(f'_PyUnicode_InternInPlace(interp, &string);')
# XXX What about "strings"?
printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);')
for value, name in sorted(strings.items()):
printer.write(f'string = &_Py_STR({name});')
printer.write(f'_PyUnicode_InternStatic(interp, &string);')
printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));')
printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);')
printer.write(END)
printer.write(after)
@ -414,15 +419,31 @@ def generate_global_object_finalizers(generated_immortal_objects):
def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]':
identifiers = set(IDENTIFIERS)
strings = {}
# Note that we store strings as they appear in C source, so the checks here
# can be defeated, e.g.:
# - "a" and "\0x61" won't be reported as duplicate.
# - "\n" appears as 2 characters.
# Probably not worth adding a C string parser.
for name, string, *_ in iter_global_strings():
if string is None:
if name not in IGNORED:
identifiers.add(name)
else:
if len(string) == 1 and ord(string) < 256:
# Give a nice message for common mistakes.
# To cover tricky cases (like "\n") we also generate C asserts.
raise ValueError(
'do not use &_PyID or &_Py_STR for one-character latin-1 '
+ f'strings, use _Py_LATIN1_CHR instead: {string!r}')
if string not in strings:
strings[string] = name
elif name != strings[string]:
raise ValueError(f'string mismatch for {name!r} ({string!r} != {strings[name]!r}')
overlap = identifiers & set(strings.keys())
if overlap:
raise ValueError(
'do not use both _PyID and _Py_DECLARE_STR for the same string: '
+ repr(overlap))
return identifiers, strings

16
Tools/clinic/libclinic/clanguage.py
View File

@ -21,6 +21,16 @@ if TYPE_CHECKING:
from libclinic.app import Clinic
def c_id(name: str) -> str:
if len(name) == 1 and ord(name) < 256:
if name.isalnum():
return f"_Py_LATIN1_CHR('{name}')"
else:
return f'_Py_LATIN1_CHR({ord(name)})'
else:
return f'&_Py_ID({name})'
class CLanguage(Language):
body_prefix = "#"
@ -167,11 +177,11 @@ class CLanguage(Language):
if argname_fmt:
conditions.append(f"nargs < {i+1} && {argname_fmt % i}")
elif fastcall:
conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, &_Py_ID({p.name}))")
conditions.append(f"nargs < {i+1} && PySequence_Contains(kwnames, {c_id(p.name)})")
containscheck = "PySequence_Contains"
codegen.add_include('pycore_runtime.h', '_Py_ID()')
else:
conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, &_Py_ID({p.name}))")
conditions.append(f"nargs < {i+1} && PyDict_Contains(kwargs, {c_id(p.name)})")
containscheck = "PyDict_Contains"
codegen.add_include('pycore_runtime.h', '_Py_ID()')
else:
@ -459,7 +469,7 @@ class CLanguage(Language):
template_dict['keywords_c'] = ' '.join('"' + k + '",'
for k in data.keywords)
keywords = [k for k in data.keywords if k]
template_dict['keywords_py'] = ' '.join('&_Py_ID(' + k + '),'
template_dict['keywords_py'] = ' '.join(c_id(k) + ','
for k in keywords)
template_dict['format_units'] = ''.join(data.format_units)
template_dict['parse_arguments'] = ', '.join(data.parse_arguments)