Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-44796: Unify TypeVar and ParamSpec substitution (GH-31143) 

Add methods __typing_subst__() in TypeVar and ParamSpec.
Simplify code by using more object-oriented approach, especially
the C code for types.GenericAlias and the Python code for
collections.abc.Callable.
This commit is contained in:
Serhiy Storchaka 2022-03-11 10:47:26 +02:00 committed by GitHub
parent 2d5835a019
commit b6a5d8590c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 114 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Include/internal/pycore_global_strings.h
View File

@ -199,6 +199,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__subclasshook__)
STRUCT_FOR_ID(__truediv__)
STRUCT_FOR_ID(__trunc__)
STRUCT_FOR_ID(__typing_subst__)
STRUCT_FOR_ID(__warningregistry__)
STRUCT_FOR_ID(__weakref__)
STRUCT_FOR_ID(__xor__)

1
Include/internal/pycore_runtime_init.h
View File

@ -822,6 +822,7 @@ extern "C" {
INIT_ID(__subclasshook__), \
INIT_ID(__truediv__), \
INIT_ID(__trunc__), \
INIT_ID(__typing_subst__), \
INIT_ID(__warningregistry__), \
INIT_ID(__weakref__), \
INIT_ID(__xor__), \

63
Lib/_collections_abc.py
View File

@ -430,25 +430,13 @@ class _CallableGenericAlias(GenericAlias):
raise TypeError(
"Callable must be used as Callable[[arg, ...], result].")
t_args, t_result = args
if isinstance(t_args, list):
if isinstance(t_args, (tuple, list)):
args = (*t_args, t_result)
elif not _is_param_expr(t_args):
raise TypeError(f"Expected a list of types, an ellipsis, "
f"ParamSpec, or Concatenate. Got {t_args}")
return super().__new__(cls, origin, args)
@property
def __parameters__(self):
params = []
for arg in self.__args__:
# Looks like a genericalias
if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple):
params.extend(arg.__parameters__)
else:
if _is_typevarlike(arg):
params.append(arg)
return tuple(dict.fromkeys(params))
def __repr__(self):
if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
return super().__repr__()
@ -468,57 +456,24 @@ class _CallableGenericAlias(GenericAlias):
# code is copied from typing's _GenericAlias and the builtin
# types.GenericAlias.
# A special case in PEP 612 where if X = Callable[P, int],
# then X[int, str] == X[[int, str]].
param_len = len(self.__parameters__)
if param_len == 0:
raise TypeError(f'{self} is not a generic class')
if not isinstance(item, tuple):
item = (item,)
if (param_len == 1 and _is_param_expr(self.__parameters__[0])
# A special case in PEP 612 where if X = Callable[P, int],
# then X[int, str] == X[[int, str]].
if (len(self.__parameters__) == 1
and _is_param_expr(self.__parameters__[0])
and item and not _is_param_expr(item[0])):
item = (list(item),)
item_len = len(item)
if item_len != param_len:
raise TypeError(f'Too {"many" if item_len > param_len else "few"}'
f' arguments for {self};'
f' actual {item_len}, expected {param_len}')
subst = dict(zip(self.__parameters__, item))
new_args = []
for arg in self.__args__:
if _is_typevarlike(arg):
if _is_param_expr(arg):
arg = subst[arg]
if not _is_param_expr(arg):
raise TypeError(f"Expected a list of types, an ellipsis, "
f"ParamSpec, or Concatenate. Got {arg}")
else:
arg = subst[arg]
# Looks like a GenericAlias
elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple):
subparams = arg.__parameters__
if subparams:
subargs = tuple(subst[x] for x in subparams)
arg = arg[subargs]
if isinstance(arg, tuple):
new_args.extend(arg)
else:
new_args.append(arg)
item = (item,)
new_args = super().__getitem__(item).__args__
# args[0] occurs due to things like Z[[int, str, bool]] from PEP 612
if not isinstance(new_args[0], list):
if not isinstance(new_args[0], (tuple, list)):
t_result = new_args[-1]
t_args = new_args[:-1]
new_args = (t_args, t_result)
return _CallableGenericAlias(Callable, tuple(new_args))
def _is_typevarlike(arg):
obj = type(arg)
# looks like a TypeVar/ParamSpec
return (obj.__module__ == 'typing'
and obj.__name__ in {'ParamSpec', 'TypeVar'})
def _is_param_expr(obj):
"""Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or
``_ConcatenateGenericAlias`` from typing.py

60
Lib/test/test_typing.py
View File

@ -360,10 +360,31 @@ class TypeVarTests(BaseTestCase):
with self.assertRaises(ValueError):
TypeVar('T', covariant=True, contravariant=True)
def test_var_substitution(self):
T = TypeVar('T')
subst = T.__typing_subst__
self.assertIs(subst(int), int)
self.assertEqual(subst(list[int]), list[int])
self.assertEqual(subst(List[int]), List[int])
self.assertEqual(subst(List), List)
self.assertIs(subst(Any), Any)
self.assertIs(subst(None), type(None))
self.assertIs(subst(T), T)
self.assertEqual(subst(int|str), int|str)
self.assertEqual(subst(Union[int, str]), Union[int, str])
def test_bad_var_substitution(self):
T = TypeVar('T')
for arg in (), (int, str):
P = ParamSpec("P")
bad_args = (
42, ..., [int], (), (int, str), Union,
Generic, Generic[T], Protocol, Protocol[T],
Final, Final[int], ClassVar, ClassVar[int],
)
for arg in bad_args:
with self.subTest(arg=arg):
with self.assertRaises(TypeError):
T.__typing_subst__(arg)
with self.assertRaises(TypeError):
List[T][arg]
with self.assertRaises(TypeError):
@ -1110,8 +1131,7 @@ class BaseCallableTests:
C2 = Callable[[KT, T], VT]
C3 = Callable[..., T]
self.assertEqual(C1[str], Callable[[int, str], str])
if Callable is typing.Callable:
self.assertEqual(C1[None], Callable[[int, type(None)], type(None)])
self.assertEqual(C1[None], Callable[[int, type(None)], type(None)])
self.assertEqual(C2[int, float, str], Callable[[int, float], str])
self.assertEqual(C3[int], Callable[..., int])
self.assertEqual(C3[NoReturn], Callable[..., NoReturn])
@ -2696,7 +2716,10 @@ class GenericTests(BaseTestCase):
for obj in objs:
self.assertNotEqual(repr(obj), '')
self.assertEqual(obj, obj)
if getattr(obj, '__parameters__', None) and len(obj.__parameters__) == 1:
if (getattr(obj, '__parameters__', None)
and not isinstance(obj, typing.TypeVar)
and isinstance(obj.__parameters__, tuple)
and len(obj.__parameters__) == 1):
self.assertEqual(obj[Any].__args__, (Any,))
if isinstance(obj, type):
for base in obj.__mro__:
@ -5748,33 +5771,30 @@ class ParamSpecTests(BaseTestCase):
self.assertEqual(G1.__args__, ((int, str), (bytes,)))
self.assertEqual(G2.__args__, ((int,), (str, bytes)))
def test_var_substitution(self):
T = TypeVar("T")
P = ParamSpec("P")
subst = P.__typing_subst__
self.assertEqual(subst((int, str)), (int, str))
self.assertEqual(subst([int, str]), (int, str))
self.assertEqual(subst([None]), (type(None),))
self.assertIs(subst(...), ...)
self.assertIs(subst(P), P)
self.assertEqual(subst(Concatenate[int, P]), Concatenate[int, P])
def test_bad_var_substitution(self):
T = TypeVar('T')
P = ParamSpec('P')
bad_args = (42, int, None, T, int|str, Union[int, str])
for arg in bad_args:
with self.subTest(arg=arg):
with self.assertRaises(TypeError):
P.__typing_subst__(arg)
with self.assertRaises(TypeError):
typing.Callable[P, T][arg, str]
with self.assertRaises(TypeError):
collections.abc.Callable[P, T][arg, str]
def test_no_paramspec_in__parameters__(self):
# ParamSpec should not be found in __parameters__
# of generics. Usages outside Callable, Concatenate
# and Generic are invalid.
T = TypeVar("T")
P = ParamSpec("P")
self.assertNotIn(P, List[P].__parameters__)
self.assertIn(T, Tuple[T, P].__parameters__)
# Test for consistency with builtin generics.
self.assertNotIn(P, list[P].__parameters__)
self.assertIn(T, tuple[T, P].__parameters__)
self.assertNotIn(P, (list[P] | int).__parameters__)
self.assertIn(T, (tuple[T, P] | int).__parameters__)
def test_paramspec_in_nested_generics(self):
# Although ParamSpec should not be found in __parameters__ of most
# generics, they probably should be found when nested in

95
Lib/typing.py
View File

@ -179,7 +179,9 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=
if (isinstance(arg, _GenericAlias) and
arg.__origin__ in invalid_generic_forms):
raise TypeError(f"{arg} is not valid as type argument")
if arg in (Any, NoReturn, Never, Self, ClassVar, Final, TypeAlias):
if arg in (Any, NoReturn, Never, Self, TypeAlias):
return arg
if allow_special_forms and arg in (ClassVar, Final):
return arg
if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol):
raise TypeError(f"Plain {arg} is not valid as type argument")
@ -217,21 +219,22 @@ def _type_repr(obj):
return repr(obj)
def _collect_type_vars(types_, typevar_types=None):
"""Collect all type variable contained
in types in order of first appearance (lexicographic order). For example::
def _collect_parameters(args):
"""Collect all type variables and parameter specifications in args
in order of first appearance (lexicographic order). For example::
_collect_type_vars((T, List[S, T])) == (T, S)
_collect_parameters((T, Callable[P, T])) == (T, P)
"""
if typevar_types is None:
typevar_types = TypeVar
tvars = []
for t in types_:
if isinstance(t, typevar_types) and t not in tvars:
tvars.append(t)
if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)):
tvars.extend([t for t in t.__parameters__ if t not in tvars])
return tuple(tvars)
parameters = []
for t in args:
if hasattr(t, '__typing_subst__'):
if t not in parameters:
parameters.append(t)
else:
for x in getattr(t, '__parameters__', ()):
if x not in parameters:
parameters.append(x)
return tuple(parameters)
def _check_generic(cls, parameters, elen):
@ -671,7 +674,6 @@ def Concatenate(self, parameters):
msg = "Concatenate[arg, ...]: each arg must be a type."
parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1])
return _ConcatenateGenericAlias(self, parameters,
_typevar_types=(TypeVar, ParamSpec),
_paramspec_tvars=True)
@ -909,6 +911,11 @@ class TypeVar(_Final, _Immutable, _BoundVarianceMixin, _root=True):
if def_mod != 'typing':
self.__module__ = def_mod
def __typing_subst__(self, arg):
msg = "Parameters to generic types must be types."
arg = _type_check(arg, msg, is_argument=True)
return arg
class TypeVarTuple(_Final, _Immutable, _root=True):
"""Type variable tuple.
@ -942,6 +949,9 @@ class TypeVarTuple(_Final, _Immutable, _root=True):
def __repr__(self):
return self._name
def __typing_subst__(self, arg):
raise AssertionError
class ParamSpecArgs(_Final, _Immutable, _root=True):
"""The args for a ParamSpec object.
@ -1052,6 +1062,14 @@ class ParamSpec(_Final, _Immutable, _BoundVarianceMixin, _root=True):
if def_mod != 'typing':
self.__module__ = def_mod
def __typing_subst__(self, arg):
if isinstance(arg, (list, tuple)):
arg = tuple(_type_check(a, "Expected a type.") for a in arg)
elif not _is_param_expr(arg):
raise TypeError(f"Expected a list of types, an ellipsis, "
f"ParamSpec, or Concatenate. Got {arg}")
return arg
def _is_dunder(attr):
return attr.startswith('__') and attr.endswith('__')
@ -1106,7 +1124,7 @@ class _BaseGenericAlias(_Final, _root=True):
def __setattr__(self, attr, val):
if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams',
'_typevar_types', '_paramspec_tvars'}:
'_paramspec_tvars'}:
super().__setattr__(attr, val)
else:
setattr(self.__origin__, attr, val)
@ -1199,7 +1217,6 @@ class _GenericAlias(_BaseGenericAlias, _root=True):
# TypeVar[bool]
def __init__(self, origin, args, *, inst=True, name=None,
_typevar_types=(TypeVar, TypeVarTuple),
_paramspec_tvars=False):
super().__init__(origin, inst=inst, name=name)
if not isinstance(args, tuple):
@ -1207,8 +1224,7 @@ class _GenericAlias(_BaseGenericAlias, _root=True):
self.__args__ = tuple(... if a is _TypingEllipsis else
() if a is _TypingEmpty else
a for a in args)
self.__parameters__ = _collect_type_vars(args, typevar_types=_typevar_types)
self._typevar_types = _typevar_types
self.__parameters__ = _collect_parameters(args)
self._paramspec_tvars = _paramspec_tvars
if not name:
self.__module__ = origin.__module__
@ -1291,26 +1307,20 @@ class _GenericAlias(_BaseGenericAlias, _root=True):
new_args = []
for old_arg in self.__args__:
if isinstance(old_arg, ParamSpec):
new_arg = new_arg_by_param[old_arg]
if not _is_param_expr(new_arg):
raise TypeError(f"Expected a list of types, an ellipsis, "
f"ParamSpec, or Concatenate. Got {new_arg}")
elif isinstance(old_arg, self._typevar_types):
new_arg = new_arg_by_param[old_arg]
elif (TypeVarTuple in self._typevar_types
and _is_unpacked_typevartuple(old_arg)):
if _is_unpacked_typevartuple(old_arg):
original_typevartuple = old_arg.__parameters__[0]
new_arg = new_arg_by_param[original_typevartuple]
elif isinstance(old_arg, (_GenericAlias, GenericAlias, types.UnionType)):
subparams = old_arg.__parameters__
if not subparams:
new_arg = old_arg
else:
subargs = tuple(new_arg_by_param[x] for x in subparams)
new_arg = old_arg[subargs]
else:
new_arg = old_arg
substfunc = getattr(old_arg, '__typing_subst__', None)
if substfunc:
new_arg = substfunc(new_arg_by_param[old_arg])
else:
subparams = getattr(old_arg, '__parameters__', ())
if not subparams:
new_arg = old_arg
else:
subargs = tuple(new_arg_by_param[x] for x in subparams)
new_arg = old_arg[subargs]
if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple):
# Consider the following `Callable`.
@ -1342,7 +1352,6 @@ class _GenericAlias(_BaseGenericAlias, _root=True):
def copy_with(self, args):
return self.__class__(self.__origin__, args, name=self._name, inst=self._inst,
_typevar_types=self._typevar_types,
_paramspec_tvars=self._paramspec_tvars)
def __repr__(self):
@ -1454,7 +1463,6 @@ class _CallableType(_SpecialGenericAlias, _root=True):
def copy_with(self, params):
return _CallableGenericAlias(self.__origin__, params,
name=self._name, inst=self._inst,
_typevar_types=(TypeVar, ParamSpec),
_paramspec_tvars=True)
def __getitem__(self, params):
@ -1675,11 +1683,8 @@ class Generic:
# don't check variadic generic arity at runtime (to reduce
# complexity of typing.py).
_check_generic(cls, params, len(cls.__parameters__))
return _GenericAlias(
cls, params,
_typevar_types=(TypeVar, TypeVarTuple, ParamSpec),
_paramspec_tvars=True,
)
return _GenericAlias(cls, params,
_paramspec_tvars=True)
def __init_subclass__(cls, *args, **kwargs):
super().__init_subclass__(*args, **kwargs)
@ -1691,9 +1696,7 @@ class Generic:
if error:
raise TypeError("Cannot inherit from plain Generic")
if '__orig_bases__' in cls.__dict__:
tvars = _collect_type_vars(
cls.__orig_bases__, (TypeVar, TypeVarTuple, ParamSpec)
)
tvars = _collect_parameters(cls.__orig_bases__)
# Look for Generic[T1, ..., Tn].
# If found, tvars must be a subset of it.
# If not found, tvars is it.

46
Objects/genericaliasobject.c
View File

@ -152,25 +152,6 @@ error:
return NULL;
}
// isinstance(obj, TypeVar) without importing typing.py.
// Returns -1 for errors.
static int
is_typevar(PyObject *obj)
{
PyTypeObject *type = Py_TYPE(obj);
if (strcmp(type->tp_name, "TypeVar") != 0) {
return 0;
}
PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__");
if (module == NULL) {
return -1;
}
int res = PyUnicode_Check(module)
&& _PyUnicode_EqualToASCIIString(module, "typing");
Py_DECREF(module);
return res;
}
// Index of item in self[:len], or -1 if not found (self is a tuple)
static Py_ssize_t
tuple_index(PyObject *self, Py_ssize_t len, PyObject *item)
@ -205,13 +186,14 @@ _Py_make_parameters(PyObject *args)
Py_ssize_t iparam = 0;
for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
PyObject *t = PyTuple_GET_ITEM(args, iarg);
int typevar = is_typevar(t);
if (typevar < 0) {
PyObject *subst;
if (_PyObject_LookupAttr(t, &_Py_ID(__typing_subst__), &subst) < 0) {
Py_DECREF(parameters);
return NULL;
}
if (typevar) {
if (subst) {
iparam += tuple_add(parameters, iparam, t);
Py_DECREF(subst);
}
else {
PyObject *subparams;
@ -295,7 +277,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
Py_ssize_t nparams = PyTuple_GET_SIZE(parameters);
if (nparams == 0) {
return PyErr_Format(PyExc_TypeError,
"There are no type variables left in %R",
"%R is not a generic class",
self);
}
int is_tuple = PyTuple_Check(item);
@ -320,23 +302,23 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
}
for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) {
PyObject *arg = PyTuple_GET_ITEM(args, iarg);
int typevar = is_typevar(arg);
if (typevar < 0) {
PyObject *subst;
if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) {
Py_DECREF(newargs);
return NULL;
}
if (typevar) {
if (subst) {
Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
assert(iparam >= 0);
arg = argitems[iparam];
Py_INCREF(arg);
arg = PyObject_CallOneArg(subst, argitems[iparam]);
Py_DECREF(subst);
}
else {
arg = subs_tvars(arg, parameters, argitems);
if (arg == NULL) {
Py_DECREF(newargs);
return NULL;
}
}
if (arg == NULL) {
Py_DECREF(newargs);
return NULL;
}
PyTuple_SET_ITEM(newargs, iarg, arg);
}