Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-119127: functools.partial placeholders (gh-119827)

This commit is contained in:
dgpb 2024-09-26 04:04:38 +03:00 committed by GitHub
parent 4defb58d38
commit d9296529eb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 682 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
Doc/library/functools.rst
View File

@ -328,6 +328,14 @@ The :mod:`functools` module defines the following functions:
Returning ``NotImplemented`` from the underlying comparison function for
unrecognised types is now supported.
.. data:: Placeholder
A singleton object used as a sentinel to reserve a place
for positional arguments when calling :func:`partial`
and :func:`partialmethod`.
.. versionadded:: 3.14
.. function:: partial(func, /, *args, **keywords)
Return a new :ref:`partial object<partial-objects>` which when called
@ -338,26 +346,67 @@ The :mod:`functools` module defines the following functions:
Roughly equivalent to::
def partial(func, /, *args, **keywords):
def newfunc(*fargs, **fkeywords):
newkeywords = {**keywords, **fkeywords}
return func(*args, *fargs, **newkeywords)
def newfunc(*more_args, **more_keywords):
keywords_union = {**keywords, **more_keywords}
return func(*args, *more_args, **keywords_union)
newfunc.func = func
newfunc.args = args
newfunc.keywords = keywords
return newfunc
The :func:`partial` is used for partial function application which "freezes"
The :func:`partial` function is used for partial function application which "freezes"
some portion of a function's arguments and/or keywords resulting in a new object
with a simplified signature. For example, :func:`partial` can be used to create
a callable that behaves like the :func:`int` function where the *base* argument
defaults to two:
defaults to ``2``:
.. doctest::
>>> from functools import partial
>>> basetwo = partial(int, base=2)
>>> basetwo.__doc__ = 'Convert base 2 string to an int.'
>>> basetwo('10010')
18
If :data:`Placeholder` sentinels are present in *args*, they will be filled first
when :func:`partial` is called. This allows custom selection of positional arguments
to be pre-filled when constructing a :ref:`partial object <partial-objects>`.
If :data:`!Placeholder` sentinels are present, all of them must be filled at call time:
.. doctest::
>>> say_to_world = partial(print, Placeholder, Placeholder, "world!")
>>> say_to_world('Hello', 'dear')
Hello dear world!
Calling ``say_to_world('Hello')`` would raise a :exc:`TypeError`, because
only one positional argument is provided, while there are two placeholders
in :ref:`partial object <partial-objects>`.
Successive :func:`partial` applications fill :data:`!Placeholder` sentinels
of the input :func:`partial` objects with new positional arguments.
A place for positional argument can be retained by inserting new
:data:`!Placeholder` sentinel to the place held by previous :data:`!Placeholder`:
.. doctest::
>>> from functools import partial, Placeholder as _
>>> remove = partial(str.replace, _, _, '')
>>> message = 'Hello, dear dear world!'
>>> remove(message, ' dear')
'Hello, world!'
>>> remove_dear = partial(remove, _, ' dear')
>>> remove_dear(message)
'Hello, world!'
>>> remove_first_dear = partial(remove_dear, _, 1)
>>> remove_first_dear(message)
'Hello, dear world!'
Note, :data:`!Placeholder` has no special treatment when used for keyword
argument of :data:`!Placeholder`.
.. versionchanged:: 3.14
Added support for :data:`Placeholder` in positional arguments.
.. class:: partialmethod(func, /, *args, **keywords)
@ -742,10 +791,7 @@ have three read-only attributes:
The keyword arguments that will be supplied when the :class:`partial` object is
called.
:class:`partial` objects are like :ref:`function objects <user-defined-funcs>`
in that they are callable, weak referenceable, and can have attributes.
There are some important differences. For instance, the
:attr:`~function.__name__` and :attr:`function.__doc__` attributes
are not created automatically. Also, :class:`partial` objects defined in
classes behave like static methods and do not transform into bound methods
during instance attribute look-up.
:class:`partial` objects are like :class:`function` objects in that they are
callable, weak referenceable, and can have attributes. There are some important
differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes
are not created automatically.

9
Doc/whatsnew/3.14.rst
View File

@ -255,6 +255,15 @@ Added support for converting any objects that have the
(Contributed by Serhiy Storchaka in :gh:`82017`.)
functools
---------
* Added support to :func:`functools.partial` and
:func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels
to reserve a place for positional arguments.
(Contributed by Dominykas Grigonis in :gh:`119127`.)
http
----

194
Lib/functools.py
View File

@ -6,17 +6,18 @@
# Written by Nick Coghlan <ncoghlan at gmail.com>,
# Raymond Hettinger <python at rcn.com>,
# and Łukasz Langa <lukasz at langa.pl>.
# Copyright (C) 2006-2013 Python Software Foundation.
# Copyright (C) 2006-2024 Python Software Foundation.
# See C source code for _functools credits/copyright
__all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES',
'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce',
'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod',
'cached_property']
'cached_property', 'Placeholder']
from abc import get_cache_token
from collections import namedtuple
# import types, weakref # Deferred to single_dispatch()
from operator import itemgetter
from reprlib import recursive_repr
from types import MethodType
from _thread import RLock
@ -274,43 +275,125 @@ except ImportError:
### partial() argument application
################################################################################
class _PlaceholderType:
"""The type of the Placeholder singleton.
Used as a placeholder for partial arguments.
"""
__instance = None
__slots__ = ()
def __init_subclass__(cls, *args, **kwargs):
raise TypeError(f"type '{cls.__name__}' is not an acceptable base type")
def __new__(cls):
if cls.__instance is None:
cls.__instance = object.__new__(cls)
return cls.__instance
def __repr__(self):
return 'Placeholder'
def __reduce__(self):
return 'Placeholder'
Placeholder = _PlaceholderType()
def _partial_prepare_merger(args):
if not args:
return 0, None
nargs = len(args)
order = []
j = nargs
for i, a in enumerate(args):
if a is Placeholder:
order.append(j)
j += 1
else:
order.append(i)
phcount = j - nargs
merger = itemgetter(*order) if phcount else None
return phcount, merger
def _partial_new(cls, func, /, *args, **keywords):
if issubclass(cls, partial):
base_cls = partial
if not callable(func):
raise TypeError("the first argument must be callable")
else:
base_cls = partialmethod
# func could be a descriptor like classmethod which isn't callable
if not callable(func) and not hasattr(func, "__get__"):
raise TypeError(f"the first argument {func!r} must be a callable "
"or a descriptor")
if args and args[-1] is Placeholder:
raise TypeError("trailing Placeholders are not allowed")
if isinstance(func, base_cls):
pto_phcount = func._phcount
tot_args = func.args
if args:
tot_args += args
if pto_phcount:
# merge args with args of `func` which is `partial`
nargs = len(args)
if nargs < pto_phcount:
tot_args += (Placeholder,) * (pto_phcount - nargs)
tot_args = func._merger(tot_args)
if nargs > pto_phcount:
tot_args += args[pto_phcount:]
phcount, merger = _partial_prepare_merger(tot_args)
else: # works for both pto_phcount == 0 and != 0
phcount, merger = pto_phcount, func._merger
keywords = {**func.keywords, **keywords}
func = func.func
else:
tot_args = args
phcount, merger = _partial_prepare_merger(tot_args)
self = object.__new__(cls)
self.func = func
self.args = tot_args
self.keywords = keywords
self._phcount = phcount
self._merger = merger
return self
def _partial_repr(self):
cls = type(self)
module = cls.__module__
qualname = cls.__qualname__
args = [repr(self.func)]
args.extend(map(repr, self.args))
args.extend(f"{k}={v!r}" for k, v in self.keywords.items())
return f"{module}.{qualname}({', '.join(args)})"
# Purely functional, no descriptor behaviour
class partial:
"""New function with partial application of the given arguments
and keywords.
"""
__slots__ = "func", "args", "keywords", "__dict__", "__weakref__"
__slots__ = ("func", "args", "keywords", "_phcount", "_merger",
"__dict__", "__weakref__")
def __new__(cls, func, /, *args, **keywords):
if not callable(func):
raise TypeError("the first argument must be callable")
if isinstance(func, partial):
args = func.args + args
keywords = {**func.keywords, **keywords}
func = func.func
self = super(partial, cls).__new__(cls)
self.func = func
self.args = args
self.keywords = keywords
return self
__new__ = _partial_new
__repr__ = recursive_repr()(_partial_repr)
def __call__(self, /, *args, **keywords):
phcount = self._phcount
if phcount:
try:
pto_args = self._merger(self.args + args)
args = args[phcount:]
except IndexError:
raise TypeError("missing positional arguments "
"in 'partial' call; expected "
f"at least {phcount}, got {len(args)}")
else:
pto_args = self.args
keywords = {**self.keywords, **keywords}
return self.func(*self.args, *args, **keywords)
@recursive_repr()
def __repr__(self):
cls = type(self)
qualname = cls.__qualname__
module = cls.__module__
args = [repr(self.func)]
args.extend(repr(x) for x in self.args)
args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items())
return f"{module}.{qualname}({', '.join(args)})"
return self.func(*pto_args, *args, **keywords)
def __get__(self, obj, objtype=None):
if obj is None:
@ -332,6 +415,10 @@ class partial:
(namespace is not None and not isinstance(namespace, dict))):
raise TypeError("invalid partial state")
if args and args[-1] is Placeholder:
raise TypeError("trailing Placeholders are not allowed")
phcount, merger = _partial_prepare_merger(args)
args = tuple(args) # just in case it's a subclass
if kwds is None:
kwds = {}
@ -344,53 +431,40 @@ class partial:
self.func = func
self.args = args
self.keywords = kwds
self._phcount = phcount
self._merger = merger
try:
from _functools import partial
from _functools import partial, Placeholder, _PlaceholderType
except ImportError:
pass
# Descriptor version
class partialmethod(object):
class partialmethod:
"""Method descriptor with partial application of the given arguments
and keywords.
Supports wrapping existing descriptors and handles non-descriptor
callables as instance methods.
"""
def __init__(self, func, /, *args, **keywords):
if not callable(func) and not hasattr(func, "__get__"):
raise TypeError("{!r} is not callable or a descriptor"
.format(func))
# func could be a descriptor like classmethod which isn't callable,
# so we can't inherit from partial (it verifies func is callable)
if isinstance(func, partialmethod):
# flattening is mandatory in order to place cls/self before all
# other arguments
# it's also more efficient since only one function will be called
self.func = func.func
self.args = func.args + args
self.keywords = {**func.keywords, **keywords}
else:
self.func = func
self.args = args
self.keywords = keywords
def __repr__(self):
cls = type(self)
module = cls.__module__
qualname = cls.__qualname__
args = [repr(self.func)]
args.extend(map(repr, self.args))
args.extend(f"{k}={v!r}" for k, v in self.keywords.items())
return f"{module}.{qualname}({', '.join(args)})"
__new__ = _partial_new
__repr__ = _partial_repr
def _make_unbound_method(self):
def _method(cls_or_self, /, *args, **keywords):
phcount = self._phcount
if phcount:
try:
pto_args = self._merger(self.args + args)
args = args[phcount:]
except IndexError:
raise TypeError("missing positional arguments "
"in 'partialmethod' call; expected "
f"at least {phcount}, got {len(args)}")
else:
pto_args = self.args
keywords = {**self.keywords, **keywords}
return self.func(cls_or_self, *self.args, *args, **keywords)
return self.func(cls_or_self, *pto_args, *args, **keywords)
_method.__isabstractmethod__ = self.__isabstractmethod__
_method.__partialmethod__ = self
return _method

24
Lib/inspect.py
View File

@ -1930,7 +1930,12 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()):
if param.kind is _POSITIONAL_ONLY:
# If positional-only parameter is bound by partial,
# it effectively disappears from the signature
new_params.pop(param_name)
# However, if it is a Placeholder it is not removed
# And also looses default value
if arg_value is functools.Placeholder:
new_params[param_name] = param.replace(default=_empty)
else:
new_params.pop(param_name)
continue
if param.kind is _POSITIONAL_OR_KEYWORD:
@ -1952,7 +1957,17 @@ def _signature_get_partial(wrapped_sig, partial, extra_args=()):
new_params[param_name] = param.replace(default=arg_value)
else:
# was passed as a positional argument
new_params.pop(param.name)
# Do not pop if it is a Placeholder
# also change kind to positional only
# and remove default
if arg_value is functools.Placeholder:
new_param = param.replace(
kind=_POSITIONAL_ONLY,
default=_empty
)
new_params[param_name] = new_param
else:
new_params.pop(param_name)
continue
if param.kind is _KEYWORD_ONLY:
@ -2446,6 +2461,11 @@ def _signature_from_callable(obj, *,
sig_params = tuple(sig.parameters.values())
assert (not sig_params or
first_wrapped_param is not sig_params[0])
# If there were placeholders set,
# first param is transformed to positional only
if partialmethod.args.count(functools.Placeholder):
first_wrapped_param = first_wrapped_param.replace(
kind=Parameter.POSITIONAL_ONLY)
new_params = (first_wrapped_param,) + sig_params
return sig.replace(parameters=new_params)

103
Lib/test/test_functools.py
View File

@ -6,6 +6,7 @@ import copy
from itertools import permutations
import pickle
from random import choice
import re
import sys
from test import support
import threading
@ -210,6 +211,51 @@ class TestPartial:
p2.new_attr = 'spam'
self.assertEqual(p2.new_attr, 'spam')
def test_placeholders_trailing_raise(self):
PH = self.module.Placeholder
for args in [(PH,), (0, PH), (0, PH, 1, PH, PH, PH)]:
with self.assertRaises(TypeError):
self.partial(capture, *args)
def test_placeholders(self):
PH = self.module.Placeholder
# 1 Placeholder
args = (PH, 0)
p = self.partial(capture, *args)
actual_args, actual_kwds = p('x')
self.assertEqual(actual_args, ('x', 0))
self.assertEqual(actual_kwds, {})
# 2 Placeholders
args = (PH, 0, PH, 1)
p = self.partial(capture, *args)
with self.assertRaises(TypeError):
p('x')
actual_args, actual_kwds = p('x', 'y')
self.assertEqual(actual_args, ('x', 0, 'y', 1))
self.assertEqual(actual_kwds, {})
def test_placeholders_optimization(self):
PH = self.module.Placeholder
p = self.partial(capture, PH, 0)
p2 = self.partial(p, PH, 1, 2, 3)
self.assertEqual(p2.args, (PH, 0, 1, 2, 3))
p3 = self.partial(p2, -1, 4)
actual_args, actual_kwds = p3(5)
self.assertEqual(actual_args, (-1, 0, 1, 2, 3, 4, 5))
self.assertEqual(actual_kwds, {})
# inner partial has placeholders and outer partial has no args case
p = self.partial(capture, PH, 0)
p2 = self.partial(p)
self.assertEqual(p2.args, (PH, 0))
self.assertEqual(p2(1), ((1, 0), {}))
def test_construct_placeholder_singleton(self):
PH = self.module.Placeholder
tp = type(PH)
self.assertIs(tp(), PH)
self.assertRaises(TypeError, tp, 1, 2)
self.assertRaises(TypeError, tp, a=1, b=2)
def test_repr(self):
args = (object(), object())
args_repr = ', '.join(repr(a) for a in args)
@ -311,6 +357,23 @@ class TestPartial:
self.assertEqual(f(2), ((2,), {}))
self.assertEqual(f(), ((), {}))
# Set State with placeholders
PH = self.module.Placeholder
f = self.partial(signature)
f.__setstate__((capture, (PH, 1), dict(a=10), dict(attr=[])))
self.assertEqual(signature(f), (capture, (PH, 1), dict(a=10), dict(attr=[])))
msg_regex = re.escape("missing positional arguments in 'partial' call; "
"expected at least 1, got 0")
with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm:
f()
self.assertEqual(f(2), ((2, 1), dict(a=10)))
# Trailing Placeholder error
f = self.partial(signature)
msg_regex = re.escape("trailing Placeholders are not allowed")
with self.assertRaisesRegex(TypeError, f'^{msg_regex}$') as cm:
f.__setstate__((capture, (1, PH), dict(a=10), dict(attr=[])))
def test_setstate_errors(self):
f = self.partial(signature)
self.assertRaises(TypeError, f.__setstate__, (capture, (), {}))
@ -456,6 +519,19 @@ class TestPartialC(TestPartial, unittest.TestCase):
self.assertIn('astr', r)
self.assertIn("['sth']", r)
def test_placeholders_refcount_smoke(self):
PH = self.module.Placeholder
# sum supports vector call
lst1, start = [], []
sum_lists = self.partial(sum, PH, start)
for i in range(10):
sum_lists([lst1, lst1])
# collections.ChainMap initializer does not support vectorcall
map1, map2 = {}, {}
partial_cm = self.partial(collections.ChainMap, PH, map1)
for i in range(10):
partial_cm(map2, map2)
class TestPartialPy(TestPartial, unittest.TestCase):
module = py_functools
@ -480,6 +556,19 @@ class TestPartialCSubclass(TestPartialC):
class TestPartialPySubclass(TestPartialPy):
partial = PyPartialSubclass
def test_subclass_optimization(self):
# `partial` input to `partial` subclass
p = py_functools.partial(min, 2)
p2 = self.partial(p, 1)
self.assertIs(p2.func, min)
self.assertEqual(p2(0), 0)
# `partial` subclass input to `partial` subclass
p = self.partial(min, 2)
p2 = self.partial(p, 1)
self.assertIs(p2.func, min)
self.assertEqual(p2(0), 0)
class TestPartialMethod(unittest.TestCase):
class A(object):
@ -617,6 +706,20 @@ class TestPartialMethod(unittest.TestCase):
p = functools.partial(f, 1)
self.assertEqual(p(2), f(1, 2))
def test_subclass_optimization(self):
class PartialMethodSubclass(functools.partialmethod):
pass
# `partialmethod` input to `partialmethod` subclass
p = functools.partialmethod(min, 2)
p2 = PartialMethodSubclass(p, 1)
self.assertIs(p2.func, min)
self.assertEqual(p2.__get__(0)(), 0)
# `partialmethod` subclass input to `partialmethod` subclass
p = PartialMethodSubclass(min, 2)
p2 = PartialMethodSubclass(p, 1)
self.assertIs(p2.func, min)
self.assertEqual(p2.__get__(0)(), 0)
class TestUpdateWrapper(unittest.TestCase):

74
Lib/test/test_inspect/test_inspect.py
View File

@ -3341,7 +3341,7 @@ class TestSignatureObject(unittest.TestCase):
...))
def test_signature_on_partial(self):
from functools import partial
from functools import partial, Placeholder
def test():
pass
@ -3396,6 +3396,25 @@ class TestSignatureObject(unittest.TestCase):
('d', ..., ..., "keyword_only")),
...))
# With Placeholder
self.assertEqual(self.signature(partial(test, Placeholder, 1)),
((('a', ..., ..., "positional_only"),
('c', ..., ..., "keyword_only"),
('d', ..., ..., "keyword_only")),
...))
self.assertEqual(self.signature(partial(test, Placeholder, 1, c=2)),
((('a', ..., ..., "positional_only"),
('c', 2, ..., "keyword_only"),
('d', ..., ..., "keyword_only")),
...))
# Ensure unittest.mock.ANY & similar do not get picked up as a Placeholder
self.assertEqual(self.signature(partial(test, unittest.mock.ANY, 1, c=2)),
((('c', 2, ..., "keyword_only"),
('d', ..., ..., "keyword_only")),
...))
def test(a, *args, b, **kwargs):
pass
@ -3443,6 +3462,15 @@ class TestSignatureObject(unittest.TestCase):
('kwargs', ..., ..., "var_keyword")),
...))
# With Placeholder
p = partial(test, Placeholder, Placeholder, 1, b=0, test=1)
self.assertEqual(self.signature(p),
((('a', ..., ..., "positional_only"),
('args', ..., ..., "var_positional"),
('b', 0, ..., "keyword_only"),
('kwargs', ..., ..., "var_keyword")),
...))
def test(a, b, c:int) -> 42:
pass
@ -3547,6 +3575,34 @@ class TestSignatureObject(unittest.TestCase):
('kwargs', ..., ..., 'var_keyword')),
...))
# Positional only With Placeholder
p = partial(foo, Placeholder, 1, c=0, d=1)
self.assertEqual(self.signature(p),
((('a', ..., ..., "positional_only"),
('c', 0, ..., "keyword_only"),
('d', 1, ..., "keyword_only"),
('kwargs', ..., ..., "var_keyword")),
...))
# Optionals Positional With Placeholder
def foo(a=0, b=1, /, c=2, d=3):
pass
# Positional
p = partial(foo, Placeholder, 1, c=0, d=1)
self.assertEqual(self.signature(p),
((('a', ..., ..., "positional_only"),
('c', 0, ..., "keyword_only"),
('d', 1, ..., "keyword_only")),
...))
# Positional or Keyword - transformed to positional
p = partial(foo, Placeholder, 1, Placeholder, 1)
self.assertEqual(self.signature(p),
((('a', ..., ..., "positional_only"),
('c', ..., ..., "positional_only")),
...))
def test_signature_on_partialmethod(self):
from functools import partialmethod
@ -3559,18 +3615,32 @@ class TestSignatureObject(unittest.TestCase):
inspect.signature(Spam.ham)
class Spam:
def test(it, a, *, c) -> 'spam':
def test(it, a, b, *, c) -> 'spam':
pass
ham = partialmethod(test, c=1)
bar = partialmethod(test, functools.Placeholder, 1, c=1)
self.assertEqual(self.signature(Spam.ham, eval_str=False),
((('it', ..., ..., 'positional_or_keyword'),
('a', ..., ..., 'positional_or_keyword'),
('b', ..., ..., 'positional_or_keyword'),
('c', 1, ..., 'keyword_only')),
'spam'))
self.assertEqual(self.signature(Spam().ham, eval_str=False),
((('a', ..., ..., 'positional_or_keyword'),
('b', ..., ..., 'positional_or_keyword'),
('c', 1, ..., 'keyword_only')),
'spam'))
# With Placeholder
self.assertEqual(self.signature(Spam.bar, eval_str=False),
((('it', ..., ..., 'positional_only'),
('a', ..., ..., 'positional_only'),
('c', 1, ..., 'keyword_only')),
'spam'))
self.assertEqual(self.signature(Spam().bar, eval_str=False),
((('a', ..., ..., 'positional_only'),
('c', 1, ..., 'keyword_only')),
'spam'))

2
Misc/NEWS.d/next/Library/2024-05-25-00-54-26.gh-issue-119127.LpPvag.rst Normal file
View File

@ -0,0 +1,2 @@
Positional arguments of :func:`functools.partial` objects
now support placeholders via :data:`functools.Placeholder`.

334
Modules/_functoolsmodule.c
View File

@ -25,6 +25,8 @@ class _functools._lru_cache_wrapper "PyObject *" "&lru_cache_type_spec"
typedef struct _functools_state {
/* this object is used delimit args and keywords in the cache keys */
PyObject *kwd_mark;
PyTypeObject *placeholder_type;
PyObject *placeholder;
PyTypeObject *partial_type;
PyTypeObject *keyobject_type;
PyTypeObject *lru_list_elem_type;
@ -41,6 +43,79 @@ get_functools_state(PyObject *module)
/* partial object **********************************************************/
// The 'Placeholder' singleton indicates which formal positional
// parameters are to be bound first when using a 'partial' object.
typedef struct {
PyObject_HEAD
} placeholderobject;
static inline _functools_state *
get_functools_state_by_type(PyTypeObject *type);
PyDoc_STRVAR(placeholder_doc,
"The type of the Placeholder singleton.\n\n"
"Used as a placeholder for partial arguments.");
static PyObject *
placeholder_repr(PyObject *op)
{
return PyUnicode_FromString("Placeholder");
}
static PyObject *
placeholder_reduce(PyObject *op, PyObject *Py_UNUSED(ignored))
{
return PyUnicode_FromString("Placeholder");
}
static PyMethodDef placeholder_methods[] = {
{"__reduce__", placeholder_reduce, METH_NOARGS, NULL},
{NULL, NULL}
};
static void
placeholder_dealloc(PyObject* placeholder)
{
/* This should never get called, but we also don't want to SEGV if
* we accidentally decref Placeholder out of existence. Instead,
* since Placeholder is an immortal object, re-set the reference count.
*/
_Py_SetImmortal(placeholder);
}
static PyObject *
placeholder_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
{
if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) {
PyErr_SetString(PyExc_TypeError, "PlaceholderType takes no arguments");
return NULL;
}
_functools_state *state = get_functools_state_by_type(type);
if (state->placeholder == NULL) {
state->placeholder = PyType_GenericNew(type, NULL, NULL);
}
return state->placeholder;
}
static PyType_Slot placeholder_type_slots[] = {
{Py_tp_dealloc, placeholder_dealloc},
{Py_tp_repr, placeholder_repr},
{Py_tp_doc, (void *)placeholder_doc},
{Py_tp_methods, placeholder_methods},
{Py_tp_new, placeholder_new},
{0, 0}
};
static PyType_Spec placeholder_type_spec = {
.name = "functools._PlaceholderType",
.basicsize = sizeof(placeholderobject),
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE,
.slots = placeholder_type_slots
};
typedef struct {
PyObject_HEAD
PyObject *fn;
@ -48,6 +123,8 @@ typedef struct {
PyObject *kw;
PyObject *dict; /* __dict__ */
PyObject *weakreflist; /* List of weak references */
PyObject *placeholder; /* Placeholder for positional arguments */
Py_ssize_t phcount; /* Number of placeholders */
vectorcallfunc vectorcall;
} partialobject;
@ -70,23 +147,38 @@ get_functools_state_by_type(PyTypeObject *type)
static PyObject *
partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
{
PyObject *func, *pargs, *nargs, *pkw;
PyObject *func, *pto_args, *new_args, *pto_kw, *phold;
partialobject *pto;
Py_ssize_t pto_phcount = 0;
Py_ssize_t new_nargs = PyTuple_GET_SIZE(args) - 1;
if (PyTuple_GET_SIZE(args) < 1) {
if (new_nargs < 0) {
PyErr_SetString(PyExc_TypeError,
"type 'partial' takes at least one argument");
return NULL;
}
func = PyTuple_GET_ITEM(args, 0);
if (!PyCallable_Check(func)) {
PyErr_SetString(PyExc_TypeError,
"the first argument must be callable");
return NULL;
}
_functools_state *state = get_functools_state_by_type(type);
if (state == NULL) {
return NULL;
}
phold = state->placeholder;
pargs = pkw = NULL;
func = PyTuple_GET_ITEM(args, 0);
/* Placeholder restrictions */
if (new_nargs && PyTuple_GET_ITEM(args, new_nargs) == phold) {
PyErr_SetString(PyExc_TypeError,
"trailing Placeholders are not allowed");
return NULL;
}
/* check wrapped function / object */
pto_args = pto_kw = NULL;
int res = PyObject_TypeCheck(func, state->partial_type);
if (res == -1) {
return NULL;
@ -95,18 +187,14 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
// We can use its underlying function directly and merge the arguments.
partialobject *part = (partialobject *)func;
if (part->dict == NULL) {
pargs = part->args;
pkw = part->kw;
pto_args = part->args;
pto_kw = part->kw;
func = part->fn;
assert(PyTuple_Check(pargs));
assert(PyDict_Check(pkw));
pto_phcount = part->phcount;
assert(PyTuple_Check(pto_args));
assert(PyDict_Check(pto_kw));
}
}
if (!PyCallable_Check(func)) {
PyErr_SetString(PyExc_TypeError,
"the first argument must be callable");
return NULL;
}
/* create partialobject structure */
pto = (partialobject *)type->tp_alloc(type, 0);
@ -114,18 +202,58 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
return NULL;
pto->fn = Py_NewRef(func);
pto->placeholder = phold;
nargs = PyTuple_GetSlice(args, 1, PY_SSIZE_T_MAX);
if (nargs == NULL) {
new_args = PyTuple_GetSlice(args, 1, new_nargs + 1);
if (new_args == NULL) {
Py_DECREF(pto);
return NULL;
}
if (pargs == NULL) {
pto->args = nargs;
/* Count placeholders */
Py_ssize_t phcount = 0;
for (Py_ssize_t i = 0; i < new_nargs - 1; i++) {
if (PyTuple_GET_ITEM(new_args, i) == phold) {
phcount++;
}
}
/* merge args with args of `func` which is `partial` */
if (pto_phcount > 0 && new_nargs > 0) {
Py_ssize_t npargs = PyTuple_GET_SIZE(pto_args);
Py_ssize_t tot_nargs = npargs;
if (new_nargs > pto_phcount) {
tot_nargs += new_nargs - pto_phcount;
}
PyObject *item;
PyObject *tot_args = PyTuple_New(tot_nargs);
for (Py_ssize_t i = 0, j = 0; i < tot_nargs; i++) {
if (i < npargs) {
item = PyTuple_GET_ITEM(pto_args, i);
if (j < new_nargs && item == phold) {
item = PyTuple_GET_ITEM(new_args, j);
j++;
pto_phcount--;
}
}
else {
item = PyTuple_GET_ITEM(new_args, j);
j++;
}
Py_INCREF(item);
PyTuple_SET_ITEM(tot_args, i, item);
}
pto->args = tot_args;
pto->phcount = pto_phcount + phcount;
Py_DECREF(new_args);
}
else if (pto_args == NULL) {
pto->args = new_args;
pto->phcount = phcount;
}
else {
pto->args = PySequence_Concat(pargs, nargs);
Py_DECREF(nargs);
pto->args = PySequence_Concat(pto_args, new_args);
pto->phcount = pto_phcount + phcount;
Py_DECREF(new_args);
if (pto->args == NULL) {
Py_DECREF(pto);
return NULL;
@ -133,7 +261,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
assert(PyTuple_Check(pto->args));
}
if (pkw == NULL || PyDict_GET_SIZE(pkw) == 0) {
if (pto_kw == NULL || PyDict_GET_SIZE(pto_kw) == 0) {
if (kw == NULL) {
pto->kw = PyDict_New();
}
@ -145,7 +273,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw)
}
}
else {
pto->kw = PyDict_Copy(pkw);
pto->kw = PyDict_Copy(pto_kw);
if (kw != NULL && pto->kw != NULL) {
if (PyDict_Merge(pto->kw, kw, 1) != 0) {
Py_DECREF(pto);
@ -225,23 +353,30 @@ partial_vectorcall(partialobject *pto, PyObject *const *args,
size_t nargsf, PyObject *kwnames)
{
PyThreadState *tstate = _PyThreadState_GET();
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
/* pto->kw is mutable, so need to check every time */
if (PyDict_GET_SIZE(pto->kw)) {
return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames);
}
Py_ssize_t pto_phcount = pto->phcount;
if (nargs < pto_phcount) {
PyErr_Format(PyExc_TypeError,
"missing positional arguments in 'partial' call; "
"expected at least %zd, got %zd", pto_phcount, nargs);
return NULL;
}
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
Py_ssize_t nargs_total = nargs;
Py_ssize_t nargskw = nargs;
if (kwnames != NULL) {
nargs_total += PyTuple_GET_SIZE(kwnames);
nargskw += PyTuple_GET_SIZE(kwnames);
}
PyObject **pto_args = _PyTuple_ITEMS(pto->args);
Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args);
/* Fast path if we're called without arguments */
if (nargs_total == 0) {
if (nargskw == 0) {
return _PyObject_VectorcallTstate(tstate, pto->fn,
pto_args, pto_nargs, NULL);
}
@ -258,29 +393,47 @@ partial_vectorcall(partialobject *pto, PyObject *const *args,
return ret;
}
Py_ssize_t newnargs_total = pto_nargs + nargs_total;
PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
PyObject *ret;
PyObject **stack;
if (newnargs_total <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
Py_ssize_t tot_nargskw = pto_nargs + nargskw - pto_phcount;
if (tot_nargskw <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
stack = small_stack;
}
else {
stack = PyMem_Malloc(newnargs_total * sizeof(PyObject *));
stack = PyMem_Malloc(tot_nargskw * sizeof(PyObject *));
if (stack == NULL) {
PyErr_NoMemory();
return NULL;
}
}
/* Copy to new stack, using borrowed references */
memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*));
memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*));
ret = _PyObject_VectorcallTstate(tstate, pto->fn,
stack, pto_nargs + nargs, kwnames);
Py_ssize_t tot_nargs;
if (pto_phcount) {
tot_nargs = pto_nargs + nargs - pto_phcount;
Py_ssize_t j = 0; // New args index
for (Py_ssize_t i = 0; i < pto_nargs; i++) {
if (pto_args[i] == pto->placeholder) {
stack[i] = args[j];
j += 1;
}
else {
stack[i] = pto_args[i];
}
}
assert(j == pto_phcount);
if (nargskw > pto_phcount) {
memcpy(stack + pto_nargs, args + j, (nargskw - j) * sizeof(PyObject*));
}
}
else {
tot_nargs = pto_nargs + nargs;
/* Copy to new stack, using borrowed references */
memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*));
memcpy(stack + pto_nargs, args, nargskw * sizeof(PyObject*));
}
PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn,
stack, tot_nargs, kwnames);
if (stack != small_stack) {
PyMem_Free(stack);
}
@ -312,40 +465,81 @@ partial_call(partialobject *pto, PyObject *args, PyObject *kwargs)
assert(PyTuple_Check(pto->args));
assert(PyDict_Check(pto->kw));
Py_ssize_t nargs = PyTuple_GET_SIZE(args);
Py_ssize_t pto_phcount = pto->phcount;
if (nargs < pto_phcount) {
PyErr_Format(PyExc_TypeError,
"missing positional arguments in 'partial' call; "
"expected at least %zd, got %zd", pto_phcount, nargs);
return NULL;
}
/* Merge keywords */
PyObject *kwargs2;
PyObject *tot_kw;
if (PyDict_GET_SIZE(pto->kw) == 0) {
/* kwargs can be NULL */
kwargs2 = Py_XNewRef(kwargs);
tot_kw = Py_XNewRef(kwargs);
}
else {
/* bpo-27840, bpo-29318: dictionary of keyword parameters must be
copied, because a function using "**kwargs" can modify the
dictionary. */
kwargs2 = PyDict_Copy(pto->kw);
if (kwargs2 == NULL) {
tot_kw = PyDict_Copy(pto->kw);
if (tot_kw == NULL) {
return NULL;
}
if (kwargs != NULL) {
if (PyDict_Merge(kwargs2, kwargs, 1) != 0) {
Py_DECREF(kwargs2);
if (PyDict_Merge(tot_kw, kwargs, 1) != 0) {
Py_DECREF(tot_kw);
return NULL;
}
}
}
/* Merge positional arguments */
/* Note: tupleconcat() is optimized for empty tuples */
PyObject *args2 = PySequence_Concat(pto->args, args);
if (args2 == NULL) {
Py_XDECREF(kwargs2);
return NULL;
PyObject *tot_args;
if (pto_phcount) {
Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args);
Py_ssize_t tot_nargs = pto_nargs + nargs - pto_phcount;
assert(tot_nargs >= 0);
tot_args = PyTuple_New(tot_nargs);
if (tot_args == NULL) {
Py_XDECREF(tot_kw);
return NULL;
}
PyObject *pto_args = pto->args;
PyObject *item;
Py_ssize_t j = 0; // New args index
for (Py_ssize_t i = 0; i < pto_nargs; i++) {
item = PyTuple_GET_ITEM(pto_args, i);
if (item == pto->placeholder) {
item = PyTuple_GET_ITEM(args, j);
j += 1;
}
Py_INCREF(item);
PyTuple_SET_ITEM(tot_args, i, item);
}
assert(j == pto_phcount);
for (Py_ssize_t i = pto_nargs; i < tot_nargs; i++) {
item = PyTuple_GET_ITEM(args, j);
Py_INCREF(item);
PyTuple_SET_ITEM(tot_args, i, item);
j += 1;
}
}
else {
/* Note: tupleconcat() is optimized for empty tuples */
tot_args = PySequence_Concat(pto->args, args);
if (tot_args == NULL) {
Py_XDECREF(tot_kw);
return NULL;
}
}
PyObject *res = PyObject_Call(pto->fn, args2, kwargs2);
Py_DECREF(args2);
Py_XDECREF(kwargs2);
PyObject *res = PyObject_Call(pto->fn, tot_args, tot_kw);
Py_DECREF(tot_args);
Py_XDECREF(tot_kw);
return res;
}
@ -461,8 +655,11 @@ partial_setstate(partialobject *pto, PyObject *state)
{
PyObject *fn, *fnargs, *kw, *dict;
if (!PyTuple_Check(state) ||
!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) ||
if (!PyTuple_Check(state)) {
PyErr_SetString(PyExc_TypeError, "invalid partial state");
return NULL;
}
if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) ||
!PyCallable_Check(fn) ||
!PyTuple_Check(fnargs) ||
(kw != Py_None && !PyDict_Check(kw)))
@ -471,6 +668,20 @@ partial_setstate(partialobject *pto, PyObject *state)
return NULL;
}
Py_ssize_t nargs = PyTuple_GET_SIZE(fnargs);
if (nargs && PyTuple_GET_ITEM(fnargs, nargs - 1) == pto->placeholder) {
PyErr_SetString(PyExc_TypeError,
"trailing Placeholders are not allowed");
return NULL;
}
/* Count placeholders */
Py_ssize_t phcount = 0;
for (Py_ssize_t i = 0; i < nargs - 1; i++) {
if (PyTuple_GET_ITEM(fnargs, i) == pto->placeholder) {
phcount++;
}
}
if(!PyTuple_CheckExact(fnargs))
fnargs = PySequence_Tuple(fnargs);
else
@ -493,10 +704,10 @@ partial_setstate(partialobject *pto, PyObject *state)
dict = NULL;
else
Py_INCREF(dict);
Py_SETREF(pto->fn, Py_NewRef(fn));
Py_SETREF(pto->args, fnargs);
Py_SETREF(pto->kw, kw);
pto->phcount = phcount;
Py_XSETREF(pto->dict, dict);
partial_setvectorcall(pto);
Py_RETURN_NONE;
@ -1498,6 +1709,21 @@ _functools_exec(PyObject *module)
return -1;
}
state->placeholder_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
&placeholder_type_spec, NULL);
if (state->placeholder_type == NULL) {
return -1;
}
if (PyModule_AddType(module, state->placeholder_type) < 0) {
return -1;
}
state->placeholder = PyObject_CallNoArgs((PyObject *)state->placeholder_type);
if (state->placeholder == NULL) {
return -1;
}
if (PyModule_AddObject(module, "Placeholder", state->placeholder) < 0) {
return -1;
}
state->partial_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
&partial_type_spec, NULL);
if (state->partial_type == NULL) {
@ -1542,6 +1768,7 @@ _functools_traverse(PyObject *module, visitproc visit, void *arg)
{
_functools_state *state = get_functools_state(module);
Py_VISIT(state->kwd_mark);
Py_VISIT(state->placeholder_type);
Py_VISIT(state->partial_type);
Py_VISIT(state->keyobject_type);
Py_VISIT(state->lru_list_elem_type);
@ -1553,6 +1780,7 @@ _functools_clear(PyObject *module)
{
_functools_state *state = get_functools_state(module);
Py_CLEAR(state->kwd_mark);
Py_CLEAR(state->placeholder_type);
Py_CLEAR(state->partial_type);
Py_CLEAR(state->keyobject_type);
Py_CLEAR(state->lru_list_elem_type);