[Python-checkins] gh-91162: Support splitting of unpacked arbitrary-length tuple over TypeVar and TypeVarTuple parameters (alt) (GH-93412)
serhiy-storchaka
webhook-mailer at python.org
Sun Jun 12 09:22:11 EDT 2022
https://github.com/python/cpython/commit/3473817106c23eca7341c931453da0341c367e1d
commit: 3473817106c23eca7341c931453da0341c367e1d
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2022-06-12T16:22:01+03:00
summary:
gh-91162: Support splitting of unpacked arbitrary-length tuple over TypeVar and TypeVarTuple parameters (alt) (GH-93412)
For example:
A[T, *Ts][*tuple[int, ...]] -> A[int, *tuple[int, ...]]
A[*Ts, T][*tuple[int, ...]] -> A[*tuple[int, ...], int]
files:
A Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst
M Include/internal/pycore_global_strings.h
M Include/internal/pycore_runtime_init.h
M Lib/test/test_typing.py
M Lib/typing.py
M Objects/genericaliasobject.c
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
index cfa8ae99d1b6d..ca970627cb2e1 100644
--- a/Include/internal/pycore_global_strings.h
+++ b/Include/internal/pycore_global_strings.h
@@ -202,6 +202,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(__truediv__)
STRUCT_FOR_ID(__trunc__)
STRUCT_FOR_ID(__typing_is_unpacked_typevartuple__)
+ STRUCT_FOR_ID(__typing_prepare_subst__)
STRUCT_FOR_ID(__typing_subst__)
STRUCT_FOR_ID(__typing_unpacked_tuple_args__)
STRUCT_FOR_ID(__warningregistry__)
diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h
index 737507f07eacc..5e57ac68776c8 100644
--- a/Include/internal/pycore_runtime_init.h
+++ b/Include/internal/pycore_runtime_init.h
@@ -824,6 +824,7 @@ extern "C" {
INIT_ID(__truediv__), \
INIT_ID(__trunc__), \
INIT_ID(__typing_is_unpacked_typevartuple__), \
+ INIT_ID(__typing_prepare_subst__), \
INIT_ID(__typing_subst__), \
INIT_ID(__typing_unpacked_tuple_args__), \
INIT_ID(__warningregistry__), \
diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index d6cd3d9bdd6a4..dfbe2d9d34c8f 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -753,14 +753,11 @@ class C(Generic[*Ts]): pass
('generic[*Ts]', '[*tuple_type[int]]', 'generic[int]'),
('generic[*Ts]', '[*tuple_type[*Ts]]', 'generic[*Ts]'),
('generic[*Ts]', '[*tuple_type[int, str]]', 'generic[int, str]'),
+ ('generic[*Ts]', '[str, *tuple_type[int, ...], bool]', 'generic[str, *tuple_type[int, ...], bool]'),
('generic[*Ts]', '[tuple_type[int, ...]]', 'generic[tuple_type[int, ...]]'),
('generic[*Ts]', '[tuple_type[int, ...], tuple_type[str, ...]]', 'generic[tuple_type[int, ...], tuple_type[str, ...]]'),
('generic[*Ts]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...]]'),
-
- # Technically, multiple unpackings are forbidden by PEP 646, but we
- # choose to be less restrictive at runtime, to allow folks room
- # to experiment. So all three of these should be valid.
- ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'generic[*tuple_type[int, ...], *tuple_type[str, ...]]'),
+ ('generic[*Ts]', '[*tuple_type[int, ...], *tuple_type[str, ...]]', 'TypeError'),
('generic[*Ts]', '[*Ts]', 'generic[*Ts]'),
('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'),
@@ -772,8 +769,6 @@ class C(Generic[*Ts]): pass
('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'),
('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'),
- ('generic[T, *Ts]', '[*tuple[int, ...]]', 'TypeError'), # Should be generic[int, *tuple[int, ...]]
-
('generic[*Ts, T]', '[int]', 'generic[int]'),
('generic[*Ts, T]', '[int, str]', 'generic[int, str]'),
('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'),
@@ -781,6 +776,14 @@ class C(Generic[*Ts]): pass
('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'),
('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'),
+ ('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'),
+ ('generic[*Ts, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], int]'),
+ ('generic[T1, *Ts, T2]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...], int]'),
+ ('generic[T, str, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, str, *tuple_type[int, ...]]'),
+ ('generic[*Ts, str, T]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], str, int]'),
+ ('generic[list[T], *Ts]', '[*tuple_type[int, ...]]', 'generic[list[int], *tuple_type[int, ...]]'),
+ ('generic[*Ts, list[T]]', '[*tuple_type[int, ...]]', 'generic[*tuple_type[int, ...], list[int]]'),
+
('generic[T, *tuple_type[int, ...]]', '[str]', 'generic[str, *tuple_type[int, ...]]'),
('generic[T1, T2, *tuple_type[int, ...]]', '[str, bool]', 'generic[str, bool, *tuple_type[int, ...]]'),
('generic[T1, *tuple_type[int, ...], T2]', '[str, bool]', 'generic[str, *tuple_type[int, ...], bool]'),
diff --git a/Lib/typing.py b/Lib/typing.py
index 40ab516f7c8ff..25cae7ffb8d78 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -1065,6 +1065,42 @@ def __repr__(self):
def __typing_subst__(self, arg):
raise TypeError("Substitution of bare TypeVarTuple is not supported")
+ def __typing_prepare_subst__(self, alias, args):
+ params = alias.__parameters__
+ typevartuple_index = params.index(self)
+ for param in enumerate(params[typevartuple_index + 1:]):
+ if isinstance(param, TypeVarTuple):
+ raise TypeError(f"More than one TypeVarTuple parameter in {alias}")
+
+ alen = len(args)
+ plen = len(params)
+ left = typevartuple_index
+ right = plen - typevartuple_index - 1
+ var_tuple_index = None
+ fillarg = None
+ for k, arg in enumerate(args):
+ if not (isinstance(arg, type) and not isinstance(arg, GenericAlias)):
+ subargs = getattr(arg, '__typing_unpacked_tuple_args__', None)
+ if subargs and len(subargs) == 2 and subargs[-1] is ...:
+ if var_tuple_index is not None:
+ raise TypeError("More than one unpacked arbitrary-length tuple argument")
+ var_tuple_index = k
+ fillarg = subargs[0]
+ if var_tuple_index is not None:
+ left = min(left, var_tuple_index)
+ right = min(right, alen - var_tuple_index - 1)
+ elif left + right > alen:
+ raise TypeError(f"Too few arguments for {alias};"
+ f" actual {alen}, expected at least {plen-1}")
+
+ return (
+ *args[:left],
+ *([fillarg]*(typevartuple_index - left)),
+ tuple(args[left: alen - right]),
+ *([fillarg]*(plen - right - left - typevartuple_index - 1)),
+ *args[alen - right:],
+ )
+
class ParamSpecArgs(_Final, _Immutable, _root=True):
"""The args for a ParamSpec object.
@@ -1184,6 +1220,8 @@ def __typing_subst__(self, arg):
f"ParamSpec, or Concatenate. Got {arg}")
return arg
+ def __typing_prepare_subst__(self, alias, args):
+ return _prepare_paramspec_params(alias, args)
def _is_dunder(attr):
return attr.startswith('__') and attr.endswith('__')
@@ -1255,44 +1293,6 @@ def __dir__(self):
+ [attr for attr in dir(self.__origin__) if not _is_dunder(attr)]))
-def _is_unpacked_tuple(x: Any) -> bool:
- # Is `x` something like `*tuple[int]` or `*tuple[int, ...]`?
- if not isinstance(x, _UnpackGenericAlias):
- return False
- # Alright, `x` is `Unpack[something]`.
-
- # `x` will always have `__args__`, because Unpack[] and Unpack[()]
- # aren't legal.
- unpacked_type = x.__args__[0]
-
- return getattr(unpacked_type, '__origin__', None) is tuple
-
-
-def _is_unpacked_arbitrary_length_tuple(x: Any) -> bool:
- if not _is_unpacked_tuple(x):
- return False
- unpacked_tuple = x.__args__[0]
-
- if not hasattr(unpacked_tuple, '__args__'):
- # It's `Unpack[tuple]`. We can't make any assumptions about the length
- # of the tuple, so it's effectively an arbitrary-length tuple.
- return True
-
- tuple_args = unpacked_tuple.__args__
- if not tuple_args:
- # It's `Unpack[tuple[()]]`.
- return False
-
- last_arg = tuple_args[-1]
- if last_arg is Ellipsis:
- # It's `Unpack[tuple[something, ...]]`, which is arbitrary-length.
- return True
-
- # If the arguments didn't end with an ellipsis, then it's not an
- # arbitrary-length tuple.
- return False
-
-
# Special typing constructs Union, Optional, Generic, Callable and Tuple
# use three special attributes for internal bookkeeping of generic types:
# * __parameters__ is a tuple of unique free type parameters of a generic
@@ -1385,10 +1385,6 @@ def __getitem__(self, args):
args = (args,)
args = tuple(_type_convert(p) for p in args)
args = _unpack_args(args)
- if (self._paramspec_tvars
- and any(isinstance(t, ParamSpec) for t in self.__parameters__)):
- args = _prepare_paramspec_params(self, args)
-
new_args = self._determine_new_args(args)
r = self.copy_with(new_args)
return r
@@ -1410,30 +1406,16 @@ def _determine_new_args(self, args):
params = self.__parameters__
# In the example above, this would be {T3: str}
- new_arg_by_param = {}
- typevartuple_index = None
- for i, param in enumerate(params):
- if isinstance(param, TypeVarTuple):
- if typevartuple_index is not None:
- raise TypeError(f"More than one TypeVarTuple parameter in {self}")
- typevartuple_index = i
-
+ for param in params:
+ prepare = getattr(param, '__typing_prepare_subst__', None)
+ if prepare is not None:
+ args = prepare(self, args)
alen = len(args)
plen = len(params)
- if typevartuple_index is not None:
- i = typevartuple_index
- j = alen - (plen - i - 1)
- if j < i:
- raise TypeError(f"Too few arguments for {self};"
- f" actual {alen}, expected at least {plen-1}")
- new_arg_by_param.update(zip(params[:i], args[:i]))
- new_arg_by_param[params[i]] = tuple(args[i: j])
- new_arg_by_param.update(zip(params[i + 1:], args[j:]))
- else:
- if alen != plen:
- raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
- f" actual {alen}, expected {plen}")
- new_arg_by_param.update(zip(params, args))
+ if alen != plen:
+ raise TypeError(f"Too {'many' if alen > plen else 'few'} arguments for {self};"
+ f" actual {alen}, expected {plen}")
+ new_arg_by_param = dict(zip(params, args))
new_args = []
for old_arg in self.__args__:
diff --git a/Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst b/Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst
new file mode 100644
index 0000000000000..09fa47c0d2384
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-06-01-11-24-13.gh-issue-91162.NxvU_u.rst
@@ -0,0 +1,5 @@
+Support splitting of unpacked arbitrary-length tuple over ``TypeVar`` and
+``TypeVarTuple`` parameters. For example:
+
+* ``A[T, *Ts][*tuple[int, ...]]`` -> ``A[int, *tuple[int, ...]]``
+* ``A[*Ts, T][*tuple[int, ...]]`` -> ``A[*tuple[int, ...], int]``
diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c
index 59420816496f0..0a0d0cc4c15b6 100644
--- a/Objects/genericaliasobject.c
+++ b/Objects/genericaliasobject.c
@@ -269,7 +269,7 @@ _Py_make_parameters(PyObject *args)
a non-empty tuple, return a new reference to obj. */
static PyObject *
subs_tvars(PyObject *obj, PyObject *params,
- PyObject **argitems, Py_ssize_t nargs, Py_ssize_t varparam)
+ PyObject **argitems, Py_ssize_t nargs)
{
PyObject *subparams;
if (_PyObject_LookupAttr(obj, &_Py_ID(__parameters__), &subparams) < 0) {
@@ -283,28 +283,28 @@ subs_tvars(PyObject *obj, PyObject *params,
Py_DECREF(subparams);
return NULL;
}
- for (Py_ssize_t i = 0, j = 0; i < nsubargs; ++i) {
+ Py_ssize_t j = 0;
+ for (Py_ssize_t i = 0; i < nsubargs; ++i) {
PyObject *arg = PyTuple_GET_ITEM(subparams, i);
Py_ssize_t iparam = tuple_index(params, nparams, arg);
- if (iparam == varparam) {
- j = tuple_extend(&subargs, j,
- argitems + iparam, nargs - nparams + 1);
- if (j < 0) {
- return NULL;
- }
- }
- else {
- if (iparam >= 0) {
- if (iparam > varparam) {
- iparam += nargs - nparams;
+ if (iparam >= 0) {
+ PyObject *param = PyTuple_GET_ITEM(params, iparam);
+ arg = argitems[iparam];
+ if (Py_TYPE(param)->tp_iter && PyTuple_Check(arg)) { // TypeVarTuple
+ j = tuple_extend(&subargs, j,
+ &PyTuple_GET_ITEM(arg, 0),
+ PyTuple_GET_SIZE(arg));
+ if (j < 0) {
+ return NULL;
}
- arg = argitems[iparam];
+ continue;
}
- Py_INCREF(arg);
- PyTuple_SET_ITEM(subargs, j, arg);
- j++;
}
+ Py_INCREF(arg);
+ PyTuple_SET_ITEM(subargs, j, arg);
+ j++;
}
+ assert(j == PyTuple_GET_SIZE(subargs));
obj = PyObject_GetItem(obj, subargs);
@@ -409,39 +409,37 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
self);
}
item = _unpack_args(item);
- int is_tuple = PyTuple_Check(item);
- Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
- PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
- Py_ssize_t varparam = nparams;
for (Py_ssize_t i = 0; i < nparams; i++) {
PyObject *param = PyTuple_GET_ITEM(parameters, i);
- if (Py_TYPE(param)->tp_iter) { // TypeVarTuple
- if (varparam < nparams) {
- Py_DECREF(item);
- return PyErr_Format(PyExc_TypeError,
- "More than one TypeVarTuple parameter in %S",
- self);
- }
- varparam = i;
- }
- }
- if (varparam < nparams) {
- if (nitems < nparams - 1) {
+ PyObject *prepare, *tmp;
+ if (_PyObject_LookupAttr(param, &_Py_ID(__typing_prepare_subst__), &prepare) < 0) {
Py_DECREF(item);
- return PyErr_Format(PyExc_TypeError,
- "Too few arguments for %R",
- self);
+ return NULL;
}
- }
- else {
- if (nitems != nparams) {
- Py_DECREF(item);
- return PyErr_Format(PyExc_TypeError,
- "Too %s arguments for %R; actual %zd, expected %zd",
- nitems > nparams ? "many" : "few",
- self, nitems, nparams);
+ if (prepare && prepare != Py_None) {
+ if (PyTuple_Check(item)) {
+ tmp = PyObject_CallFunction(prepare, "OO", self, item);
+ }
+ else {
+ tmp = PyObject_CallFunction(prepare, "O(O)", self, item);
+ }
+ Py_DECREF(prepare);
+ Py_SETREF(item, tmp);
+ if (item == NULL) {
+ return NULL;
+ }
}
}
+ int is_tuple = PyTuple_Check(item);
+ Py_ssize_t nitems = is_tuple ? PyTuple_GET_SIZE(item) : 1;
+ PyObject **argitems = is_tuple ? &PyTuple_GET_ITEM(item, 0) : &item;
+ if (nitems != nparams) {
+ Py_DECREF(item);
+ return PyErr_Format(PyExc_TypeError,
+ "Too %s arguments for %R; actual %zd, expected %zd",
+ nitems > nparams ? "many" : "few",
+ self, nitems, nparams);
+ }
/* Replace all type variables (specified by parameters)
with corresponding values specified by argitems.
t = list[T]; t[int] -> newargs = [int]
@@ -471,22 +469,11 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje
if (subst) {
Py_ssize_t iparam = tuple_index(parameters, nparams, arg);
assert(iparam >= 0);
- if (iparam == varparam) {
- Py_DECREF(subst);
- Py_DECREF(newargs);
- Py_DECREF(item);
- PyErr_SetString(PyExc_TypeError,
- "Substitution of bare TypeVarTuple is not supported");
- return NULL;
- }
- if (iparam > varparam) {
- iparam += nitems - nparams;
- }
arg = PyObject_CallOneArg(subst, argitems[iparam]);
Py_DECREF(subst);
}
else {
- arg = subs_tvars(arg, parameters, argitems, nitems, varparam);
+ arg = subs_tvars(arg, parameters, argitems, nitems);
}
if (arg == NULL) {
Py_DECREF(newargs);
More information about the Python-checkins
mailing list