[Python-checkins] bpo-32873: Treat type variables and special typing forms as immutable by copy and pickle (GH-6216)
Ivan Levkivskyi
webhook-mailer at python.org
Mon Mar 26 18:01:21 EDT 2018
https://github.com/python/cpython/commit/834940375ae88bc95794226dd8eff1f25fba1cf9
commit: 834940375ae88bc95794226dd8eff1f25fba1cf9
branch: master
author: Ivan Levkivskyi <levkivskyi at gmail.com>
committer: GitHub <noreply at github.com>
date: 2018-03-26T23:01:12+01:00
summary:
bpo-32873: Treat type variables and special typing forms as immutable by copy and pickle (GH-6216)
This also fixes python/typing#512
This also fixes python/typing#511
As was discussed in both issues, some typing forms deserve to be treated
as immutable by copy and pickle modules, so that:
* copy(X) is X
* deepcopy(X) is X
* loads(dumps(X)) is X # pickled by reference
This PR adds such behaviour to:
* Type variables
* Special forms like Union, Any, ClassVar
* Unsubscripted generic aliases to containers like List, Mapping, Iterable
This not only resolves inconsistencies mentioned in the issues, but also
improves backwards compatibility with previous versions of Python
(including 3.6).
Note that this requires some dances with __module__ for type variables
(similar to NamedTuple) because the class TypeVar itself is define in typing,
while type variables should get module where they were defined.
https://bugs.python.org/issue32873
files:
A Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst
M Lib/test/test_typing.py
M Lib/typing.py
diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
index f56caa13a295..09e39fec45e4 100644
--- a/Lib/test/test_typing.py
+++ b/Lib/test/test_typing.py
@@ -1057,20 +1057,20 @@ class C(B[int]):
self.assertEqual(x.foo, 42)
self.assertEqual(x.bar, 'abc')
self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'})
- samples = [Any, Union, Tuple, Callable, ClassVar]
+ samples = [Any, Union, Tuple, Callable, ClassVar,
+ Union[int, str], ClassVar[List], Tuple[int, ...], Callable[[str], bytes]]
for s in samples:
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
z = pickle.dumps(s, proto)
x = pickle.loads(z)
self.assertEqual(s, x)
- more_samples = [List, typing.Iterable, typing.Type]
+ more_samples = [List, typing.Iterable, typing.Type, List[int],
+ typing.Type[typing.Mapping]]
for s in more_samples:
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
z = pickle.dumps(s, proto)
x = pickle.loads(z)
- self.assertEqual(repr(s), repr(x)) # TODO: fix this
- # see also comment in test_copy_and_deepcopy
- # the issue is typing/#512
+ self.assertEqual(s, x)
def test_copy_and_deepcopy(self):
T = TypeVar('T')
@@ -1082,7 +1082,27 @@ class Node(Generic[T]): ...
Union['T', int], List['T'], typing.Mapping['T', int]]
for t in things + [Any]:
self.assertEqual(t, copy(t))
- self.assertEqual(repr(t), repr(deepcopy(t))) # Use repr() because of TypeVars
+ self.assertEqual(t, deepcopy(t))
+
+ def test_immutability_by_copy_and_pickle(self):
+ # Special forms like Union, Any, etc., generic aliases to containers like List,
+ # Mapping, etc., and type variabcles are considered immutable by copy and pickle.
+ global TP, TPB, TPV # for pickle
+ TP = TypeVar('TP')
+ TPB = TypeVar('TPB', bound=int)
+ TPV = TypeVar('TPV', bytes, str)
+ for X in [TP, TPB, TPV, List, typing.Mapping, ClassVar, typing.Iterable,
+ Union, Any, Tuple, Callable]:
+ self.assertIs(copy(X), X)
+ self.assertIs(deepcopy(X), X)
+ self.assertIs(pickle.loads(pickle.dumps(X)), X)
+ # Check that local type variables are copyable.
+ TL = TypeVar('TL')
+ TLB = TypeVar('TLB', bound=int)
+ TLV = TypeVar('TLV', bytes, str)
+ for X in [TL, TLB, TLV]:
+ self.assertIs(copy(X), X)
+ self.assertIs(deepcopy(X), X)
def test_copy_generic_instances(self):
T = TypeVar('T')
diff --git a/Lib/typing.py b/Lib/typing.py
index 56126cfc0247..510574c413e3 100644
--- a/Lib/typing.py
+++ b/Lib/typing.py
@@ -285,8 +285,17 @@ def __init_subclass__(self, *args, **kwds):
if '_root' not in kwds:
raise TypeError("Cannot subclass special typing classes")
+class _Immutable:
+ """Mixin to indicate that object should not be copied."""
-class _SpecialForm(_Final, _root=True):
+ def __copy__(self):
+ return self
+
+ def __deepcopy__(self, memo):
+ return self
+
+
+class _SpecialForm(_Final, _Immutable, _root=True):
"""Internal indicator of special typing constructs.
See _doc instance attribute for specific docs.
"""
@@ -328,8 +337,8 @@ def __hash__(self):
def __repr__(self):
return 'typing.' + self._name
- def __copy__(self):
- return self # Special forms are immutable.
+ def __reduce__(self):
+ return self._name
def __call__(self, *args, **kwds):
raise TypeError(f"Cannot instantiate {self!r}")
@@ -496,7 +505,11 @@ def __repr__(self):
return f'ForwardRef({self.__forward_arg__!r})'
-class TypeVar(_Final, _root=True):
+def _find_name(mod, name):
+ return getattr(sys.modules[mod], name)
+
+
+class TypeVar(_Final, _Immutable, _root=True):
"""Type variable.
Usage::
@@ -536,10 +549,12 @@ def longest(x: A, y: A) -> A:
T.__covariant__ == False
T.__contravariant__ = False
A.__constraints__ == (str, bytes)
+
+ Note that only type variables defined in global scope can be pickled.
"""
__slots__ = ('__name__', '__bound__', '__constraints__',
- '__covariant__', '__contravariant__')
+ '__covariant__', '__contravariant__', '_def_mod')
def __init__(self, name, *constraints, bound=None,
covariant=False, contravariant=False):
@@ -558,6 +573,7 @@ def __init__(self, name, *constraints, bound=None,
self.__bound__ = _type_check(bound, "Bound must be a type.")
else:
self.__bound__ = None
+ self._def_mod = sys._getframe(1).f_globals['__name__'] # for pickling
def __getstate__(self):
return {'name': self.__name__,
@@ -582,6 +598,9 @@ def __repr__(self):
prefix = '~'
return prefix + self.__name__
+ def __reduce__(self):
+ return (_find_name, (self._def_mod, self.__name__))
+
# Special typing constructs Union, Optional, Generic, Callable and Tuple
# use three special attributes for internal bookkeeping of generic types:
@@ -724,6 +743,11 @@ def __subclasscheck__(self, cls):
raise TypeError("Subscripted generics cannot be used with"
" class and instance checks")
+ def __reduce__(self):
+ if self._special:
+ return self._name
+ return super().__reduce__()
+
class _VariadicGenericAlias(_GenericAlias, _root=True):
"""Same as _GenericAlias above but for variadic aliases. Currently,
diff --git a/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst
new file mode 100644
index 000000000000..99f8088cf138
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-03-24-19-54-48.bpo-32873.cHyoAm.rst
@@ -0,0 +1,3 @@
+Treat type variables and special typing forms as immutable by copy and
+pickle. This fixes several minor issues and inconsistencies, and improves
+backwards compatibility with Python 3.6.
More information about the Python-checkins
mailing list