[Python-checkins] bpo-26579: Add object.__getstate__(). (GH-2821)
serhiy-storchaka
webhook-mailer at python.org
Wed Apr 6 13:00:20 EDT 2022
https://github.com/python/cpython/commit/884eba3c76916889fd6bff3b37b8552bfb4f9566
commit: 884eba3c76916889fd6bff3b37b8552bfb4f9566
branch: main
author: Serhiy Storchaka <storchaka at gmail.com>
committer: serhiy-storchaka <storchaka at gmail.com>
date: 2022-04-06T20:00:14+03:00
summary:
bpo-26579: Add object.__getstate__(). (GH-2821)
Copying and pickling instances of subclasses of builtin types
bytearray, set, frozenset, collections.OrderedDict, collections.deque,
weakref.WeakSet, and datetime.tzinfo now copies and pickles instance attributes
implemented as slots.
files:
A Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst
M Doc/library/pickle.rst
M Doc/whatsnew/3.11.rst
M Include/object.h
M Lib/_weakrefset.py
M Lib/collections/__init__.py
M Lib/copyreg.py
M Lib/datetime.py
M Lib/email/headerregistry.py
M Lib/test/datetimetester.py
M Lib/test/pickletester.py
M Lib/test/test_bytes.py
M Lib/test/test_deque.py
M Lib/test/test_descrtut.py
M Lib/test/test_ordered_dict.py
M Lib/test/test_set.py
M Lib/test/test_weakset.py
M Lib/test/test_xml_etree.py
M Modules/_collectionsmodule.c
M Modules/_datetimemodule.c
M Objects/bytearrayobject.c
M Objects/clinic/typeobject.c.h
M Objects/odictobject.c
M Objects/setobject.c
M Objects/typeobject.c
diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index be48561ed10ac..a8ad5d437aaab 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -509,9 +509,8 @@ The following types can be pickled:
* classes that are defined at the top level of a module
-* instances of such classes whose :attr:`~object.__dict__` or the result of
- calling :meth:`__getstate__` is picklable (see section :ref:`pickle-inst` for
- details).
+* instances of such classes whose the result of calling :meth:`__getstate__`
+ is picklable (see section :ref:`pickle-inst` for details).
Attempts to pickle unpicklable objects will raise the :exc:`PicklingError`
exception; when this happens, an unspecified number of bytes may have already
@@ -611,11 +610,31 @@ methods:
.. method:: object.__getstate__()
- Classes can further influence how their instances are pickled; if the class
- defines the method :meth:`__getstate__`, it is called and the returned object
- is pickled as the contents for the instance, instead of the contents of the
- instance's dictionary. If the :meth:`__getstate__` method is absent, the
- instance's :attr:`~object.__dict__` is pickled as usual.
+ Classes can further influence how their instances are pickled by overriding
+ the method :meth:`__getstate__`. It is called and the returned object
+ is pickled as the contents for the instance, instead of a default state.
+ There are several cases:
+
+ * For a class that has no instance :attr:`~object.__dict__` and no
+ :attr:`~object.__slots__`, the default state is ``None``.
+
+ * For a class that has an instance :attr:`~object.__dict__` and no
+ :attr:`~object.__slots__`, the default state is ``self.__dict__``.
+
+ * For a class that has an instance :attr:`~object.__dict__` and
+ :attr:`~object.__slots__`, the default state is a tuple consisting of two
+ dictionaries: ``self.__dict__``, and a dictionary mapping slot
+ names to slot values. Only slots that have a value are
+ included in the latter.
+
+ * For a class that has :attr:`~object.__slots__` and no instance
+ :attr:`~object.__dict__`, the default state is a tuple whose first item
+ is ``None`` and whose second item is a dictionary mapping slot names
+ to slot values described in the previous bullet.
+
+ .. versionchanged:: 3.11
+ Added the default implementation of the ``__getstate__()`` method in the
+ :class:`object` class.
.. method:: object.__setstate__(state)
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 96902cf61b46d..4c9b32d9a9452 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -187,6 +187,15 @@ Other Language Changes
protocols correspondingly.
(Contributed by Serhiy Storchaka in :issue:`12022`.)
+* Added :meth:`object.__getstate__` which provides the default
+ implementation of the ``__getstate__()`` method. :mod:`Copying <copy>`
+ and :mod:`pickling <pickle>` instances of subclasses of builtin types
+ :class:`bytearray`, :class:`set`, :class:`frozenset`,
+ :class:`collections.OrderedDict`, :class:`collections.deque`,
+ :class:`weakref.WeakSet`, and :class:`datetime.tzinfo` now copies and
+ pickles instance attributes implemented as :term:`slots <__slots__>`.
+ (Contributed by Serhiy Storchaka in :issue:`26579`.)
+
Other CPython Implementation Changes
====================================
diff --git a/Include/object.h b/Include/object.h
index 317515d2c8151..0b4b55ea1ded9 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -299,6 +299,11 @@ PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *);
*/
PyAPI_FUNC(PyObject *) PyObject_Dir(PyObject *);
+/* Pickle support. */
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyObject_GetState(PyObject *);
+#endif
+
/* Helpers for printing recursive container types */
PyAPI_FUNC(int) Py_ReprEnter(PyObject *);
diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py
index 2a27684324d80..489eec714e0d4 100644
--- a/Lib/_weakrefset.py
+++ b/Lib/_weakrefset.py
@@ -80,8 +80,7 @@ def __contains__(self, item):
return wr in self.data
def __reduce__(self):
- return (self.__class__, (list(self),),
- getattr(self, '__dict__', None))
+ return self.__class__, (list(self),), self.__getstate__()
def add(self, item):
if self._pending_removals:
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py
index 264435cb41b2f..7af8dcd526df8 100644
--- a/Lib/collections/__init__.py
+++ b/Lib/collections/__init__.py
@@ -271,10 +271,22 @@ def __repr__(self):
def __reduce__(self):
'Return state information for pickling'
- inst_dict = vars(self).copy()
- for k in vars(OrderedDict()):
- inst_dict.pop(k, None)
- return self.__class__, (), inst_dict or None, None, iter(self.items())
+ state = self.__getstate__()
+ if state:
+ if isinstance(state, tuple):
+ state, slots = state
+ else:
+ slots = {}
+ state = state.copy()
+ slots = slots.copy()
+ for k in vars(OrderedDict()):
+ state.pop(k, None)
+ slots.pop(k, None)
+ if slots:
+ state = state, slots
+ else:
+ state = state or None
+ return self.__class__, (), state, None, iter(self.items())
def copy(self):
'od.copy() -> a shallow copy of od'
diff --git a/Lib/copyreg.py b/Lib/copyreg.py
index 356db6f083e39..c8a52a2dc63a2 100644
--- a/Lib/copyreg.py
+++ b/Lib/copyreg.py
@@ -89,6 +89,10 @@ def _reduce_ex(self, proto):
except AttributeError:
dict = None
else:
+ if (type(self).__getstate__ is object.__getstate__ and
+ getattr(self, "__slots__", None)):
+ raise TypeError("a class that defines __slots__ without "
+ "defining __getstate__ cannot be pickled")
dict = getstate()
if dict:
return _reconstructor, args, dict
diff --git a/Lib/datetime.py b/Lib/datetime.py
index 6bf37ccfab7ac..260b1de38877a 100644
--- a/Lib/datetime.py
+++ b/Lib/datetime.py
@@ -1169,15 +1169,7 @@ def __reduce__(self):
args = getinitargs()
else:
args = ()
- getstate = getattr(self, "__getstate__", None)
- if getstate:
- state = getstate()
- else:
- state = getattr(self, "__dict__", None) or None
- if state is None:
- return (self.__class__, args)
- else:
- return (self.__class__, args, state)
+ return (self.__class__, args, self.__getstate__())
class IsoCalendarDate(tuple):
diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py
index b590d69e8b744..543141dc427eb 100644
--- a/Lib/email/headerregistry.py
+++ b/Lib/email/headerregistry.py
@@ -218,7 +218,7 @@ def __reduce__(self):
self.__class__.__bases__,
str(self),
),
- self.__dict__)
+ self.__getstate__())
@classmethod
def _reconstruct(cls, value):
diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py
index efea40d3f4ff1..335cded3b5fac 100644
--- a/Lib/test/datetimetester.py
+++ b/Lib/test/datetimetester.py
@@ -139,8 +139,8 @@ class PicklableFixedOffset(FixedOffset):
def __init__(self, offset=None, name=None, dstoffset=None):
FixedOffset.__init__(self, offset, name, dstoffset)
- def __getstate__(self):
- return self.__dict__
+class PicklableFixedOffsetWithSlots(PicklableFixedOffset):
+ __slots__ = '_FixedOffset__offset', '_FixedOffset__name', 'spam'
class _TZInfo(tzinfo):
def utcoffset(self, datetime_module):
@@ -202,6 +202,7 @@ def test_pickling_subclass(self):
offset = timedelta(minutes=-300)
for otype, args in [
(PicklableFixedOffset, (offset, 'cookie')),
+ (PicklableFixedOffsetWithSlots, (offset, 'cookie')),
(timezone, (offset,)),
(timezone, (offset, "EST"))]:
orig = otype(*args)
@@ -217,6 +218,7 @@ def test_pickling_subclass(self):
self.assertIs(type(derived), otype)
self.assertEqual(derived.utcoffset(None), offset)
self.assertEqual(derived.tzname(None), oname)
+ self.assertFalse(hasattr(derived, 'spam'))
def test_issue23600(self):
DSTDIFF = DSTOFFSET = timedelta(hours=1)
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index f13d42f664880..63fa7604fbfc9 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -2382,9 +2382,11 @@ def test_reduce_calls_base(self):
def test_bad_getattr(self):
# Issue #3514: crash when there is an infinite loop in __getattr__
x = BadGetattr()
- for proto in protocols:
+ for proto in range(2):
with support.infinite_recursion():
self.assertRaises(RuntimeError, self.dumps, x, proto)
+ for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
+ s = self.dumps(x, proto)
def test_reduce_bad_iterator(self):
# Issue4176: crash when 4th and 5th items of __reduce__()
diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py
index fd8e1d4bdc44c..b457ff6ca849c 100644
--- a/Lib/test/test_bytes.py
+++ b/Lib/test/test_bytes.py
@@ -1940,28 +1940,30 @@ def test_join(self):
def test_pickle(self):
a = self.type2test(b"abcd")
a.x = 10
- a.y = self.type2test(b"efgh")
+ a.z = self.type2test(b"efgh")
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
b = pickle.loads(pickle.dumps(a, proto))
self.assertNotEqual(id(a), id(b))
self.assertEqual(a, b)
self.assertEqual(a.x, b.x)
- self.assertEqual(a.y, b.y)
+ self.assertEqual(a.z, b.z)
self.assertEqual(type(a), type(b))
- self.assertEqual(type(a.y), type(b.y))
+ self.assertEqual(type(a.z), type(b.z))
+ self.assertFalse(hasattr(b, 'y'))
def test_copy(self):
a = self.type2test(b"abcd")
a.x = 10
- a.y = self.type2test(b"efgh")
+ a.z = self.type2test(b"efgh")
for copy_method in (copy.copy, copy.deepcopy):
b = copy_method(a)
self.assertNotEqual(id(a), id(b))
self.assertEqual(a, b)
self.assertEqual(a.x, b.x)
- self.assertEqual(a.y, b.y)
+ self.assertEqual(a.z, b.z)
self.assertEqual(type(a), type(b))
- self.assertEqual(type(a.y), type(b.y))
+ self.assertEqual(type(a.z), type(b.z))
+ self.assertFalse(hasattr(b, 'y'))
def test_fromhex(self):
b = self.type2test.fromhex('1a2B30')
@@ -1994,6 +1996,9 @@ def __init__(me, *args, **kwargs):
class ByteArraySubclass(bytearray):
pass
+class ByteArraySubclassWithSlots(bytearray):
+ __slots__ = ('x', 'y', '__dict__')
+
class BytesSubclass(bytes):
pass
@@ -2014,6 +2019,9 @@ def __init__(me, newarg=1, *args, **kwargs):
x = subclass(newarg=4, source=b"abcd")
self.assertEqual(x, b"abcd")
+class ByteArraySubclassWithSlotsTest(SubclassTest, unittest.TestCase):
+ basetype = bytearray
+ type2test = ByteArraySubclassWithSlots
class BytesSubclassTest(SubclassTest, unittest.TestCase):
basetype = bytes
diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py
index 0be3feca01049..ae1dfacd7262e 100644
--- a/Lib/test/test_deque.py
+++ b/Lib/test/test_deque.py
@@ -781,6 +781,9 @@ def test_runtime_error_on_empty_deque(self):
class Deque(deque):
pass
+class DequeWithSlots(deque):
+ __slots__ = ('x', 'y', '__dict__')
+
class DequeWithBadIter(deque):
def __iter__(self):
raise TypeError
@@ -810,40 +813,28 @@ def test_basics(self):
self.assertEqual(len(d), 0)
def test_copy_pickle(self):
-
- d = Deque('abc')
-
- e = d.__copy__()
- self.assertEqual(type(d), type(e))
- self.assertEqual(list(d), list(e))
-
- e = Deque(d)
- self.assertEqual(type(d), type(e))
- self.assertEqual(list(d), list(e))
-
- for proto in range(pickle.HIGHEST_PROTOCOL + 1):
- s = pickle.dumps(d, proto)
- e = pickle.loads(s)
- self.assertNotEqual(id(d), id(e))
- self.assertEqual(type(d), type(e))
- self.assertEqual(list(d), list(e))
-
- d = Deque('abcde', maxlen=4)
-
- e = d.__copy__()
- self.assertEqual(type(d), type(e))
- self.assertEqual(list(d), list(e))
-
- e = Deque(d)
- self.assertEqual(type(d), type(e))
- self.assertEqual(list(d), list(e))
-
- for proto in range(pickle.HIGHEST_PROTOCOL + 1):
- s = pickle.dumps(d, proto)
- e = pickle.loads(s)
- self.assertNotEqual(id(d), id(e))
- self.assertEqual(type(d), type(e))
- self.assertEqual(list(d), list(e))
+ for cls in Deque, DequeWithSlots:
+ for d in cls('abc'), cls('abcde', maxlen=4):
+ d.x = ['x']
+ d.z = ['z']
+
+ e = d.__copy__()
+ self.assertEqual(type(d), type(e))
+ self.assertEqual(list(d), list(e))
+
+ e = cls(d)
+ self.assertEqual(type(d), type(e))
+ self.assertEqual(list(d), list(e))
+
+ for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+ s = pickle.dumps(d, proto)
+ e = pickle.loads(s)
+ self.assertNotEqual(id(d), id(e))
+ self.assertEqual(type(d), type(e))
+ self.assertEqual(list(d), list(e))
+ self.assertEqual(e.x, d.x)
+ self.assertEqual(e.z, d.z)
+ self.assertFalse(hasattr(e, 'y'))
def test_pickle_recursive(self):
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py
index 2af683e707ca8..e01a31a74695d 100644
--- a/Lib/test/test_descrtut.py
+++ b/Lib/test/test_descrtut.py
@@ -181,6 +181,7 @@ def merge(self, other):
'__ge__',
'__getattribute__',
'__getitem__',
+ '__getstate__',
'__gt__',
'__hash__',
'__iadd__',
diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py
index d51296a2d1244..37447fd249b8c 100644
--- a/Lib/test/test_ordered_dict.py
+++ b/Lib/test/test_ordered_dict.py
@@ -287,6 +287,8 @@ def test_copying(self):
# and have a repr/eval round-trip
pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
od = OrderedDict(pairs)
+ od.x = ['x']
+ od.z = ['z']
def check(dup):
msg = "\ncopy: %s\nod: %s" % (dup, od)
self.assertIsNot(dup, od, msg)
@@ -295,13 +297,27 @@ def check(dup):
self.assertEqual(len(dup), len(od))
self.assertEqual(type(dup), type(od))
check(od.copy())
- check(copy.copy(od))
- check(copy.deepcopy(od))
+ dup = copy.copy(od)
+ check(dup)
+ self.assertIs(dup.x, od.x)
+ self.assertIs(dup.z, od.z)
+ self.assertFalse(hasattr(dup, 'y'))
+ dup = copy.deepcopy(od)
+ check(dup)
+ self.assertEqual(dup.x, od.x)
+ self.assertIsNot(dup.x, od.x)
+ self.assertEqual(dup.z, od.z)
+ self.assertIsNot(dup.z, od.z)
+ self.assertFalse(hasattr(dup, 'y'))
# pickle directly pulls the module, so we have to fake it
with replaced_module('collections', self.module):
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
with self.subTest(proto=proto):
- check(pickle.loads(pickle.dumps(od, proto)))
+ dup = pickle.loads(pickle.dumps(od, proto))
+ check(dup)
+ self.assertEqual(dup.x, od.x)
+ self.assertEqual(dup.z, od.z)
+ self.assertFalse(hasattr(dup, 'y'))
check(eval(repr(od)))
update_test = OrderedDict()
update_test.update(od)
@@ -846,6 +862,23 @@ class OrderedDict(c_coll.OrderedDict):
pass
+class PurePythonOrderedDictWithSlotsCopyingTests(unittest.TestCase):
+
+ module = py_coll
+ class OrderedDict(py_coll.OrderedDict):
+ __slots__ = ('x', 'y')
+ test_copying = OrderedDictTests.test_copying
+
+
+ at unittest.skipUnless(c_coll, 'requires the C version of the collections module')
+class CPythonOrderedDictWithSlotsCopyingTests(unittest.TestCase):
+
+ module = c_coll
+ class OrderedDict(c_coll.OrderedDict):
+ __slots__ = ('x', 'y')
+ test_copying = OrderedDictTests.test_copying
+
+
class PurePythonGeneralMappingTests(mapping_tests.BasicTestMappingProtocol):
@classmethod
diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py
index 03b911920e179..3b57517a86101 100644
--- a/Lib/test/test_set.py
+++ b/Lib/test/test_set.py
@@ -227,14 +227,17 @@ def test_sub_and_super(self):
def test_pickling(self):
for i in range(pickle.HIGHEST_PROTOCOL + 1):
+ if type(self.s) not in (set, frozenset):
+ self.s.x = ['x']
+ self.s.z = ['z']
p = pickle.dumps(self.s, i)
dup = pickle.loads(p)
self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup))
if type(self.s) not in (set, frozenset):
- self.s.x = 10
- p = pickle.dumps(self.s, i)
- dup = pickle.loads(p)
self.assertEqual(self.s.x, dup.x)
+ self.assertEqual(self.s.z, dup.z)
+ self.assertFalse(hasattr(self.s, 'y'))
+ del self.s.x, self.s.z
def test_iterator_pickling(self):
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
@@ -808,6 +811,21 @@ def test_singleton_empty_frozenset(self):
# All empty frozenset subclass instances should have different ids
self.assertEqual(len(set(map(id, efs))), len(efs))
+
+class SetSubclassWithSlots(set):
+ __slots__ = ('x', 'y', '__dict__')
+
+class TestSetSubclassWithSlots(unittest.TestCase):
+ thetype = SetSubclassWithSlots
+ setUp = TestJointOps.setUp
+ test_pickling = TestJointOps.test_pickling
+
+class FrozenSetSubclassWithSlots(frozenset):
+ __slots__ = ('x', 'y', '__dict__')
+
+class TestFrozenSetSubclassWithSlots(TestSetSubclassWithSlots):
+ thetype = FrozenSetSubclassWithSlots
+
# Tests taken from test_sets.py =============================================
empty_set = set()
diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py
index 9b31d5fce3472..d6c88596cff71 100644
--- a/Lib/test/test_weakset.py
+++ b/Lib/test/test_weakset.py
@@ -1,5 +1,6 @@
import unittest
from weakref import WeakSet
+import copy
import string
from collections import UserString as ustr
from collections.abc import Set, MutableSet
@@ -15,6 +16,12 @@ class RefCycle:
def __init__(self):
self.cycle = self
+class WeakSetSubclass(WeakSet):
+ pass
+
+class WeakSetWithSlots(WeakSet):
+ __slots__ = ('x', 'y')
+
class TestWeakSet(unittest.TestCase):
@@ -447,6 +454,30 @@ def test_abc(self):
self.assertIsInstance(self.s, Set)
self.assertIsInstance(self.s, MutableSet)
+ def test_copying(self):
+ for cls in WeakSet, WeakSetWithSlots:
+ s = cls(self.items)
+ s.x = ['x']
+ s.z = ['z']
+
+ dup = copy.copy(s)
+ self.assertIsInstance(dup, cls)
+ self.assertEqual(dup, s)
+ self.assertIsNot(dup, s)
+ self.assertIs(dup.x, s.x)
+ self.assertIs(dup.z, s.z)
+ self.assertFalse(hasattr(dup, 'y'))
+
+ dup = copy.deepcopy(s)
+ self.assertIsInstance(dup, cls)
+ self.assertEqual(dup, s)
+ self.assertIsNot(dup, s)
+ self.assertEqual(dup.x, s.x)
+ self.assertIsNot(dup.x, s.x)
+ self.assertEqual(dup.z, s.z)
+ self.assertIsNot(dup.z, s.z)
+ self.assertFalse(hasattr(dup, 'y'))
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py
index d2bdc4f7f0444..60a41506d8795 100644
--- a/Lib/test/test_xml_etree.py
+++ b/Lib/test/test_xml_etree.py
@@ -2524,8 +2524,7 @@ def test_pickle_issue18997(self):
<group><dogs>4</dogs>
</group>"""
e1 = dumper.fromstring(XMLTEXT)
- if hasattr(e1, '__getstate__'):
- self.assertEqual(e1.__getstate__()['tag'], 'group')
+ self.assertEqual(e1.__getstate__()['tag'], 'group')
e2 = self.pickleRoundTrip(e1, 'xml.etree.ElementTree',
dumper, loader, proto)
self.assertEqual(e2.tag, 'group')
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst b/Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst
new file mode 100644
index 0000000000000..9afd1bfa50084
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-07-23-11-28-45.bpo-26579.lpCY8R.rst
@@ -0,0 +1,7 @@
+Added ``object.__getstate__`` which provides the default implementation of
+the ``__getstate__()`` method.
+
+Copying and pickling instances of subclasses of builtin types bytearray,
+set, frozenset, collections.OrderedDict, collections.deque, weakref.WeakSet,
+and datetime.tzinfo now copies and pickles instance attributes implemented as
+slots.
diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c
index b47f977731758..f78e2613fc243 100644
--- a/Modules/_collectionsmodule.c
+++ b/Modules/_collectionsmodule.c
@@ -1347,27 +1347,24 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg)
static PyObject *
deque_reduce(dequeobject *deque, PyObject *Py_UNUSED(ignored))
{
- PyObject *dict, *it;
+ PyObject *state, *it;
- if (_PyObject_LookupAttr((PyObject *)deque, &_Py_ID(__dict__), &dict) < 0) {
+ state = _PyObject_GetState((PyObject *)deque);
+ if (state == NULL) {
return NULL;
}
- if (dict == NULL) {
- dict = Py_None;
- Py_INCREF(dict);
- }
it = PyObject_GetIter((PyObject *)deque);
if (it == NULL) {
- Py_DECREF(dict);
+ Py_DECREF(state);
return NULL;
}
if (deque->maxlen < 0) {
- return Py_BuildValue("O()NN", Py_TYPE(deque), dict, it);
+ return Py_BuildValue("O()NN", Py_TYPE(deque), state, it);
}
else {
- return Py_BuildValue("O(()n)NN", Py_TYPE(deque), deque->maxlen, dict, it);
+ return Py_BuildValue("O(()n)NN", Py_TYPE(deque), deque->maxlen, state, it);
}
}
diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index ae97190bccbde..fc766c3c94360 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -3736,9 +3736,8 @@ static PyObject *
tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
{
PyObject *args, *state;
- PyObject *getinitargs, *getstate;
+ PyObject *getinitargs;
_Py_IDENTIFIER(__getinitargs__);
- _Py_IDENTIFIER(__getstate__);
if (_PyObject_LookupAttrId(self, &PyId___getinitargs__, &getinitargs) < 0) {
return NULL;
@@ -3754,34 +3753,13 @@ tzinfo_reduce(PyObject *self, PyObject *Py_UNUSED(ignored))
return NULL;
}
- if (_PyObject_LookupAttrId(self, &PyId___getstate__, &getstate) < 0) {
+ state = _PyObject_GetState(self);
+ if (state == NULL) {
Py_DECREF(args);
return NULL;
}
- if (getstate != NULL) {
- state = PyObject_CallNoArgs(getstate);
- Py_DECREF(getstate);
- if (state == NULL) {
- Py_DECREF(args);
- return NULL;
- }
- }
- else {
- PyObject **dictptr;
- state = Py_None;
- dictptr = _PyObject_GetDictPtr(self);
- if (dictptr && *dictptr && PyDict_GET_SIZE(*dictptr)) {
- state = *dictptr;
- }
- Py_INCREF(state);
- }
- if (state == Py_None) {
- Py_DECREF(state);
- return Py_BuildValue("(ON)", Py_TYPE(self), args);
- }
- else
- return Py_BuildValue("(ONN)", Py_TYPE(self), args, state);
+ return Py_BuildValue("(ONN)", Py_TYPE(self), args, state);
}
static PyMethodDef tzinfo_methods[] = {
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 3849337fbf77f..f784e0448191f 100644
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -2122,35 +2122,26 @@ bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep)
static PyObject *
_common_reduce(PyByteArrayObject *self, int proto)
{
- PyObject *dict;
- char *buf;
+ PyObject *state;
+ const char *buf;
- if (_PyObject_LookupAttr((PyObject *)self, &_Py_ID(__dict__), &dict) < 0) {
+ state = _PyObject_GetState((PyObject *)self);
+ if (state == NULL) {
return NULL;
}
- if (dict == NULL) {
- dict = Py_None;
- Py_INCREF(dict);
- }
+ if (!Py_SIZE(self)) {
+ return Py_BuildValue("(O()N)", Py_TYPE(self), state);
+ }
buf = PyByteArray_AS_STRING(self);
if (proto < 3) {
/* use str based reduction for backwards compatibility with Python 2.x */
- PyObject *latin1;
- if (Py_SIZE(self))
- latin1 = PyUnicode_DecodeLatin1(buf, Py_SIZE(self), NULL);
- else
- latin1 = PyUnicode_FromString("");
- return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict);
+ PyObject *latin1 = PyUnicode_DecodeLatin1(buf, Py_SIZE(self), NULL);
+ return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", state);
}
else {
/* use more efficient byte based reduction */
- if (Py_SIZE(self)) {
- return Py_BuildValue("(O(y#)N)", Py_TYPE(self), buf, Py_SIZE(self), dict);
- }
- else {
- return Py_BuildValue("(O()N)", Py_TYPE(self), dict);
- }
+ return Py_BuildValue("(O(y#)N)", Py_TYPE(self), buf, Py_SIZE(self), state);
}
}
diff --git a/Objects/clinic/typeobject.c.h b/Objects/clinic/typeobject.c.h
index 8c70d76d916db..dee3139bd3d82 100644
--- a/Objects/clinic/typeobject.c.h
+++ b/Objects/clinic/typeobject.c.h
@@ -130,6 +130,24 @@ type___sizeof__(PyTypeObject *self, PyObject *Py_UNUSED(ignored))
return type___sizeof___impl(self);
}
+PyDoc_STRVAR(object___getstate____doc__,
+"__getstate__($self, /)\n"
+"--\n"
+"\n"
+"Helper for pickle.");
+
+#define OBJECT___GETSTATE___METHODDEF \
+ {"__getstate__", (PyCFunction)object___getstate__, METH_NOARGS, object___getstate____doc__},
+
+static PyObject *
+object___getstate___impl(PyObject *self);
+
+static PyObject *
+object___getstate__(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+ return object___getstate___impl(self);
+}
+
PyDoc_STRVAR(object___reduce____doc__,
"__reduce__($self, /)\n"
"--\n"
@@ -243,4 +261,4 @@ object___dir__(PyObject *self, PyObject *Py_UNUSED(ignored))
{
return object___dir___impl(self);
}
-/*[clinic end generated code: output=b4fb62939b08baf9 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=a30090032b8e6195 input=a9049054013a1b77]*/
diff --git a/Objects/odictobject.c b/Objects/odictobject.c
index c207593ab79f7..f5b8b3e6cdd76 100644
--- a/Objects/odictobject.c
+++ b/Objects/odictobject.c
@@ -947,23 +947,13 @@ PyDoc_STRVAR(odict_reduce__doc__, "Return state information for pickling");
static PyObject *
odict_reduce(register PyODictObject *od, PyObject *Py_UNUSED(ignored))
{
- PyObject *dict = NULL, *result = NULL;
+ PyObject *state, *result = NULL;
PyObject *items_iter, *items, *args = NULL;
/* capture any instance state */
- dict = PyObject_GetAttr((PyObject *)od, &_Py_ID(__dict__));
- if (dict == NULL)
+ state = _PyObject_GetState((PyObject *)od);
+ if (state == NULL)
goto Done;
- else {
- /* od.__dict__ isn't necessarily a dict... */
- Py_ssize_t dict_len = PyObject_Length(dict);
- if (dict_len == -1)
- goto Done;
- if (!dict_len) {
- /* nothing to pickle in od.__dict__ */
- Py_CLEAR(dict);
- }
- }
/* build the result */
args = PyTuple_New(0);
@@ -979,11 +969,11 @@ odict_reduce(register PyODictObject *od, PyObject *Py_UNUSED(ignored))
if (items_iter == NULL)
goto Done;
- result = PyTuple_Pack(5, Py_TYPE(od), args, dict ? dict : Py_None, Py_None, items_iter);
+ result = PyTuple_Pack(5, Py_TYPE(od), args, state, Py_None, items_iter);
Py_DECREF(items_iter);
Done:
- Py_XDECREF(dict);
+ Py_XDECREF(state);
Py_XDECREF(args);
return result;
diff --git a/Objects/setobject.c b/Objects/setobject.c
index ef2190de914b3..4b6a8b8dfb679 100644
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -1947,7 +1947,7 @@ an exception when an element is missing from the set.");
static PyObject *
set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored))
{
- PyObject *keys=NULL, *args=NULL, *result=NULL, *dict=NULL;
+ PyObject *keys=NULL, *args=NULL, *result=NULL, *state=NULL;
keys = PySequence_List((PyObject *)so);
if (keys == NULL)
@@ -1955,18 +1955,14 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored))
args = PyTuple_Pack(1, keys);
if (args == NULL)
goto done;
- if (_PyObject_LookupAttr((PyObject *)so, &_Py_ID(__dict__), &dict) < 0) {
+ state = _PyObject_GetState((PyObject *)so);
+ if (state == NULL)
goto done;
- }
- if (dict == NULL) {
- dict = Py_None;
- Py_INCREF(dict);
- }
- result = PyTuple_Pack(3, Py_TYPE(so), args, dict);
+ result = PyTuple_Pack(3, Py_TYPE(so), args, state);
done:
Py_XDECREF(args);
Py_XDECREF(keys);
- Py_XDECREF(dict);
+ Py_XDECREF(state);
return result;
}
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 5de8c3d2ece43..53e4f0781d65b 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -4960,143 +4960,175 @@ _PyType_GetSlotNames(PyTypeObject *cls)
}
static PyObject *
-_PyObject_GetState(PyObject *obj, int required)
+object_getstate_default(PyObject *obj, int required)
{
PyObject *state;
- PyObject *getstate;
+ PyObject *slotnames;
- if (_PyObject_LookupAttr(obj, &_Py_ID(__getstate__), &getstate) < 0) {
+ if (required && Py_TYPE(obj)->tp_itemsize) {
+ PyErr_Format(PyExc_TypeError,
+ "cannot pickle %.200s objects",
+ Py_TYPE(obj)->tp_name);
+ return NULL;
+ }
+
+ if (_PyObject_IsInstanceDictEmpty(obj)) {
+ state = Py_None;
+ Py_INCREF(state);
+ }
+ else {
+ state = PyObject_GenericGetDict(obj, NULL);
+ if (state == NULL) {
+ return NULL;
+ }
+ }
+
+ slotnames = _PyType_GetSlotNames(Py_TYPE(obj));
+ if (slotnames == NULL) {
+ Py_DECREF(state);
return NULL;
}
- if (getstate == NULL) {
- PyObject *slotnames;
- if (required && Py_TYPE(obj)->tp_itemsize) {
+ assert(slotnames == Py_None || PyList_Check(slotnames));
+ if (required) {
+ Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize;
+ if (Py_TYPE(obj)->tp_dictoffset &&
+ (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0)
+ {
+ basicsize += sizeof(PyObject *);
+ }
+ if (Py_TYPE(obj)->tp_weaklistoffset) {
+ basicsize += sizeof(PyObject *);
+ }
+ if (slotnames != Py_None) {
+ basicsize += sizeof(PyObject *) * PyList_GET_SIZE(slotnames);
+ }
+ if (Py_TYPE(obj)->tp_basicsize > basicsize) {
+ Py_DECREF(slotnames);
+ Py_DECREF(state);
PyErr_Format(PyExc_TypeError,
"cannot pickle '%.200s' object",
Py_TYPE(obj)->tp_name);
return NULL;
}
- if (_PyObject_IsInstanceDictEmpty(obj)) {
- state = Py_None;
- Py_INCREF(state);
- }
- else {
- state = PyObject_GenericGetDict(obj, NULL);
- if (state == NULL) {
- return NULL;
- }
- }
+ }
- slotnames = _PyType_GetSlotNames(Py_TYPE(obj));
- if (slotnames == NULL) {
+ if (slotnames != Py_None && PyList_GET_SIZE(slotnames) > 0) {
+ PyObject *slots;
+ Py_ssize_t slotnames_size, i;
+
+ slots = PyDict_New();
+ if (slots == NULL) {
+ Py_DECREF(slotnames);
Py_DECREF(state);
return NULL;
}
- assert(slotnames == Py_None || PyList_Check(slotnames));
- if (required) {
- Py_ssize_t basicsize = PyBaseObject_Type.tp_basicsize;
- if (Py_TYPE(obj)->tp_dictoffset &&
- (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0)
- {
- basicsize += sizeof(PyObject *);
+ slotnames_size = PyList_GET_SIZE(slotnames);
+ for (i = 0; i < slotnames_size; i++) {
+ PyObject *name, *value;
+
+ name = PyList_GET_ITEM(slotnames, i);
+ Py_INCREF(name);
+ value = PyObject_GetAttr(obj, name);
+ if (_PyObject_LookupAttr(obj, name, &value) < 0) {
+ Py_DECREF(name);
+ goto error;
+ }
+ if (value == NULL) {
+ Py_DECREF(name);
+ /* It is not an error if the attribute is not present. */
}
- if (Py_TYPE(obj)->tp_weaklistoffset) {
- basicsize += sizeof(PyObject *);
+ else {
+ int err = PyDict_SetItem(slots, name, value);
+ Py_DECREF(name);
+ Py_DECREF(value);
+ if (err) {
+ goto error;
+ }
}
- if (slotnames != Py_None) {
- basicsize += sizeof(PyObject *) * PyList_GET_SIZE(slotnames);
+
+ /* The list is stored on the class so it may mutate while we
+ iterate over it */
+ if (slotnames_size != PyList_GET_SIZE(slotnames)) {
+ PyErr_Format(PyExc_RuntimeError,
+ "__slotsname__ changed size during iteration");
+ goto error;
}
- if (Py_TYPE(obj)->tp_basicsize > basicsize) {
+
+ /* We handle errors within the loop here. */
+ if (0) {
+ error:
Py_DECREF(slotnames);
+ Py_DECREF(slots);
Py_DECREF(state);
- PyErr_Format(PyExc_TypeError,
- "cannot pickle '%.200s' object",
- Py_TYPE(obj)->tp_name);
return NULL;
}
}
- if (slotnames != Py_None && PyList_GET_SIZE(slotnames) > 0) {
- PyObject *slots;
- Py_ssize_t slotnames_size, i;
+ /* If we found some slot attributes, pack them in a tuple along
+ the original attribute dictionary. */
+ if (PyDict_GET_SIZE(slots) > 0) {
+ PyObject *state2;
- slots = PyDict_New();
- if (slots == NULL) {
+ state2 = PyTuple_Pack(2, state, slots);
+ Py_DECREF(state);
+ if (state2 == NULL) {
Py_DECREF(slotnames);
- Py_DECREF(state);
+ Py_DECREF(slots);
return NULL;
}
+ state = state2;
+ }
+ Py_DECREF(slots);
+ }
+ Py_DECREF(slotnames);
- slotnames_size = PyList_GET_SIZE(slotnames);
- for (i = 0; i < slotnames_size; i++) {
- PyObject *name, *value;
-
- name = PyList_GET_ITEM(slotnames, i);
- Py_INCREF(name);
- if (_PyObject_LookupAttr(obj, name, &value) < 0) {
- goto error;
- }
- if (value == NULL) {
- Py_DECREF(name);
- /* It is not an error if the attribute is not present. */
- }
- else {
- int err = PyDict_SetItem(slots, name, value);
- Py_DECREF(name);
- Py_DECREF(value);
- if (err) {
- goto error;
- }
- }
-
- /* The list is stored on the class so it may mutate while we
- iterate over it */
- if (slotnames_size != PyList_GET_SIZE(slotnames)) {
- PyErr_Format(PyExc_RuntimeError,
- "__slotsname__ changed size during iteration");
- goto error;
- }
-
- /* We handle errors within the loop here. */
- if (0) {
- error:
- Py_DECREF(slotnames);
- Py_DECREF(slots);
- Py_DECREF(state);
- return NULL;
- }
- }
+ return state;
+}
- /* If we found some slot attributes, pack them in a tuple along
- the original attribute dictionary. */
- if (PyDict_GET_SIZE(slots) > 0) {
- PyObject *state2;
+static PyObject *
+object_getstate(PyObject *obj, int required)
+{
+ PyObject *getstate, *state;
- state2 = PyTuple_Pack(2, state, slots);
- Py_DECREF(state);
- if (state2 == NULL) {
- Py_DECREF(slotnames);
- Py_DECREF(slots);
- return NULL;
- }
- state = state2;
- }
- Py_DECREF(slots);
- }
- Py_DECREF(slotnames);
+ getstate = PyObject_GetAttr(obj, &_Py_ID(__getstate__));
+ if (getstate == NULL) {
+ return NULL;
}
- else { /* getstate != NULL */
+ if (PyCFunction_Check(getstate) &&
+ PyCFunction_GET_SELF(getstate) == obj &&
+ PyCFunction_GET_FUNCTION(getstate) == object___getstate__)
+ {
+ /* If __getstate__ is not overriden pass the required argument. */
+ state = object_getstate_default(obj, required);
+ }
+ else {
state = _PyObject_CallNoArgs(getstate);
- Py_DECREF(getstate);
- if (state == NULL)
- return NULL;
}
-
+ Py_DECREF(getstate);
return state;
}
+PyObject *
+_PyObject_GetState(PyObject *obj)
+{
+ return object_getstate(obj, 0);
+}
+
+/*[clinic input]
+object.__getstate__
+
+Helper for pickle.
+[clinic start generated code]*/
+
+static PyObject *
+object___getstate___impl(PyObject *self)
+/*[clinic end generated code: output=5a2500dcb6217e9e input=692314d8fbe194ee]*/
+{
+ return object_getstate_default(self, 0);
+}
+
static int
_PyObject_GetNewArguments(PyObject *obj, PyObject **args, PyObject **kwargs)
{
@@ -5309,8 +5341,7 @@ reduce_newobj(PyObject *obj)
return NULL;
}
- state = _PyObject_GetState(obj,
- !hasargs && !PyList_Check(obj) && !PyDict_Check(obj));
+ state = object_getstate(obj, !(hasargs || PyList_Check(obj) || PyDict_Check(obj)));
if (state == NULL) {
Py_DECREF(newobj);
Py_DECREF(newargs);
@@ -5558,6 +5589,7 @@ object___dir___impl(PyObject *self)
static PyMethodDef object_methods[] = {
OBJECT___REDUCE_EX___METHODDEF
OBJECT___REDUCE___METHODDEF
+ OBJECT___GETSTATE___METHODDEF
{"__subclasshook__", object_subclasshook, METH_CLASS | METH_VARARGS,
object_subclasshook_doc},
{"__init_subclass__", object_init_subclass, METH_CLASS | METH_NOARGS,
More information about the Python-checkins
mailing list