[Python-checkins] cpython: Issue #24164: Objects that need calling ``__new__`` with keyword arguments,
serhiy.storchaka
python-checkins at python.org
Sat Oct 10 15:42:59 EDT 2015
https://hg.python.org/cpython/rev/bc5894a3a0e6
changeset: 98658:bc5894a3a0e6
user: Serhiy Storchaka <storchaka at gmail.com>
date: Sat Oct 10 22:42:18 2015 +0300
summary:
Issue #24164: Objects that need calling ``__new__`` with keyword arguments,
can now be pickled using pickle protocols older than protocol version 4.
files:
Doc/whatsnew/3.6.rst | 9 +++
Lib/pickle.py | 17 ++++-
Lib/test/pickletester.py | 6 +-
Misc/NEWS | 3 +
Modules/_pickle.c | 78 ++++++++++++++++++++++++---
Objects/typeobject.c | 16 +----
6 files changed, 97 insertions(+), 32 deletions(-)
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -112,6 +112,15 @@
(Contributed by Joe Jevnik in :issue:`24379`.)
+pickle
+------
+
+Objects that need calling ``__new__`` with keyword arguments, can now be pickled
+using :ref:`pickle protocols <pickle-protocols>` older than protocol version 4.
+Protocol version 4 already supports this case. (Contributed by Serhiy
+Storchaka in :issue:`24164`.)
+
+
rlcomplete
----------
diff --git a/Lib/pickle.py b/Lib/pickle.py
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -27,6 +27,7 @@
from copyreg import dispatch_table
from copyreg import _extension_registry, _inverted_registry, _extension_cache
from itertools import islice
+from functools import partial
import sys
from sys import maxsize
from struct import pack, unpack
@@ -544,7 +545,7 @@
write = self.write
func_name = getattr(func, "__name__", "")
- if self.proto >= 4 and func_name == "__newobj_ex__":
+ if self.proto >= 2 and func_name == "__newobj_ex__":
cls, args, kwargs = args
if not hasattr(cls, "__new__"):
raise PicklingError("args[0] from {} args has no __new__"
@@ -552,10 +553,16 @@
if obj is not None and cls is not obj.__class__:
raise PicklingError("args[0] from {} args has the wrong class"
.format(func_name))
- save(cls)
- save(args)
- save(kwargs)
- write(NEWOBJ_EX)
+ if self.proto >= 4:
+ save(cls)
+ save(args)
+ save(kwargs)
+ write(NEWOBJ_EX)
+ else:
+ func = partial(cls.__new__, cls, *args, **kwargs)
+ save(func)
+ save(())
+ write(REDUCE)
elif self.proto >= 2 and func_name == "__newobj__":
# A __reduce__ implementation can direct protocol 2 or newer to
# use the more efficient NEWOBJ opcode, while still
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1580,16 +1580,14 @@
x.abc = 666
for proto in protocols:
with self.subTest(proto=proto):
- if 2 <= proto < 4:
- self.assertRaises(ValueError, self.dumps, x, proto)
- continue
s = self.dumps(x, proto)
if proto < 1:
self.assertIn(b'\nL64206', s) # LONG
elif proto < 2:
self.assertIn(b'M\xce\xfa', s) # BININT2
+ elif proto < 4:
+ self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE
else:
- assert proto >= 4
self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE
self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s))
self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s),
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -51,6 +51,9 @@
Library
-------
+- Issue #24164: Objects that need calling ``__new__`` with keyword arguments,
+ can now be pickled using pickle protocols older than protocol version 4.
+
- Issue #25364: zipfile now works in threads disabled builds.
- Issue #25328: smtpd's SMTPChannel now correctly raises a ValueError if both
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -153,6 +153,9 @@
PyObject *codecs_encode;
/* builtins.getattr, used for saving nested names with protocol < 4 */
PyObject *getattr;
+ /* functools.partial, used for implementing __newobj_ex__ with protocols
+ 2 and 3 */
+ PyObject *partial;
} PickleState;
/* Forward declaration of the _pickle module definition. */
@@ -200,6 +203,7 @@
PyObject *copyreg = NULL;
PyObject *compat_pickle = NULL;
PyObject *codecs = NULL;
+ PyObject *functools = NULL;
builtins = PyEval_GetBuiltins();
if (builtins == NULL)
@@ -314,12 +318,21 @@
}
Py_CLEAR(codecs);
+ functools = PyImport_ImportModule("functools");
+ if (!functools)
+ goto error;
+ st->partial = PyObject_GetAttrString(functools, "partial");
+ if (!st->partial)
+ goto error;
+ Py_CLEAR(functools);
+
return 0;
error:
Py_CLEAR(copyreg);
Py_CLEAR(compat_pickle);
Py_CLEAR(codecs);
+ Py_CLEAR(functools);
_Pickle_ClearState(st);
return -1;
}
@@ -3533,11 +3546,9 @@
PyErr_Clear();
}
else if (PyUnicode_Check(name)) {
- if (self->proto >= 4) {
- _Py_IDENTIFIER(__newobj_ex__);
- use_newobj_ex = PyUnicode_Compare(
- name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0;
- }
+ _Py_IDENTIFIER(__newobj_ex__);
+ use_newobj_ex = PyUnicode_Compare(
+ name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0;
if (!use_newobj_ex) {
_Py_IDENTIFIER(__newobj__);
use_newobj = PyUnicode_Compare(
@@ -3581,11 +3592,58 @@
return -1;
}
- if (save(self, cls, 0) < 0 ||
- save(self, args, 0) < 0 ||
- save(self, kwargs, 0) < 0 ||
- _Pickler_Write(self, &newobj_ex_op, 1) < 0) {
- return -1;
+ if (self->proto >= 4) {
+ if (save(self, cls, 0) < 0 ||
+ save(self, args, 0) < 0 ||
+ save(self, kwargs, 0) < 0 ||
+ _Pickler_Write(self, &newobj_ex_op, 1) < 0) {
+ return -1;
+ }
+ }
+ else {
+ PyObject *newargs;
+ PyObject *cls_new;
+ Py_ssize_t i;
+ _Py_IDENTIFIER(__new__);
+
+ newargs = PyTuple_New(Py_SIZE(args) + 2);
+ if (newargs == NULL)
+ return -1;
+
+ cls_new = _PyObject_GetAttrId(cls, &PyId___new__);
+ if (cls_new == NULL) {
+ Py_DECREF(newargs);
+ return -1;
+ }
+ PyTuple_SET_ITEM(newargs, 0, cls_new);
+ Py_INCREF(cls);
+ PyTuple_SET_ITEM(newargs, 1, cls);
+ for (i = 0; i < Py_SIZE(args); i++) {
+ PyObject *item = PyTuple_GET_ITEM(args, i);
+ Py_INCREF(item);
+ PyTuple_SET_ITEM(newargs, i + 2, item);
+ }
+
+ callable = PyObject_Call(st->partial, newargs, kwargs);
+ Py_DECREF(newargs);
+ if (callable == NULL)
+ return -1;
+
+ newargs = PyTuple_New(0);
+ if (newargs == NULL) {
+ Py_DECREF(callable);
+ return -1;
+ }
+
+ if (save(self, callable, 0) < 0 ||
+ save(self, newargs, 0) < 0 ||
+ _Pickler_Write(self, &reduce_op, 1) < 0) {
+ Py_DECREF(newargs);
+ Py_DECREF(callable);
+ return -1;
+ }
+ Py_DECREF(newargs);
+ Py_DECREF(callable);
}
}
else if (use_newobj) {
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -4101,7 +4101,7 @@
}
static PyObject *
-reduce_newobj(PyObject *obj, int proto)
+reduce_newobj(PyObject *obj)
{
PyObject *args = NULL, *kwargs = NULL;
PyObject *copyreg;
@@ -4153,7 +4153,7 @@
}
Py_DECREF(args);
}
- else if (proto >= 4) {
+ else {
_Py_IDENTIFIER(__newobj_ex__);
newobj = _PyObject_GetAttrId(copyreg, &PyId___newobj_ex__);
@@ -4171,16 +4171,6 @@
return NULL;
}
}
- else {
- PyErr_SetString(PyExc_ValueError,
- "must use protocol 4 or greater to copy this "
- "object; since __getnewargs_ex__ returned "
- "keyword arguments.");
- Py_DECREF(args);
- Py_DECREF(kwargs);
- Py_DECREF(copyreg);
- return NULL;
- }
state = _PyObject_GetState(obj);
if (state == NULL) {
@@ -4225,7 +4215,7 @@
PyObject *copyreg, *res;
if (proto >= 2)
- return reduce_newobj(self, proto);
+ return reduce_newobj(self);
copyreg = import_copyreg();
if (!copyreg)
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list