[Python-checkins] CVS: python/dist/src/Modules cPickle.c,2.65,2.66
Jeremy Hylton
jhylton@users.sourceforge.net
Mon, 15 Oct 2001 14:38:00 -0700
Update of /cvsroot/python/python/dist/src/Modules
In directory usw-pr-cvs1:/tmp/cvs-serv16510/Modules
Modified Files:
cPickle.c
Log Message:
Better fix for core dumps on recursive objects in fast mode.
Raise ValueError when an object contains an arbitrarily nested
reference to itself. (The previous fix just produced invalid
pickles.)
Solution is very much like Py_ReprEnter() and Py_ReprLeave():
fast_save_enter() and fast_save_leave() that tracks the fast_container
limit and keeps a fast_memo of objects currently being pickled.
The cost of the solution is moderately expensive for deeply nested
structures, but it still seems to be faster than normal pickling,
based on tests with deeply nested lists.
Once FAST_LIMIT is exceeded, the new code is about twice as slow as
fast-mode code that doesn't check for recursion. It's still twice as
fast as the normal pickling code. In the absence of deeply nested
structures, I couldn't measure a difference.
Index: cPickle.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Modules/cPickle.c,v
retrieving revision 2.65
retrieving revision 2.66
diff -C2 -d -r2.65 -r2.66
*** cPickle.c 2001/10/12 04:11:06 2.65
--- cPickle.c 2001/10/15 21:37:58 2.66
***************
*** 319,322 ****
--- 319,323 ----
PyObject *dispatch_table;
int fast_container; /* count nested container dumps */
+ PyObject *fast_memo;
} Picklerobject;
***************
*** 888,891 ****
--- 889,937 ----
static int
+ fast_save_enter(Picklerobject *self, PyObject *obj)
+ {
+ /* if fast_container < 0, we're doing an error exit. */
+ if (++self->fast_container >= FAST_LIMIT) {
+ PyObject *key = NULL;
+ if (self->fast_memo == NULL) {
+ self->fast_memo = PyDict_New();
+ if (self->fast_memo == NULL) {
+ self->fast_container = -1;
+ return 0;
+ }
+ }
+ key = PyLong_FromVoidPtr(obj);
+ if (key == NULL)
+ return 0;
+ if (PyDict_GetItem(self->fast_memo, key)) {
+ PyErr_Format(PyExc_ValueError,
+ "fast mode: can't pickle cyclic objects including object type %s at %p",
+ obj->ob_type->tp_name, obj);
+ self->fast_container = -1;
+ return 0;
+ }
+ if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {
+ self->fast_container = -1;
+ return 0;
+ }
+ }
+ return 1;
+ }
+
+ int
+ fast_save_leave(Picklerobject *self, PyObject *obj)
+ {
+ if (self->fast_container-- >= FAST_LIMIT) {
+ PyObject *key = PyLong_FromVoidPtr(obj);
+ if (key == NULL)
+ return 0;
+ if (PyDict_DelItem(self->fast_memo, key) < 0) {
+ return 0;
+ }
+ }
+ return 1;
+ }
+
+ static int
save_none(Picklerobject *self, PyObject *args) {
static char none = NONE;
***************
*** 1358,1370 ****
save_list(Picklerobject *self, PyObject *args) {
PyObject *element = 0;
! int s_len, len, i, using_appends, res = -1, unfast = 0;
char s[3];
static char append = APPEND, appends = APPENDS;
! if (self->fast && self->fast_container++ > FAST_LIMIT) {
! self->fast = 0;
! unfast = 1;
! }
if (self->bin) {
--- 1404,1414 ----
save_list(Picklerobject *self, PyObject *args) {
PyObject *element = 0;
! int s_len, len, i, using_appends, res = -1;
char s[3];
static char append = APPEND, appends = APPENDS;
! if (self->fast && !fast_save_enter(self, args))
! goto finally;
if (self->bin) {
***************
*** 1418,1426 ****
finally:
! if (self->fast || unfast) {
! self->fast_container--;
! if (unfast && self->fast_container < FAST_LIMIT)
! self->fast = 1;
! }
return res;
--- 1462,1467 ----
finally:
! if (self->fast && !fast_save_leave(self, args))
! res = -1;
return res;
***************
*** 1431,1443 ****
save_dict(Picklerobject *self, PyObject *args) {
PyObject *key = 0, *value = 0;
! int i, len, res = -1, using_setitems, unfast = 0;
char s[3];
static char setitem = SETITEM, setitems = SETITEMS;
! if (self->fast && self->fast_container++ > FAST_LIMIT) {
! self->fast = 0;
! unfast = 1;
! }
if (self->bin) {
--- 1472,1482 ----
save_dict(Picklerobject *self, PyObject *args) {
PyObject *key = 0, *value = 0;
! int i, len, res = -1, using_setitems;
char s[3];
static char setitem = SETITEM, setitems = SETITEMS;
! if (self->fast && !fast_save_enter(self, args))
! goto finally;
if (self->bin) {
***************
*** 1492,1501 ****
finally:
! if (self->fast || unfast) {
! self->fast_container--;
! if (unfast && self->fast_container < FAST_LIMIT)
! self->fast = 1;
! }
!
return res;
--- 1531,1536 ----
finally:
! if (self->fast && !fast_save_leave(self, args))
! res = -1;
return res;
***************
*** 1508,1519 ****
*getinitargs_func = 0, *getstate_func = 0, *class_args = 0;
char *module_str, *name_str;
! int module_size, name_size, res = -1, unfast = 0;
static char inst = INST, obj = OBJ, build = BUILD;
! if (self->fast && self->fast_container++ > FAST_LIMIT) {
! self->fast = 0;
! unfast = 1;
! }
if ((*self->write_func)(self, &MARKv, 1) < 0)
--- 1543,1552 ----
*getinitargs_func = 0, *getstate_func = 0, *class_args = 0;
char *module_str, *name_str;
! int module_size, name_size, res = -1;
static char inst = INST, obj = OBJ, build = BUILD;
! if (self->fast && !fast_save_enter(self, args))
! goto finally;
if ((*self->write_func)(self, &MARKv, 1) < 0)
***************
*** 1623,1631 ****
finally:
! if (self->fast || unfast) {
! self->fast_container--;
! if (unfast && self->fast_container < FAST_LIMIT)
! self->fast = 1;
! }
Py_XDECREF(module);
--- 1656,1661 ----
finally:
! if (self->fast && !fast_save_leave(self, args))
! res = -1;
Py_XDECREF(module);
***************
*** 1670,1674 ****
if (mod == NULL) {
/* Py_ErrClear(); ?? */
! cPickle_ErrFormat(PicklingError,
"Can't pickle %s: it's not found as %s.%s",
"OSS", args, module, global_name);
--- 1700,1704 ----
if (mod == NULL) {
/* Py_ErrClear(); ?? */
! cPickle_ErrFormat(PicklingError,
"Can't pickle %s: it's not found as %s.%s",
"OSS", args, module, global_name);
***************
*** 2252,2255 ****
--- 2282,2286 ----
self->fast = 0;
self->fast_container = 0;
+ self->fast_memo = NULL;
self->buf_size = 0;
self->dispatch_table = NULL;
***************
*** 2340,2343 ****
--- 2371,2375 ----
Py_XDECREF(self->write);
Py_XDECREF(self->memo);
+ Py_XDECREF(self->fast_memo);
Py_XDECREF(self->arg);
Py_XDECREF(self->file);