[Python-checkins] python/dist/src/Modules cPickle.c,2.115,2.116
tim_one@users.sourceforge.net
tim_one@users.sourceforge.net
Sun, 02 Feb 2003 12:29:41 -0800
Update of /cvsroot/python/python/dist/src/Modules
In directory sc8-pr-cvs1:/tmp/cvs-serv28209/Modules
Modified Files:
cPickle.c
Log Message:
cPickle support for TUPLE[123]. Incidentally plugged several undetected
overflow holes in Pdata_grow().
Index: cPickle.c
===================================================================
RCS file: /cvsroot/python/python/dist/src/Modules/cPickle.c,v
retrieving revision 2.115
retrieving revision 2.116
diff -C2 -d -r2.115 -r2.116
*** cPickle.c 2 Feb 2003 18:29:33 -0000 2.115
--- cPickle.c 2 Feb 2003 20:29:39 -0000 2.116
***************
*** 111,115 ****
typedef struct {
PyObject_HEAD
! int length, size;
PyObject **data;
} Pdata;
--- 111,116 ----
typedef struct {
PyObject_HEAD
! int length; /* number of initial slots in data currently used */
! int size; /* number of slots in data allocated */
PyObject **data;
} Pdata;
***************
*** 121,128 ****
PyObject **p;
! for (i=self->length, p=self->data; --i >= 0; p++) Py_DECREF(*p);
!
! if (self->data) free(self->data);
!
PyObject_Del(self);
}
--- 122,130 ----
PyObject **p;
! for (i = self->length, p = self->data; --i >= 0; p++) {
! Py_DECREF(*p);
! }
! if (self->data)
! free(self->data);
PyObject_Del(self);
}
***************
*** 141,149 ****
Pdata *self;
! if (!( self = PyObject_New(Pdata, &PdataType))) return NULL;
! self->size=8;
! self->length=0;
! self->data=malloc(self->size * sizeof(PyObject*));
! if (self->data) return (PyObject*)self;
Py_DECREF(self);
return PyErr_NoMemory();
--- 143,153 ----
Pdata *self;
! if (!(self = PyObject_New(Pdata, &PdataType)))
! return NULL;
! self->size = 8;
! self->length = 0;
! self->data = malloc(self->size * sizeof(PyObject*));
! if (self->data)
! return (PyObject*)self;
Py_DECREF(self);
return PyErr_NoMemory();
***************
*** 157,160 ****
--- 161,167 ----
}
+ /* Retain only the initial clearto items. If clearto >= the current
+ * number of items, this is a (non-erroneous) NOP.
+ */
static int
Pdata_clear(Pdata *self, int clearto)
***************
*** 166,202 ****
if (clearto >= self->length) return 0;
! for (i=self->length, p=self->data+clearto; --i >= clearto; p++)
Py_DECREF(*p);
! self->length=clearto;
return 0;
}
-
static int
Pdata_grow(Pdata *self)
{
! if (! self->size) {
! PyErr_NoMemory();
! return -1;
! }
! self->size *= 2;
! self->data = realloc(self->data, self->size*sizeof(PyObject*));
! if (! self->data) {
! self->size = 0;
! PyErr_NoMemory();
! return -1;
! }
return 0;
- }
! #define PDATA_POP(D,V) { \
! if ((D)->length) V=D->data[--((D)->length)]; \
! else { \
! PyErr_SetString(UnpicklingError, "bad pickle data"); \
! V=NULL; \
! } \
}
static PyObject *
--- 173,226 ----
if (clearto >= self->length) return 0;
! for (i = self->length, p = self->data + clearto;
! --i >= clearto;
! p++) {
Py_DECREF(*p);
! }
! self->length = clearto;
return 0;
}
static int
Pdata_grow(Pdata *self)
{
! int bigger;
! size_t nbytes;
!
! if (! self->size)
! goto nomemory;
! bigger = self->size << 1;
! if (bigger <= 0)
! goto nomemory;
! if ((int)(size_t)bigger != bigger)
! goto nomemory;
! nbytes = (size_t)bigger * sizeof(PyObject *);
! if (nbytes / sizeof(PyObject *) != (size_t)bigger)
! goto nomemory;
! self->data = realloc(self->data, nbytes);
! if (self->data == NULL)
! goto nomemory;
! self->size = bigger;
return 0;
! nomemory:
! self->size = 0;
! PyErr_NoMemory();
! return -1;
}
+ /* D is a Pdata *. Pop the topmost element and store it into V, which
+ * must be an lvalue holding PyObject *. On stack underflow, UnpicklingError
+ * is raised and V is set to NULL. D and V may be evaluated several times.
+ */
+ #define PDATA_POP(D, V) { \
+ if ((D)->length) \
+ (V) = (D)->data[--((D)->length)]; \
+ else { \
+ PyErr_SetString(UnpicklingError, "bad pickle data"); \
+ (V) = NULL; \
+ } \
+ }
static PyObject *
***************
*** 206,215 ****
int i, j, l;
! l=self->length-start;
! if (!( r=PyTuple_New(l))) return NULL;
! for (i=start, j=0 ; j < l; i++, j++)
PyTuple_SET_ITEM(r, j, self->data[i]);
! self->length=start;
return r;
}
--- 230,241 ----
int i, j, l;
! l = self->length-start;
! r = PyTuple_New(l);
! if (r == NULL)
! return NULL;
! for (i = start, j = 0 ; j < l; i++, j++)
PyTuple_SET_ITEM(r, j, self->data[i]);
! self->length = start;
return r;
}
***************
*** 1445,1524 ****
#endif
static int
save_tuple(Picklerobject *self, PyObject *args)
{
! PyObject *element = 0, *py_tuple_id = 0;
! int len, i, res = -1;
static char tuple = TUPLE;
!
! if (self->write_func(self, &MARKv, 1) < 0)
! goto finally;
if ((len = PyTuple_Size(args)) < 0)
goto finally;
! for (i = 0; i < len; i++) {
! if (!( element = PyTuple_GET_ITEM((PyTupleObject *)args, i)))
! goto finally;
! if (save(self, element, 0) < 0)
! goto finally;
}
! if (!( py_tuple_id = PyLong_FromVoidPtr(args)))
goto finally;
! if (len) {
if (PyDict_GetItem(self->memo, py_tuple_id)) {
! if (self->bin) {
! static char pop_mark = POP_MARK;
!
! if (self->write_func(self, &pop_mark, 1) < 0)
goto finally;
! }
! else {
! static char pop = POP;
!
! for (i = 0; i <= len; i++) {
! if (self->write_func(self, &pop, 1) < 0)
! goto finally;
! }
! }
!
if (get(self, py_tuple_id) < 0)
goto finally;
-
res = 0;
goto finally;
}
}
! if (self->write_func(self, &tuple, 1) < 0) {
goto finally;
}
! if (put(self, args) < 0)
goto finally;
! res = 0;
finally:
Py_XDECREF(py_tuple_id);
-
return res;
}
static int
- save_empty_tuple(Picklerobject *self, PyObject *args)
- {
- static char tuple = EMPTY_TUPLE;
-
- return self->write_func(self, &tuple, 1);
- }
-
-
- static int
save_list(Picklerobject *self, PyObject *args)
{
--- 1471,1611 ----
#endif
+ /* A helper for save_tuple. Push the len elements in tuple t on the stack. */
+ static int
+ store_tuple_elememts(Picklerobject *self, PyObject *t, int len)
+ {
+ int i;
+ int res = -1; /* guilty until proved innocent */
+
+ assert(PyTuple_Size(t) == len);
+
+ for (i = 0; i < len; i++) {
+ PyObject *element = PyTuple_GET_ITEM(t, i);
+ if (element == NULL)
+ goto finally;
+ if (save(self, element, 0) < 0)
+ goto finally;
+ }
+ res = 0;
+
+ finally:
+ return res;
+ }
+
+ /* Tuples are ubiquitous in the pickle protocols, so many techniques are
+ * used across protocols to minimize the space needed to pickle them.
+ * Tuples are also the only builtin immuatable type that can be recursive
+ * (a tuple can be reached from itself), and that requires some subtle
+ * magic so that it works in all cases. IOW, this is a long routine.
+ */
static int
save_tuple(Picklerobject *self, PyObject *args)
{
! PyObject *py_tuple_id = NULL;
! int len, i;
! int res = -1;
static char tuple = TUPLE;
! static char pop = POP;
! static char pop_mark = POP_MARK;
! static char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
if ((len = PyTuple_Size(args)) < 0)
goto finally;
! if (len == 0) {
! char c_str[2];
! if (self->proto) {
! c_str[0] = EMPTY_TUPLE;
! len = 1;
! }
! else {
! c_str[0] = MARK;
! c_str[1] = TUPLE;
! len = 2;
! }
! if (self->write_func(self, c_str, len) >= 0)
! res = 0;
! /* Don't memoize an empty tuple. */
! goto finally;
}
! /* A non-empty tuple. */
!
! /* id(tuple) isn't in the memo now. If it shows up there after
! * saving the tuple elements, the tuple must be recursive, in
! * which case we'll pop everything we put on the stack, and fetch
! * its value from the memo.
! */
! py_tuple_id = PyLong_FromVoidPtr(args);
! if (py_tuple_id == NULL)
goto finally;
! if (len <= 3 && self->proto >= 2) {
! /* Use TUPLE{1,2,3} opcodes. */
! if (store_tuple_elememts(self, args, len) < 0)
! goto finally;
if (PyDict_GetItem(self->memo, py_tuple_id)) {
! /* pop the len elements */
! for (i = 0; i < len; ++i)
! if (self->write_func(self, &pop, 1) < 0)
goto finally;
! /* fetch from memo */
if (get(self, py_tuple_id) < 0)
goto finally;
res = 0;
goto finally;
}
+ /* Not recursive. */
+ if (self->write_func(self, len2opcode + len, 1) < 0)
+ goto finally;
+ goto memoize;
}
! /* proto < 2 and len > 0, or proto >= 2 and len > 3.
! * Generate MARK elt1 elt2 ... TUPLE
! */
! if (self->write_func(self, &MARKv, 1) < 0)
! goto finally;
!
! if (store_tuple_elememts(self, args, len) < 0)
! goto finally;
!
! if (PyDict_GetItem(self->memo, py_tuple_id)) {
! /* pop the stack stuff we pushed */
! if (self->bin) {
! if (self->write_func(self, &pop_mark, 1) < 0)
! goto finally;
! }
! else {
! /* Note that we pop one more than len, to remove
! * the MARK too.
! */
! for (i = 0; i <= len; i++)
! if (self->write_func(self, &pop, 1) < 0)
! goto finally;
! }
! /* fetch from memo */
! if (get(self, py_tuple_id) >= 0)
! res = 0;
goto finally;
}
! /* Not recursive. */
! if (self->write_func(self, &tuple, 1) < 0)
goto finally;
! memoize:
! if (put(self, args) >= 0)
! res = 0;
finally:
Py_XDECREF(py_tuple_id);
return res;
}
static int
save_list(Picklerobject *self, PyObject *args)
{
***************
*** 1973,1977 ****
static int
! save(Picklerobject *self, PyObject *args, int pers_save)
{
PyTypeObject *type;
--- 2060,2064 ----
static int
! save(Picklerobject *self, PyObject *args, int pers_save)
{
PyTypeObject *type;
***************
*** 2029,2035 ****
case 't':
! if (type == &PyTuple_Type && PyTuple_Size(args)==0) {
! if (self->bin) res = save_empty_tuple(self, args);
! else res = save_tuple(self, args);
goto finally;
}
--- 2116,2121 ----
case 't':
! if (type == &PyTuple_Type && PyTuple_Size(args) == 0) {
! res = save_tuple(self, args);
goto finally;
}
***************
*** 3194,3202 ****
static int
! load_empty_tuple(Unpicklerobject *self)
{
! PyObject *tup;
! if (!( tup=PyTuple_New(0))) return -1;
PDATA_PUSH(self->stack, tup, -1);
return 0;
--- 3280,3298 ----
static int
! load_counted_tuple(Unpicklerobject *self, int len)
{
! PyObject *tup = PyTuple_New(len);
! if (tup == NULL)
! return -1;
!
! while (--len >= 0) {
! PyObject *element;
!
! PDATA_POP(self->stack, element);
! if (element == NULL)
! return -1;
! PyTuple_SET_ITEM(tup, len, element);
! }
PDATA_PUSH(self->stack, tup, -1);
return 0;
***************
*** 4019,4023 ****
case EMPTY_TUPLE:
! if (load_empty_tuple(self) < 0)
break;
continue;
--- 4115,4134 ----
case EMPTY_TUPLE:
! if (load_counted_tuple(self, 0) < 0)
! break;
! continue;
!
! case TUPLE1:
! if (load_counted_tuple(self, 1) < 0)
! break;
! continue;
!
! case TUPLE2:
! if (load_counted_tuple(self, 2) < 0)
! break;
! continue;
!
! case TUPLE3:
! if (load_counted_tuple(self, 3) < 0)
break;
continue;
***************
*** 4347,4351 ****
case EMPTY_TUPLE:
! if (load_empty_tuple(self) < 0)
break;
continue;
--- 4458,4477 ----
case EMPTY_TUPLE:
! if (load_counted_tuple(self, 0) < 0)
! break;
! continue;
!
! case TUPLE1:
! if (load_counted_tuple(self, 1) < 0)
! break;
! continue;
!
! case TUPLE2:
! if (load_counted_tuple(self, 2) < 0)
! break;
! continue;
!
! case TUPLE3:
! if (load_counted_tuple(self, 3) < 0)
break;
continue;