[Python-3000-checkins] r56438 - in python/branches/py3k-struni: Lib/pickle.py Lib/test/test_datetime.py Modules/arraymodule.c Modules/cPickle.c Modules/datetimemodule.c Objects/bytesobject.c Objects/exceptions.c Python/modsupport.c
martin.v.loewis
python-3000-checkins at python.org
Wed Jul 18 04:28:28 CEST 2007
Author: martin.v.loewis
Date: Wed Jul 18 04:28:27 2007
New Revision: 56438
Modified:
python/branches/py3k-struni/Lib/pickle.py
python/branches/py3k-struni/Lib/test/test_datetime.py
python/branches/py3k-struni/Modules/arraymodule.c
python/branches/py3k-struni/Modules/cPickle.c
python/branches/py3k-struni/Modules/datetimemodule.c
python/branches/py3k-struni/Objects/bytesobject.c
python/branches/py3k-struni/Objects/exceptions.c
python/branches/py3k-struni/Python/modsupport.c
Log:
Change Py_BuildValue to generate Unicode objects for
's' and 'c' codes.
Change pickle to dump bytes objects using the 'S'
code, and to load the 'S' code as byte objects.
Change datetime and array to generate and expect
bytes objects in reduce/unreduce.
Modified: python/branches/py3k-struni/Lib/pickle.py
==============================================================================
--- python/branches/py3k-struni/Lib/pickle.py (original)
+++ python/branches/py3k-struni/Lib/pickle.py Wed Jul 18 04:28:27 2007
@@ -506,6 +506,20 @@
self.memoize(obj)
dispatch[str8] = save_string
+ def save_bytes(self, obj):
+ # Like save_string
+ if self.bin:
+ n = len(obj)
+ if n < 256:
+ self.write(SHORT_BINSTRING + bytes([n]) + bytes(obj))
+ else:
+ self.write(BINSTRING + pack("<i", n) + bytes(obj))
+ else:
+ # Strip leading 'b'
+ self.write(STRING + bytes(repr(obj).lstrip("b")) + b'\n')
+ self.memoize(obj)
+ dispatch[bytes] = save_bytes
+
def save_unicode(self, obj, pack=struct.pack):
if self.bin:
encoded = obj.encode('utf-8')
@@ -931,12 +945,12 @@
break
else:
raise ValueError, "insecure string pickle"
- self.append(str8(codecs.escape_decode(rep)[0]))
+ self.append(bytes(codecs.escape_decode(rep)[0]))
dispatch[STRING[0]] = load_string
def load_binstring(self):
len = mloads(b'i' + self.read(4))
- self.append(str8(self.read(len)))
+ self.append(self.read(len))
dispatch[BINSTRING[0]] = load_binstring
def load_unicode(self):
@@ -950,7 +964,7 @@
def load_short_binstring(self):
len = ord(self.read(1))
- self.append(str8(self.read(len)))
+ self.append(self.read(len))
dispatch[SHORT_BINSTRING[0]] = load_short_binstring
def load_tuple(self):
Modified: python/branches/py3k-struni/Lib/test/test_datetime.py
==============================================================================
--- python/branches/py3k-struni/Lib/test/test_datetime.py (original)
+++ python/branches/py3k-struni/Lib/test/test_datetime.py Wed Jul 18 04:28:27 2007
@@ -1097,8 +1097,7 @@
# This shouldn't blow up because of the month byte alone. If
# the implementation changes to do more-careful checking, it may
# blow up because other fields are insane.
- # XXX Maybe this will have to become bytes?
- self.theclass(str8(base[:2] + chr(ord_byte) + base[3:]))
+ self.theclass(bytes(base[:2] + chr(ord_byte) + base[3:]))
#############################################################################
# datetime tests
Modified: python/branches/py3k-struni/Modules/arraymodule.c
==============================================================================
--- python/branches/py3k-struni/Modules/arraymodule.c (original)
+++ python/branches/py3k-struni/Modules/arraymodule.c Wed Jul 18 04:28:27 2007
@@ -1126,7 +1126,7 @@
Py_INCREF(dict);
}
if (array->ob_size > 0) {
- result = Py_BuildValue("O(cs#)O",
+ result = Py_BuildValue("O(cy#)O",
array->ob_type,
array->ob_descr->typecode,
array->ob_item,
Modified: python/branches/py3k-struni/Modules/cPickle.c
==============================================================================
--- python/branches/py3k-struni/Modules/cPickle.c (original)
+++ python/branches/py3k-struni/Modules/cPickle.c Wed Jul 18 04:28:27 2007
@@ -1151,6 +1151,92 @@
}
+static int
+save_bytes(Picklerobject *self, PyObject *args, int doput)
+{
+ int size, len;
+ PyObject *repr=0;
+
+ if ((size = PyBytes_Size(args)) < 0)
+ return -1;
+
+ if (!self->bin) {
+ char *repr_str;
+
+ static char string = STRING;
+
+ if (!( repr = PyObject_ReprStr8(args)))
+ return -1;
+
+ if ((len = PyString_Size(repr)) < 0)
+ goto err;
+ repr_str = PyString_AS_STRING((PyStringObject *)repr);
+
+ /* Strip leading 's' due to repr() of str8() returning s'...' */
+ if (repr_str[0] == 'b') {
+ repr_str++;
+ len--;
+ }
+
+ if (self->write_func(self, &string, 1) < 0)
+ goto err;
+
+ if (self->write_func(self, repr_str, len) < 0)
+ goto err;
+
+ if (self->write_func(self, "\n", 1) < 0)
+ goto err;
+
+ Py_XDECREF(repr);
+ }
+ else {
+ int i;
+ char c_str[5];
+
+ if ((size = PyBytes_Size(args)) < 0)
+ return -1;
+
+ if (size < 256) {
+ c_str[0] = SHORT_BINSTRING;
+ c_str[1] = size;
+ len = 2;
+ }
+ else if (size <= INT_MAX) {
+ c_str[0] = BINSTRING;
+ for (i = 1; i < 5; i++)
+ c_str[i] = (int)(size >> ((i - 1) * 8));
+ len = 5;
+ }
+ else
+ return -1; /* string too large */
+
+ if (self->write_func(self, c_str, len) < 0)
+ return -1;
+
+ if (size > 128 && Pdata_Check(self->file)) {
+ if (write_other(self, NULL, 0) < 0) return -1;
+ PDATA_APPEND(self->file, args, -1);
+ }
+ else {
+ if (self->write_func(self,
+ PyBytes_AsString(args),
+ size) < 0)
+ return -1;
+ }
+ }
+
+ if (doput)
+ if (put(self, args) < 0)
+ return -1;
+
+ return 0;
+
+ err:
+ Py_XDECREF(repr);
+ return -1;
+}
+
+
/* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
backslash and newline characters to \uXXXX escapes. */
static PyObject *
@@ -2086,11 +2172,11 @@
type = args->ob_type;
switch (type->tp_name[0]) {
- case 'b':
+ case 'b': /* XXX may want to save short byte strings here. */
if (args == Py_False || args == Py_True) {
res = save_bool(self, args);
goto finally;
- }
+ }
break;
case 'i':
if (type == &PyLong_Type) {
@@ -2197,6 +2283,11 @@
res = save_global(self, args, NULL);
goto finally;
}
+ else if (type == &PyBytes_Type) {
+ res = save_bytes(self, args, 1);
+ goto finally;
+ }
+ break;
}
if (!pers_save && self->inst_pers_func) {
@@ -3131,11 +3222,17 @@
goto insecure;
/********************************************/
+ /* XXX avoid going through str8 here. */
str = PyString_DecodeEscape(p, len, NULL, 0, NULL);
free(s);
if (str) {
- PDATA_PUSH(self->stack, str, -1);
- res = 0;
+ PyObject *str2 = PyBytes_FromStringAndSize(
+ PyString_AsString(str), PyString_Size(str));
+ Py_DECREF(str);
+ if (str2) {
+ PDATA_PUSH(self->stack, str2, -1);
+ res = 0;
+ }
}
return res;
@@ -3160,7 +3257,7 @@
if (self->read_func(self, &s, l) < 0)
return -1;
- if (!( py_string = PyString_FromStringAndSize(s, l)))
+ if (!( py_string = PyBytes_FromStringAndSize(s, l)))
return -1;
PDATA_PUSH(self->stack, py_string, -1);
@@ -3182,7 +3279,7 @@
if (self->read_func(self, &s, l) < 0) return -1;
- if (!( py_string = PyString_FromStringAndSize(s, l))) return -1;
+ if (!( py_string = PyBytes_FromStringAndSize(s, l))) return -1;
PDATA_PUSH(self->stack, py_string, -1);
return 0;
Modified: python/branches/py3k-struni/Modules/datetimemodule.c
==============================================================================
--- python/branches/py3k-struni/Modules/datetimemodule.c (original)
+++ python/branches/py3k-struni/Modules/datetimemodule.c Wed Jul 18 04:28:27 2007
@@ -2183,15 +2183,15 @@
/* Check for invocation from pickle with __getstate__ state */
if (PyTuple_GET_SIZE(args) == 1 &&
- PyString_Check(state = PyTuple_GET_ITEM(args, 0)) &&
- PyString_GET_SIZE(state) == _PyDateTime_DATE_DATASIZE &&
- MONTH_IS_SANE(PyString_AS_STRING(state)[2]))
+ PyBytes_Check(state = PyTuple_GET_ITEM(args, 0)) &&
+ PyBytes_GET_SIZE(state) == _PyDateTime_DATE_DATASIZE &&
+ MONTH_IS_SANE(PyBytes_AS_STRING(state)[2]))
{
PyDateTime_Date *me;
me = (PyDateTime_Date *) (type->tp_alloc(type, 0));
if (me != NULL) {
- char *pdata = PyString_AS_STRING(state);
+ char *pdata = PyBytes_AS_STRING(state);
memcpy(me->data, pdata, _PyDateTime_DATE_DATASIZE);
me->hashcode = -1;
}
@@ -2509,13 +2509,13 @@
return clone;
}
-static PyObject *date_getstate(PyDateTime_Date *self);
+static PyObject *date_getstate(PyDateTime_Date *self, int hashable);
static long
date_hash(PyDateTime_Date *self)
{
if (self->hashcode == -1) {
- PyObject *temp = date_getstate(self);
+ PyObject *temp = date_getstate(self, 1);
if (temp != NULL) {
self->hashcode = PyObject_Hash(temp);
Py_DECREF(temp);
@@ -2543,18 +2543,22 @@
/* __getstate__ isn't exposed */
static PyObject *
-date_getstate(PyDateTime_Date *self)
+date_getstate(PyDateTime_Date *self, int hashable)
{
- return Py_BuildValue(
- "(N)",
- PyString_FromStringAndSize((char *)self->data,
- _PyDateTime_DATE_DATASIZE));
+ PyObject* field;
+ if (hashable)
+ field = PyString_FromStringAndSize(
+ (char*)self->data, _PyDateTime_DATE_DATASIZE);
+ else
+ field = PyBytes_FromStringAndSize(
+ (char*)self->data, _PyDateTime_DATE_DATASIZE);
+ return Py_BuildValue("(N)", field);
}
static PyObject *
date_reduce(PyDateTime_Date *self, PyObject *arg)
{
- return Py_BuildValue("(ON)", self->ob_type, date_getstate(self));
+ return Py_BuildValue("(ON)", self->ob_type, date_getstate(self, 0));
}
static PyMethodDef date_methods[] = {
@@ -2998,9 +3002,9 @@
/* Check for invocation from pickle with __getstate__ state */
if (PyTuple_GET_SIZE(args) >= 1 &&
PyTuple_GET_SIZE(args) <= 2 &&
- PyString_Check(state = PyTuple_GET_ITEM(args, 0)) &&
- PyString_GET_SIZE(state) == _PyDateTime_TIME_DATASIZE &&
- ((unsigned char) (PyString_AS_STRING(state)[0])) < 24)
+ PyBytes_Check(state = PyTuple_GET_ITEM(args, 0)) &&
+ PyBytes_GET_SIZE(state) == _PyDateTime_TIME_DATASIZE &&
+ ((unsigned char) (PyBytes_AS_STRING(state)[0])) < 24)
{
PyDateTime_Time *me;
char aware;
@@ -3016,7 +3020,7 @@
aware = (char)(tzinfo != Py_None);
me = (PyDateTime_Time *) (type->tp_alloc(type, aware));
if (me != NULL) {
- char *pdata = PyString_AS_STRING(state);
+ char *pdata = PyBytes_AS_STRING(state);
memcpy(me->data, pdata, _PyDateTime_TIME_DATASIZE);
me->hashcode = -1;
@@ -3331,7 +3335,7 @@
PyObject *basestate;
PyObject *result = NULL;
- basestate = PyString_FromStringAndSize((char *)self->data,
+ basestate = PyBytes_FromStringAndSize((char *)self->data,
_PyDateTime_TIME_DATASIZE);
if (basestate != NULL) {
if (! HASTZINFO(self) || self->tzinfo == Py_None)
@@ -3513,9 +3517,9 @@
/* Check for invocation from pickle with __getstate__ state */
if (PyTuple_GET_SIZE(args) >= 1 &&
PyTuple_GET_SIZE(args) <= 2 &&
- PyString_Check(state = PyTuple_GET_ITEM(args, 0)) &&
- PyString_GET_SIZE(state) == _PyDateTime_DATETIME_DATASIZE &&
- MONTH_IS_SANE(PyString_AS_STRING(state)[2]))
+ PyBytes_Check(state = PyTuple_GET_ITEM(args, 0)) &&
+ PyBytes_GET_SIZE(state) == _PyDateTime_DATETIME_DATASIZE &&
+ MONTH_IS_SANE(PyBytes_AS_STRING(state)[2]))
{
PyDateTime_DateTime *me;
char aware;
@@ -3531,7 +3535,7 @@
aware = (char)(tzinfo != Py_None);
me = (PyDateTime_DateTime *) (type->tp_alloc(type , aware));
if (me != NULL) {
- char *pdata = PyString_AS_STRING(state);
+ char *pdata = PyBytes_AS_STRING(state);
memcpy(me->data, pdata, _PyDateTime_DATETIME_DATASIZE);
me->hashcode = -1;
@@ -4375,8 +4379,8 @@
PyObject *basestate;
PyObject *result = NULL;
- basestate = PyString_FromStringAndSize((char *)self->data,
- _PyDateTime_DATETIME_DATASIZE);
+ basestate = PyBytes_FromStringAndSize((char *)self->data,
+ _PyDateTime_DATETIME_DATASIZE);
if (basestate != NULL) {
if (! HASTZINFO(self) || self->tzinfo == Py_None)
result = PyTuple_Pack(1, basestate);
Modified: python/branches/py3k-struni/Objects/bytesobject.c
==============================================================================
--- python/branches/py3k-struni/Objects/bytesobject.c (original)
+++ python/branches/py3k-struni/Objects/bytesobject.c Wed Jul 18 04:28:27 2007
@@ -2724,6 +2724,9 @@
static PyObject *
bytes_reduce(PyBytesObject *self)
{
+ /* XXX: This currently returns a Py_UNICODE-widened string
+ in the tuple which is completely useless. Pickle stopped
+ using it for that reason. */
return Py_BuildValue("(O(s#))",
self->ob_type,
self->ob_bytes == NULL ? "" : self->ob_bytes,
Modified: python/branches/py3k-struni/Objects/exceptions.c
==============================================================================
--- python/branches/py3k-struni/Objects/exceptions.c (original)
+++ python/branches/py3k-struni/Objects/exceptions.c Wed Jul 18 04:28:27 2007
@@ -831,28 +831,32 @@
static PyObject *
SyntaxError_str(PySyntaxErrorObject *self)
{
- int have_filename = 0;
int have_lineno = 0;
+ char *filename = 0;
/* XXX -- do all the additional formatting with filename and
lineno here */
- have_filename = (self->filename != NULL) &&
- PyString_Check(self->filename);
+ if (self->filename) {
+ if (PyString_Check(self->filename))
+ filename = PyString_AsString(self->filename);
+ else if (PyUnicode_Check(self->filename))
+ filename = PyUnicode_AsString(self->filename);
+ }
have_lineno = (self->lineno != NULL) && PyInt_CheckExact(self->lineno);
- if (!have_filename && !have_lineno)
+ if (!filename && !have_lineno)
return PyObject_Unicode(self->msg ? self->msg : Py_None);
- if (have_filename && have_lineno)
+ if (filename && have_lineno)
return PyUnicode_FromFormat("%S (%s, line %ld)",
self->msg ? self->msg : Py_None,
- my_basename(PyString_AS_STRING(self->filename)),
+ my_basename(filename),
PyInt_AsLong(self->lineno));
- else if (have_filename)
+ else if (filename)
return PyUnicode_FromFormat("%S (%s)",
self->msg ? self->msg : Py_None,
- my_basename(PyString_AS_STRING(self->filename)));
+ my_basename(filename));
else /* only have_lineno */
return PyUnicode_FromFormat("%S (line %ld)",
self->msg ? self->msg : Py_None,
Modified: python/branches/py3k-struni/Python/modsupport.c
==============================================================================
--- python/branches/py3k-struni/Python/modsupport.c (original)
+++ python/branches/py3k-struni/Python/modsupport.c Wed Jul 18 04:28:27 2007
@@ -387,7 +387,7 @@
{
char p[1];
p[0] = (char)va_arg(*p_va, int);
- return PyString_FromStringAndSize(p, 1);
+ return PyUnicode_FromStringAndSize(p, 1);
}
case 'C':
{
@@ -438,7 +438,7 @@
}
n = (Py_ssize_t)m;
}
- v = PyString_FromStringAndSize(str, n);
+ v = PyUnicode_FromStringAndSize(str, n);
}
return v;
}
More information about the Python-3000-checkins
mailing list