[Python-3000-checkins] r65886 - in python/branches/py3k: Lib/test/test_codecs.py Lib/test/test_memoryview.py Misc/NEWS Objects/memoryobject.c
antoine.pitrou
python-3000-checkins at python.org
Wed Aug 20 00:09:34 CEST 2008
Author: antoine.pitrou
Date: Wed Aug 20 00:09:34 2008
New Revision: 65886
Log:
Issue #2394: implement more of the memoryview API.
Modified:
python/branches/py3k/Lib/test/test_codecs.py
python/branches/py3k/Lib/test/test_memoryview.py
python/branches/py3k/Misc/NEWS
python/branches/py3k/Objects/memoryobject.c
Modified: python/branches/py3k/Lib/test/test_codecs.py
==============================================================================
--- python/branches/py3k/Lib/test/test_codecs.py (original)
+++ python/branches/py3k/Lib/test/test_codecs.py Wed Aug 20 00:09:34 2008
@@ -621,7 +621,7 @@
def test_bug1601501(self):
# SF bug #1601501: check that the codec works with a buffer
- str(b"\xef\xbb\xbf", "utf-8-sig")
+ self.assertEquals(str(b"\xef\xbb\xbf", "utf-8-sig"), "")
def test_bom(self):
d = codecs.getincrementaldecoder("utf-8-sig")()
Modified: python/branches/py3k/Lib/test/test_memoryview.py
==============================================================================
--- python/branches/py3k/Lib/test/test_memoryview.py (original)
+++ python/branches/py3k/Lib/test/test_memoryview.py Wed Aug 20 00:09:34 2008
@@ -5,8 +5,166 @@
import unittest
import test.support
+import sys
-class MemoryviewTest(unittest.TestCase):
+
+class CommonMemoryTests:
+ #
+ # Tests common to direct memoryviews and sliced memoryviews
+ #
+
+ base_object = b"abcdef"
+
+ def check_getitem_with_type(self, tp):
+ b = tp(self.base_object)
+ oldrefcount = sys.getrefcount(b)
+ m = self._view(b)
+ self.assertEquals(m[0], b"a")
+ self.assert_(isinstance(m[0], bytes), type(m[0]))
+ self.assertEquals(m[5], b"f")
+ self.assertEquals(m[-1], b"f")
+ self.assertEquals(m[-6], b"a")
+ # Bounds checking
+ self.assertRaises(IndexError, lambda: m[6])
+ self.assertRaises(IndexError, lambda: m[-7])
+ self.assertRaises(IndexError, lambda: m[sys.maxsize])
+ self.assertRaises(IndexError, lambda: m[-sys.maxsize])
+ # Type checking
+ self.assertRaises(TypeError, lambda: m[None])
+ self.assertRaises(TypeError, lambda: m[0.0])
+ self.assertRaises(TypeError, lambda: m["a"])
+ m = None
+ self.assertEquals(sys.getrefcount(b), oldrefcount)
+
+ def test_getitem_readonly(self):
+ self.check_getitem_with_type(bytes)
+
+ def test_getitem_writable(self):
+ self.check_getitem_with_type(bytearray)
+
+ def test_setitem_readonly(self):
+ b = self.base_object
+ oldrefcount = sys.getrefcount(b)
+ m = self._view(b)
+ def setitem(value):
+ m[0] = value
+ self.assertRaises(TypeError, setitem, b"a")
+ self.assertRaises(TypeError, setitem, 65)
+ self.assertRaises(TypeError, setitem, memoryview(b"a"))
+ m = None
+ self.assertEquals(sys.getrefcount(b), oldrefcount)
+
+ def test_setitem_writable(self):
+ b = bytearray(self.base_object)
+ oldrefcount = sys.getrefcount(b)
+ m = self._view(b)
+ m[0] = b"0"
+ self._check_contents(b, b"0bcdef")
+ m[1:3] = b"12"
+ self._check_contents(b, b"012def")
+ m[1:1] = b""
+ self._check_contents(b, b"012def")
+ m[:] = b"abcdef"
+ self._check_contents(b, b"abcdef")
+
+ # Overlapping copies of a view into itself
+ m[0:3] = m[2:5]
+ self._check_contents(b, b"cdedef")
+ m[:] = b"abcdef"
+ m[2:5] = m[0:3]
+ self._check_contents(b, b"ababcf")
+
+ def setitem(key, value):
+ m[key] = value
+ # Bounds checking
+ self.assertRaises(IndexError, setitem, 6, b"a")
+ self.assertRaises(IndexError, setitem, -7, b"a")
+ self.assertRaises(IndexError, setitem, sys.maxsize, b"a")
+ self.assertRaises(IndexError, setitem, -sys.maxsize, b"a")
+ # Wrong index/slice types
+ self.assertRaises(TypeError, setitem, 0.0, b"a")
+ self.assertRaises(TypeError, setitem, (0,), b"a")
+ self.assertRaises(TypeError, setitem, "a", b"a")
+ # Trying to resize the memory object
+ self.assertRaises(ValueError, setitem, 0, b"")
+ self.assertRaises(ValueError, setitem, 0, b"ab")
+ self.assertRaises(ValueError, setitem, slice(1,1), b"a")
+ self.assertRaises(ValueError, setitem, slice(0,2), b"a")
+
+ m = None
+ self.assertEquals(sys.getrefcount(b), oldrefcount)
+
+ def test_len(self):
+ self.assertEquals(len(self._view(self.base_object)), 6)
+
+ def test_tobytes(self):
+ m = self._view(self.base_object)
+ b = m.tobytes()
+ self.assertEquals(b, b"abcdef")
+ self.assert_(isinstance(b, bytes), type(b))
+
+ def test_tolist(self):
+ m = self._view(self.base_object)
+ l = m.tolist()
+ self.assertEquals(l, list(b"abcdef"))
+
+ def test_compare(self):
+ # memoryviews can compare for equality with other objects
+ # having the buffer interface.
+ m = self._view(self.base_object)
+ for tp in (bytes, bytearray):
+ self.assertTrue(m == tp(b"abcdef"))
+ self.assertFalse(m != tp(b"abcdef"))
+ self.assertFalse(m == tp(b"abcde"))
+ self.assertTrue(m != tp(b"abcde"))
+ self.assertFalse(m == tp(b"abcde1"))
+ self.assertTrue(m != tp(b"abcde1"))
+ self.assertTrue(m == m)
+ self.assertTrue(m == m[:])
+ self.assertTrue(m[0:6] == m[:])
+ self.assertFalse(m[0:5] == m)
+
+ # Comparison with objects which don't support the buffer API
+ self.assertFalse(m == "abc")
+ self.assertTrue(m != "abc")
+ self.assertFalse("abc" == m)
+ self.assertTrue("abc" != m)
+
+ # Unordered comparisons
+ for c in (m, b"abcdef"):
+ self.assertRaises(TypeError, lambda: m < c)
+ self.assertRaises(TypeError, lambda: c <= m)
+ self.assertRaises(TypeError, lambda: m >= c)
+ self.assertRaises(TypeError, lambda: c > m)
+
+ def check_attributes_with_type(self, tp):
+ b = tp(self.base_object)
+ m = self._view(b)
+ self.assertEquals(m.format, 'B')
+ self.assertEquals(m.itemsize, 1)
+ self.assertEquals(m.ndim, 1)
+ self.assertEquals(m.shape, (6,))
+ self.assertEquals(m.size, 6)
+ self.assertEquals(m.strides, (1,))
+ self.assertEquals(m.suboffsets, None)
+ return m
+
+ def test_attributes_readonly(self):
+ m = self.check_attributes_with_type(bytes)
+ self.assertEquals(m.readonly, True)
+
+ def test_attributes_writable(self):
+ m = self.check_attributes_with_type(bytearray)
+ self.assertEquals(m.readonly, False)
+
+
+class MemoryviewTest(unittest.TestCase, CommonMemoryTests):
+
+ def _view(self, obj):
+ return memoryview(obj)
+
+ def _check_contents(self, obj, contents):
+ self.assertEquals(obj, contents)
def test_constructor(self):
ob = b'test'
@@ -17,8 +175,38 @@
self.assertRaises(TypeError, memoryview, argument=ob)
self.assertRaises(TypeError, memoryview, ob, argument=True)
+
+class MemorySliceTest(unittest.TestCase, CommonMemoryTests):
+ base_object = b"XabcdefY"
+
+ def _view(self, obj):
+ m = memoryview(obj)
+ return m[1:7]
+
+ def _check_contents(self, obj, contents):
+ self.assertEquals(obj[1:7], contents)
+
+ def test_refs(self):
+ m = memoryview(b"ab")
+ oldrefcount = sys.getrefcount(m)
+ m[1:2]
+ self.assertEquals(sys.getrefcount(m), oldrefcount)
+
+
+class MemorySliceSliceTest(unittest.TestCase, CommonMemoryTests):
+ base_object = b"XabcdefY"
+
+ def _view(self, obj):
+ m = memoryview(obj)
+ return m[:7][1:]
+
+ def _check_contents(self, obj, contents):
+ self.assertEquals(obj[1:7], contents)
+
+
def test_main():
- test.support.run_unittest(MemoryviewTest)
+ test.support.run_unittest(
+ MemoryviewTest, MemorySliceTest, MemorySliceSliceTest)
if __name__ == "__main__":
Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS (original)
+++ python/branches/py3k/Misc/NEWS Wed Aug 20 00:09:34 2008
@@ -12,6 +12,13 @@
Core and Builtins
-----------------
+- Issue #2394: implement more of the memoryview API, with the caveat that
+ only one-dimensional contiguous buffers are supported and exercised right
+ now. Slicing, slice assignment and comparison (equality and inequality)
+ have been added. Also, the tolist() method has been implemented, but only
+ for byte buffers. Endly, the API has been updated to return bytes objects
+ wherever it used to return bytearrays.
+
- Issue #3560: clean up the new C PyMemoryView API so that naming is
internally consistent; add macros PyMemoryView_GET_BASE() and
PyMemoryView_GET_BUFFER() to access useful properties of a memory views
Modified: python/branches/py3k/Objects/memoryobject.c
==============================================================================
--- python/branches/py3k/Objects/memoryobject.c (original)
+++ python/branches/py3k/Objects/memoryobject.c Wed Aug 20 00:09:34 2008
@@ -13,8 +13,8 @@
}
if (self->view.obj == NULL)
return 0;
- return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(self->base, NULL,
- PyBUF_FULL);
+ return self->view.obj->ob_type->tp_as_buffer->bf_getbuffer(
+ self->view.obj, NULL, PyBUF_FULL);
}
static void
@@ -37,9 +37,14 @@
&PyMemoryView_Type);
if (mview == NULL) return NULL;
mview->base = NULL;
+ /* XXX there should be an API to duplicate a buffer object */
mview->view = *info;
- if (info->obj)
- Py_INCREF(mview->view.obj);
+ if (info->shape == &(info->len))
+ mview->view.shape = &(mview->view.len);
+ if (info->strides == &(info->itemsize))
+ mview->view.strides = &(mview->view.itemsize);
+ /* NOTE: mview->view.obj should already have been incref'ed as
+ part of PyBuffer_FillInfo(). */
return (PyObject *)mview;
}
@@ -258,12 +263,12 @@
"for a non-contiguousobject.");
return NULL;
}
- bytes = PyByteArray_FromStringAndSize(NULL, view->len);
+ bytes = PyBytes_FromStringAndSize(NULL, view->len);
if (bytes == NULL) {
PyBuffer_Release(view);
return NULL;
}
- dest = PyByteArray_AS_STRING(bytes);
+ dest = PyBytes_AS_STRING(bytes);
/* different copying strategy depending on whether
or not any pointer de-referencing is needed
*/
@@ -386,17 +391,45 @@
static PyObject *
memory_tobytes(PyMemoryViewObject *mem, PyObject *noargs)
{
- return PyByteArray_FromObject((PyObject *)mem);
+ return PyObject_CallFunctionObjArgs(
+ (PyObject *) &PyBytes_Type, mem, NULL);
}
+/* TODO: rewrite this function using the struct module to unpack
+ each buffer item */
+
static PyObject *
memory_tolist(PyMemoryViewObject *mem, PyObject *noargs)
{
- /* This should construct a (nested) list of unpacked objects
- possibly using the struct module.
- */
- Py_INCREF(Py_NotImplemented);
- return Py_NotImplemented;
+ Py_buffer *view = &(mem->view);
+ Py_ssize_t i;
+ PyObject *res, *item;
+ char *buf;
+
+ if (strcmp(view->format, "B") || view->itemsize != 1) {
+ PyErr_SetString(PyExc_NotImplementedError,
+ "tolist() only supports byte views");
+ return NULL;
+ }
+ if (view->ndim != 1) {
+ PyErr_SetString(PyExc_NotImplementedError,
+ "tolist() only supports one-dimensional objects");
+ return NULL;
+ }
+ res = PyList_New(view->len);
+ if (res == NULL)
+ return NULL;
+ buf = view->buf;
+ for (i = 0; i < view->len; i++) {
+ item = PyLong_FromUnsignedLong((unsigned char) *buf);
+ if (item == NULL) {
+ Py_DECREF(res);
+ return NULL;
+ }
+ PyList_SET_ITEM(res, i, item);
+ buf++;
+ }
+ return res;
}
@@ -412,7 +445,7 @@
memory_dealloc(PyMemoryViewObject *self)
{
if (self->view.obj != NULL) {
- if (PyTuple_Check(self->base)) {
+ if (self->base && PyTuple_Check(self->base)) {
/* Special case when first element is generic object
with buffer interface and the second element is a
contiguous "shadow" that must be copied back into
@@ -454,8 +487,8 @@
if (PyObject_GetBuffer((PyObject *)self, &view, PyBUF_FULL) < 0)
return NULL;
- res = PyByteArray_FromStringAndSize(NULL, view.len);
- PyBuffer_ToContiguous(PyByteArray_AS_STRING(res), &view, view.len, 'C');
+ res = PyBytes_FromStringAndSize(NULL, view.len);
+ PyBuffer_ToContiguous(PyBytes_AS_STRING(res), &view, view.len, 'C');
PyBuffer_Release(&view);
return res;
}
@@ -511,7 +544,7 @@
if (result < 0) {
result += view->shape[0];
}
- if ((result < 0) || (result > view->shape[0])) {
+ if ((result < 0) || (result >= view->shape[0])) {
PyErr_SetString(PyExc_IndexError,
"index out of bounds");
return NULL;
@@ -525,7 +558,7 @@
{
ptr = *((char **)ptr) + view->suboffsets[0];
}
- return PyByteArray_FromStringAndSize(ptr, view->itemsize);
+ return PyBytes_FromStringAndSize(ptr, view->itemsize);
}
else {
/* Return a new memory-view object */
@@ -537,10 +570,46 @@
return PyMemoryView_FromBuffer(&newview);
}
}
+ else if (PySlice_Check(key)) {
+ Py_ssize_t start, stop, step, slicelength;
+
+ if (PySlice_GetIndicesEx((PySliceObject*)key, view->len,
+ &start, &stop, &step, &slicelength) < 0) {
+ return NULL;
+ }
- /* Need to support getting a sliced view */
- Py_INCREF(Py_NotImplemented);
- return Py_NotImplemented;
+ if (step == 1 && view->ndim == 1) {
+ Py_buffer newview;
+ void *newbuf = (char *) view->buf
+ + start * view->itemsize;
+ int newflags = view->readonly
+ ? PyBUF_CONTIG_RO : PyBUF_CONTIG;
+
+ /* XXX There should be an API to create a subbuffer */
+ if (view->obj != NULL) {
+ if (PyObject_GetBuffer(view->obj,
+ &newview, newflags) == -1)
+ return NULL;
+ }
+ else {
+ newview = *view;
+ }
+ newview.buf = newbuf;
+ newview.len = slicelength;
+ newview.format = view->format;
+ if (view->shape == &(view->len))
+ newview.shape = &(newview.len);
+ if (view->strides == &(view->itemsize))
+ newview.strides = &(newview.itemsize);
+ return PyMemoryView_FromBuffer(&newview);
+ }
+ PyErr_SetNone(PyExc_NotImplementedError);
+ return NULL;
+ }
+ PyErr_Format(PyExc_TypeError,
+ "cannot index memory using \"%.200s\"",
+ key->ob_type->tp_name);
+ return NULL;
}
@@ -548,9 +617,138 @@
static int
memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
{
- return 0;
+ Py_ssize_t start, len, bytelen, i;
+ Py_buffer srcview;
+ Py_buffer *view = &(self->view);
+ char *srcbuf, *destbuf;
+
+ if (view->readonly) {
+ PyErr_SetString(PyExc_TypeError,
+ "cannot modify read-only memory");
+ return -1;
+ }
+ if (view->ndim != 1) {
+ PyErr_SetNone(PyExc_NotImplementedError);
+ return -1;
+ }
+ if (PyIndex_Check(key)) {
+ start = PyNumber_AsSsize_t(key, NULL);
+ if (start == -1 && PyErr_Occurred())
+ return -1;
+ if (start < 0) {
+ start += view->shape[0];
+ }
+ if ((start < 0) || (start >= view->shape[0])) {
+ PyErr_SetString(PyExc_IndexError,
+ "index out of bounds");
+ return -1;
+ }
+ len = 1;
+ }
+ else if (PySlice_Check(key)) {
+ Py_ssize_t stop, step;
+
+ if (PySlice_GetIndicesEx((PySliceObject*)key, view->len,
+ &start, &stop, &step, &len) < 0) {
+ return -1;
+ }
+ if (step != 1) {
+ PyErr_SetNone(PyExc_NotImplementedError);
+ return -1;
+ }
+ }
+ else {
+ PyErr_Format(PyExc_TypeError,
+ "cannot index memory using \"%.200s\"",
+ key->ob_type->tp_name);
+ return -1;
+ }
+ if (PyObject_GetBuffer(value, &srcview, PyBUF_CONTIG_RO) == -1) {
+ return -1;
+ }
+ /* XXX should we allow assignment of different item sizes
+ as long as the byte length is the same?
+ (e.g. assign 2 shorts to a 4-byte slice) */
+ if (srcview.itemsize != view->itemsize) {
+ PyErr_Format(PyExc_TypeError,
+ "mismatching item sizes for \"%.200s\" and \"%.200s\"",
+ view->obj->ob_type->tp_name, srcview.obj->ob_type->tp_name);
+ goto _error;
+ }
+ if (srcview.len != len) {
+ PyErr_SetString(PyExc_ValueError,
+ "cannot modify size of memoryview object");
+ goto _error;
+ }
+ /* Do the actual copy */
+ destbuf = (char *) view->buf + start * view->itemsize;
+ srcbuf = (char *) srcview.buf;
+ bytelen = len * view->itemsize;
+ if (destbuf + bytelen < srcbuf || srcbuf + bytelen < destbuf)
+ /* No overlapping */
+ memcpy(destbuf, srcbuf, bytelen);
+ else if (destbuf < srcbuf) {
+ /* Copy in ascending order */
+ for (i = 0; i < bytelen; i++)
+ destbuf[i] = srcbuf[i];
+ }
+ else {
+ /* Copy in descencing order */
+ for (i = bytelen - 1; i >= 0; i--)
+ destbuf[i] = srcbuf[i];
+ }
+
+ PyBuffer_Release(&srcview);
+ return 0;
+
+_error:
+ PyBuffer_Release(&srcview);
+ return -1;
+}
+
+static PyObject *
+memory_richcompare(PyObject *v, PyObject *w, int op)
+{
+ Py_buffer vv, ww;
+ int equal = 0;
+ PyObject *res;
+
+ vv.obj = NULL;
+ ww.obj = NULL;
+ if (op != Py_EQ && op != Py_NE)
+ goto _notimpl;
+ if (PyObject_GetBuffer(v, &vv, PyBUF_CONTIG_RO) == -1) {
+ PyErr_Clear();
+ goto _notimpl;
+ }
+ if (PyObject_GetBuffer(w, &ww, PyBUF_CONTIG_RO) == -1) {
+ PyErr_Clear();
+ goto _notimpl;
+ }
+
+ if (vv.itemsize != ww.itemsize || vv.len != ww.len)
+ goto _end;
+
+ equal = !memcmp(vv.buf, ww.buf, vv.len * vv.itemsize);
+
+_end:
+ PyBuffer_Release(&vv);
+ PyBuffer_Release(&ww);
+ if ((equal && op == Py_EQ) || (!equal && op == Py_NE))
+ res = Py_True;
+ else
+ res = Py_False;
+ Py_INCREF(res);
+ return res;
+
+_notimpl:
+ PyBuffer_Release(&vv);
+ PyBuffer_Release(&ww);
+ Py_INCREF(Py_NotImplemented);
+ return Py_NotImplemented;
}
+
/* As mapping */
static PyMappingMethods memory_as_mapping = {
(lenfunc)memory_length, /*mp_length*/
@@ -591,7 +789,7 @@
memory_doc, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
- 0, /* tp_richcompare */
+ memory_richcompare, /* tp_richcompare */
0, /* tp_weaklistoffset */
0, /* tp_iter */
0, /* tp_iternext */
More information about the Python-3000-checkins
mailing list