[Jython-checkins] jython: PyArray.fromstring accepts buffer API objects
jeff.allen
jython-checkins at python.org
Fri Nov 15 00:01:38 CET 2013
http://hg.python.org/jython/rev/db2d93777787
changeset: 7150:db2d93777787
user: Jeff Allen <ja.py at farowl.co.uk>
date: Mon Nov 11 22:21:12 2013 +0000
summary:
PyArray.fromstring accepts buffer API objects
Also test in test_array to accept sliced/strided buffers.
files:
Lib/test/test_array.py | 35 ++++-
src/org/python/core/PyArray.java | 143 ++++++++++++++++--
2 files changed, 159 insertions(+), 19 deletions(-)
diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py
--- a/Lib/test/test_array.py
+++ b/Lib/test/test_array.py
@@ -246,6 +246,39 @@
self.assertEqual(a, b)
if a.itemsize>1 and self.typecode not in ('b', 'B'):
self.assertRaises(ValueError, b.fromstring, "x")
+ # Test from byte string available via buffer API (Jython addition)
+ if test_support.is_jython:
+ for buftype in (buffer, memoryview, bytearray):
+ b = array.array(self.typecode)
+ b.fromstring(buftype(a.tostring()))
+ self.assertEqual(a, b)
+
+ @unittest.skipUnless(test_support.is_jython, "Jython supports memoryview slices")
+ def test_tofromstring_sliced(self):
+ a = array.array(self.typecode, self.example)
+ r = bytearray(a.tostring())
+ R = len(r)
+ D = 3*R
+
+ def checkSlice(x, y, z=None):
+ # Scatter the bytes of a.tostring() into d
+ d = bytearray(D)
+ d[x:y:z] = r
+ # Now gather through a memoryview slice
+ with memoryview(d) as m:
+ # Requires proper use of strides when z not None and not 1
+ b = array.array(self.typecode)
+ b.fromstring(m[x:y:z])
+ self.assertEqual(a, b)
+
+ # The slices all have R elements and the whole range D = 3*R
+ checkSlice(None, R)
+ checkSlice(2, 2+R)
+ checkSlice(D-R, None)
+ checkSlice(None, None, 3)
+ checkSlice(None, None, -3)
+ checkSlice(None, D-R-1, -1)
+ checkSlice(R-1, None, -1)
def test_filewrite(self):
a = array.array(self.typecode, 2*self.example)
@@ -828,7 +861,7 @@
# SF bug #1486663 -- this used to erroneously raise a TypeError
ArraySubclassWithKwargs('b', newarg=1)
- @unittest.skipIf(not test_support.is_jython, "array supports buffer interface in Jython")
+ @unittest.skipUnless(test_support.is_jython, "array supports buffer interface in Jython")
def test_resize_forbidden(self):
# Test that array resizing is forbidden with buffer exports (Jython addition).
# Test adapted from corresponding one in test_bytes.
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -77,19 +77,26 @@
@ExposedNew
static final PyObject array_new(PyNewWrapper new_, boolean init, PyType subtype,
PyObject[] args, String[] keywords) {
+
if (new_.for_type != subtype && keywords.length > 0) {
+ /*
+ * We're constructing as a base for a derived type (via PyDerived) and there are
+ * keywords. The effective args locally should not include the keywords.
+ */
int argc = args.length - keywords.length;
PyObject[] justArgs = new PyObject[argc];
System.arraycopy(args, 0, justArgs, 0, argc);
args = justArgs;
}
+
+ // Build the argument parser for this call
ArgParser ap =
new ArgParser("array", args, Py.NoKeywords,
new String[] {"typecode", "initializer"}, 1);
ap.noKeywords();
+
+ // Retrieve the mandatory type code that determines the element type
PyObject obj = ap.getPyObject(0);
- PyObject initial = ap.getPyObject(1, null);
-
Class<?> type;
String typecode;
if (obj instanceof PyString && !(obj instanceof PyUnicode)) {
@@ -106,32 +113,48 @@
+ obj.getType().fastGetName());
}
+ /*
+ * Create a 'blank canvas' of the appropriate concrete class.
+ */
PyArray self;
if (new_.for_type == subtype) {
self = new PyArray(subtype);
} else {
self = new PyArrayDerived(subtype);
}
+
// Initialize the typecode (and validate type) before creating the backing Array
class2char(type);
self.setup(type, Array.newInstance(type, 0));
self.typecode = typecode;
+
+ /*
+ * The initialiser may be omitted, or may validly be one of several types in the broad
+ * categories of a byte string (which is treated as a machine representation of the data) or
+ * an iterable yielding values assignable to the elements. There is special treatment for
+ * type 'u' Unicode.
+ */
+ PyObject initial = ap.getPyObject(1, null);
if (initial == null) {
- return self;
- }
- if (initial instanceof PyList) {
+ // Fall through
+
+ } else if (initial instanceof PyList) {
self.fromlist(initial);
+
} else if (initial instanceof PyString && !(initial instanceof PyUnicode)) {
self.fromstring(initial.toString());
+
} else if ("u".equals(typecode)) {
if (initial instanceof PyUnicode) {
self.extendArray(((PyUnicode)initial).toCodePoints());
} else {
self.extendUnicodeIter(initial);
}
+
} else {
self.extendInternal(initial);
}
+
return self;
}
@@ -707,7 +730,7 @@
* Append items from <code>iterable</code> to the end of the array. If iterable is another
* array, it must have exactly the same type code; if not, TypeError will be raised. If iterable
* is not an array, it must be iterable and its elements must be the right type to be appended
- * to the array. Changed in version 2.4: Formerly, the argument could only be another array.
+ * to the array.
*
* @param iterable iterable object used to extend the array
*/
@@ -733,8 +756,9 @@
throw Py.TypeError("an integer is required");
}
- } else if (iterable instanceof PyString) {
- fromstring(((PyString)iterable).toString());
+// } else if (iterable instanceof PyString) {
+// // XXX CPython treats a str/bytes as an iterable, not as previously here:
+// fromstring(((PyString)iterable).toString());
} else if (iterable instanceof PyArray) {
PyArray source = (PyArray)iterable;
@@ -1067,12 +1091,14 @@
}
/**
- * Appends items from the string, interpreting the string as an array of machine values (as if
- * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+ * Appends items from the object, which is a byte string of some kind (PyString or object with
+ * the buffer interface providing bytes) The string of bytes is interpreted as an array of
+ * machine values (as if it had been read from a file using the {@link #fromfile(PyObject, int)
+ * fromfile()} method).
*
* @param input string of bytes containing array data
*/
- public void fromstring(String input) {
+ public void fromstring(PyObject input) {
array_fromstring(input);
}
@@ -1082,27 +1108,108 @@
*
* @param input string of bytes containing array data
*/
+ public void fromstring(String input) {
+ frombytesInternal(StringUtil.toBytes(input));
+ }
+
+ /**
+ * Appends items from the string, interpreting the string as an array of machine values (as if
+ * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+ *
+ * @param input string of bytes containing array data
+ */
@ExposedMethod
- final void array_fromstring(String input) {
+ final void array_fromstring(PyObject input) {
+
+ if (input instanceof BufferProtocol) {
+
+ if (input instanceof PyUnicode) {
+ // Unicode is treated as specifying a byte string via the default encoding.
+ String s = ((PyUnicode)input).encode();
+ frombytesInternal(StringUtil.toBytes(s));
+
+ } else {
+ // Access the bytes
+ PyBuffer pybuf = ((BufferProtocol)input).getBuffer(PyBUF.STRIDED_RO);
+ try {
+ // Provide argument as stream of bytes for fromstream method
+ if (pybuf.getNdim() == 1) {
+ if (pybuf.getStrides()[0] == 1) {
+ // Data are contiguous in a byte[]
+ PyBuffer.Pointer b = pybuf.getBuf();
+ frombytesInternal(b.storage, b.offset, pybuf.getLen());
+ } else {
+ // As frombytesInternal only knows contiguous bytes, make a copy.
+ byte[] copy = new byte[pybuf.getLen()];
+ pybuf.copyTo(copy, 0);
+ frombytesInternal(copy);
+ }
+ } else {
+ // Currently don't support n-dimensional sources
+ throw Py.ValueError("multi-dimensional buffer not supported");
+ }
+ } finally {
+ pybuf.release();
+ }
+ }
+
+ } else {
+ String fmt = "must be string or read-only buffer, not %s";
+ throw Py.TypeError(String.format(fmt, input.getType().fastGetName()));
+ }
+ }
+
+ /**
+ * Common code supporting Java and Python versions of <code>.fromstring()</code>
+ *
+ * @param input string of bytes encoding the array data
+ */
+ private final void fromstringInternal(String input) {
+ frombytesInternal(StringUtil.toBytes(input));
+ }
+
+ /**
+ * Common code supporting Java and Python versions of <code>.fromstring()</code> or
+ * <code>.frombytes()</code> (Python 3.2+ name).
+ *
+ * @param bytes array containing the new array data in machine encoding
+ */
+ private final void frombytesInternal(byte[] bytes) {
+ frombytesInternal(bytes, 0, bytes.length);
+ }
+
+ /**
+ * Common code supporting Java and Python versions of <code>.fromstring()</code> or
+ * <code>.frombytes()</code> (Python 3.2+ name).
+ *
+ * @param bytes array containing the new array data in machine encoding
+ * @param offset of the first byte to read
+ * @param count of bytes to read
+ */
+ private final void frombytesInternal(byte[] bytes, int offset, int count) {
+
+ // Access the bytes
+ int origsize = delegate.getSize();
// Check validity wrt array itemsize
int itemsize = getStorageSize();
- int strlen = input.length();
- if ((strlen % itemsize) != 0) {
+ if ((count % itemsize) != 0) {
throw Py.ValueError("string length not a multiple of item size");
}
- // Prohibited operation if exporting a buffer
+ // Prohibited operation if we are exporting a buffer
resizeCheck();
- // Provide argument as stream of bytes for fromstream method
- ByteArrayInputStream bis = new ByteArrayInputStream(StringUtil.toBytes(input));
- int origsize = delegate.getSize();
try {
+
+ // Provide argument as stream of bytes for fromstream method
+ ByteArrayInputStream bis = new ByteArrayInputStream(bytes, offset, count);
fromStream(bis);
+
} catch (EOFException e) {
// stubbed catch for fromStream throws
throw Py.EOFError("not enough items in string");
+
} catch (IOException e) {
// discard anything successfully loaded
delegate.setSize(origsize);
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list