[Jython-checkins] jython: _io.FileIO: support readinto/write array.array
jeff.allen
jython-checkins at python.org
Tue Dec 25 02:05:19 CET 2012
http://hg.python.org/jython/rev/b2523e388129
changeset: 6913:b2523e388129
user: Jeff Allen <ja...py at farowl.co.uk>
date: Thu Dec 20 17:06:22 2012 +0000
summary:
_io.FileIO: support readinto/write array.array
Make PyArray.tostream public and add a suitable public fromstream and use them from PyFileIO.
test_io now scores fail/error/skip = 11/15/99, test_fileio = 0/0/1 and test_array = 0/0/12 (unchanged).
files:
Lib/_jyio.py | 6 +-
src/org/python/core/PyArray.java | 236 ++++++----
src/org/python/modules/_io/PyFileIO.java | 93 ++-
3 files changed, 209 insertions(+), 126 deletions(-)
diff --git a/Lib/_jyio.py b/Lib/_jyio.py
--- a/Lib/_jyio.py
+++ b/Lib/_jyio.py
@@ -27,6 +27,7 @@
import codecs
import warnings
import errno
+import array
# Import thread instead of threading to reduce startup cost
try:
from thread import allocate_lock as Lock
@@ -555,7 +556,10 @@
# raise BlockingIOError with characters_written == 0.)
self._flush_unlocked()
before = len(self._write_buf)
- self._write_buf.extend(b)
+ if isinstance(b, array.array): # _pyio.py version fails on array.array
+ self._write_buf.extend(b.tostring()) # Jython version works (while needed)
+ else:
+ self._write_buf.extend(b)
written = len(self._write_buf) - before
if len(self._write_buf) > self.buffer_size:
try:
diff --git a/src/org/python/core/PyArray.java b/src/org/python/core/PyArray.java
--- a/src/org/python/core/PyArray.java
+++ b/src/org/python/core/PyArray.java
@@ -858,100 +858,146 @@
* @throws EOFException
*/
private int fromStream(InputStream is, int count) throws IOException, EOFException {
- DataInputStream dis = new DataInputStream(is);
- // current number of items present
+
+ // Current number of items present
int origsize = delegate.getSize();
- // position to start inserting into
- int index = origsize;
- // create capacity for 'count' items
- delegate.ensureCapacity(index + count);
- if (type.isPrimitive()) {
- switch (typecode.charAt(0)) {
- case 'z':
- for (int i = 0; i < count; i++, index++) {
- Array.setBoolean(data, index, dis.readBoolean());
- delegate.size++;
- }
- break;
- case 'b':
- for (int i = 0; i < count; i++, index++) {
- Array.setByte(data, index, dis.readByte());
- delegate.size++;
- }
- break;
- case 'B':
- for (int i = 0; i < count; i++, index++) {
- Array.setShort(data, index, unsignedByte(dis.readByte()));
- delegate.size++;
- }
- break;
- case 'u':
- // use 32-bit integers since we want UCS-4 storage
- for (int i = 0; i < count; i++, index++) {
- Array.setInt(data, index, dis.readInt());
- delegate.size++;
- }
- break;
- case 'c':
- for (int i = 0; i < count; i++, index++) {
- Array.setChar(data, index, (char)(dis.readByte() & 0xff));
- delegate.size++;
- }
- break;
- case 'h':
- for (int i = 0; i < count; i++, index++) {
- Array.setShort(data, index, dis.readShort());
- delegate.size++;
- }
- break;
- case 'H':
- for (int i = 0; i < count; i++, index++) {
- Array.setInt(data, index, unsignedShort(dis.readShort()));
- delegate.size++;
- }
- break;
- case 'i':
- for (int i = 0; i < count; i++, index++) {
- Array.setInt(data, index, dis.readInt());
- delegate.size++;
- }
- break;
- case 'I':
- for (int i = 0; i < count; i++, index++) {
- Array.setLong(data, index, unsignedInt(dis.readInt()));
- delegate.size++;
- }
- break;
- case 'l':
- for (int i = 0; i < count; i++, index++) {
- Array.setLong(data, index, dis.readLong());
- delegate.size++;
- }
- break;
- case 'L': // faking it
- for (int i = 0; i < count; i++, index++) {
- Array.setLong(data, index, dis.readLong());
- delegate.size++;
- }
- break;
- case 'f':
- for (int i = 0; i < count; i++, index++) {
- Array.setFloat(data, index, dis.readFloat());
- delegate.size++;
- }
- break;
- case 'd':
- for (int i = 0; i < count; i++, index++) {
- Array.setDouble(data, index, dis.readDouble());
- delegate.size++;
- }
- break;
- }
- }
- dis.close();
- return (index - origsize);
+
+ // Reserve capacity for 'count' items
+ delegate.setSize(origsize + count);
+
+ // Read into the array, after the current contents, up to new size (or EOF thrown)
+ int n = fromStream(is, origsize, delegate.getSize(), true);
+ return n - origsize;
}
+ /**
+ * Read primitive values from a stream into the array without resizing. Data is read until the
+ * array is filled or the stream runs out. If the stream does not contain a whole number of
+ * items (possible if the item size is not one byte), the behaviour in respect of the final
+ * partial item and straem position is not defined.
+ *
+ * @param is InputStream to source the data from
+ * @return number of primitives successfully read
+ * @throws IOException reflecting I/O errors during reading
+ */
+ public int fillFromStream(InputStream is) throws IOException {
+ return fromStream(is, 0, delegate.size, false);
+ }
+
+ /**
+ * Helper for reading primitive values from a stream into a slice of the array. Data is read
+ * until the array slice is filled or the stream runs out. The purpose of the method is to
+ * concentrate in one place the manipulation of bytes into the several primitive element types
+ * on behalf of {@link #fillFromStream(InputStream)} etc.. Since different read methods respond
+ * differently to it, the caller must specify whether the exhaustion of the stream (EOF) should
+ * be treated as an error or not. If the stream does not contain a whole number of items
+ * (possible if the item size is not one byte), the behaviour in respect of the final partial
+ * item and stream position is not defined.
+ *
+ * @param dis data stream source for the values
+ * @param index first element index to read
+ * @param limit first element index <b>not</b> to read
+ * @param eofIsError if true, treat EOF as expected way to end
+ * @return index of first element not read (<code>=limit</code>, if not ended by EOF)
+ * @throws IOException reflecting I/O errors during reading
+ * @throws EOFException if stream ends before read is satisfied and eofIsError is true
+ */
+ private int fromStream(InputStream is, int index, int limit, boolean eofIsError)
+ throws IOException, EOFException {
+
+ // We need a wrapper capable of encoding the data
+ DataInputStream dis = new DataInputStream(is);
+
+ try {
+ // We have to deal with each primitive type as a distinct case
+ if (type.isPrimitive()) {
+ switch (typecode.charAt(0)) {
+ case 'z':
+ for (; index < limit; index++) {
+ Array.setBoolean(data, index, dis.readBoolean());
+ }
+ break;
+ case 'b':
+ for (; index < limit; index++) {
+ Array.setByte(data, index, dis.readByte());
+ }
+ break;
+ case 'B':
+ for (; index < limit; index++) {
+ Array.setShort(data, index, unsignedByte(dis.readByte()));
+ }
+ break;
+ case 'u':
+ // use 32-bit integers since we want UCS-4 storage
+ for (; index < limit; index++) {
+ Array.setInt(data, index, dis.readInt());
+ }
+ break;
+ case 'c':
+ for (; index < limit; index++) {
+ Array.setChar(data, index, (char)(dis.readByte() & 0xff));
+ }
+ break;
+ case 'h':
+ for (; index < limit; index++) {
+ Array.setShort(data, index, dis.readShort());
+ }
+ break;
+ case 'H':
+ for (; index < limit; index++) {
+ Array.setInt(data, index, unsignedShort(dis.readShort()));
+ }
+ break;
+ case 'i':
+ for (; index < limit; index++) {
+ Array.setInt(data, index, dis.readInt());
+ }
+ break;
+ case 'I':
+ for (; index < limit; index++) {
+ Array.setLong(data, index, unsignedInt(dis.readInt()));
+ }
+ break;
+ case 'l':
+ for (; index < limit; index++) {
+ Array.setLong(data, index, dis.readLong());
+ }
+ break;
+ case 'L': // faking it
+ for (; index < limit; index++) {
+ Array.setLong(data, index, dis.readLong());
+ }
+ break;
+ case 'f':
+ for (; index < limit; index++) {
+ Array.setFloat(data, index, dis.readFloat());
+ }
+ break;
+ case 'd':
+ for (; index < limit; index++) {
+ Array.setDouble(data, index, dis.readDouble());
+ }
+ break;
+ }
+ }
+
+ } catch (EOFException eof) {
+ if (eofIsError) {
+ throw eof;
+ }
+ // EOF = end of reading: excess odd bytes read inside dis.readXXX() discarded
+ }
+
+ // index points to the first element *not* written
+ return index;
+ }
+
+ /**
+ * Appends items from the string, interpreting the string as an array of machine values (as if
+ * it had been read from a file using the {@link #fromfile(PyObject, int) fromfile()} method).
+ *
+ * @param input string of bytes containing array data
+ */
public void fromstring(String input) {
array_fromstring(input);
}
@@ -1568,12 +1614,10 @@
* types.
*
* @param os OutputStream to sink the array data to
- *
- * @return number of primitives successfully written
- *
+ * @return number of bytes successfully written
* @throws IOException
*/
- private int toStream(OutputStream os) throws IOException {
+ public int toStream(OutputStream os) throws IOException {
DataOutputStream dos = new DataOutputStream(os);
switch (typecode.charAt(0)) {
case 'z':
@@ -1643,7 +1687,7 @@
}
break;
}
- return dos.size();
+ return dos.size(); // bytes written
}
private static byte signedByte(short x) {
diff --git a/src/org/python/modules/_io/PyFileIO.java b/src/org/python/modules/_io/PyFileIO.java
--- a/src/org/python/modules/_io/PyFileIO.java
+++ b/src/org/python/modules/_io/PyFileIO.java
@@ -1,13 +1,20 @@
/* Copyright (c)2012 Jython Developers */
package org.python.modules._io;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channel;
+import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
import org.python.core.ArgParser;
import org.python.core.BuiltinDocs;
import org.python.core.Py;
+import org.python.core.PyArray;
import org.python.core.PyBuffer;
import org.python.core.PyException;
import org.python.core.PyJavaType;
@@ -61,7 +68,8 @@
/** The mode as a PyString based on readable and writable */
@ExposedGet(doc = "String giving the file mode: 'rb', 'rb+', or 'wb'")
public final PyString mode;
- @ExposedSet(name="mode")
+
+ @ExposedSet(name = "mode")
public final void mode_readonly(PyString value) {
readonlyAttributeError("mode");
}
@@ -218,26 +226,41 @@
@ExposedMethod(doc = readinto_doc)
final PyLong FileIO_readinto(PyObject buf) {
-
+ int count;
if (!readable) { // ... (or closed)
throw tailoredValueError("read");
}
- // Perform the operation through a buffer view on the object
- PyBuffer pybuf = writablePyBuffer(buf);
- try {
- PyBuffer.Pointer bp = pybuf.getBuf();
- ByteBuffer byteBuffer = ByteBuffer.wrap(bp.storage, bp.offset, pybuf.getLen());
- int count;
- synchronized (ioDelegate) {
- count = ioDelegate.readinto(byteBuffer);
+ if (buf instanceof PyArray) {
+ // Special case: PyArray knows how to read into itself
+ PyArray a = (PyArray)buf;
+
+ try {
+ ReadableByteChannel ch = ioDelegate.getChannel();
+ InputStream is = Channels.newInputStream(ch);
+ count = a.fillFromStream(is);
+ count *= a.getItemsize();
+ } catch (IOException ioe) {
+ throw Py.IOError(ioe);
}
- return new PyLong(count);
- } finally {
- // Must unlock the PyBuffer view from client's object
- pybuf.release();
+ } else {
+ // Perform the operation through a buffer view on the object
+ PyBuffer pybuf = writablePyBuffer(buf);
+
+ try {
+ PyBuffer.Pointer bp = pybuf.getBuf();
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bp.storage, bp.offset, pybuf.getLen());
+ synchronized (ioDelegate) {
+ count = ioDelegate.readinto(byteBuffer);
+ }
+ } finally {
+ // Must unlock the PyBuffer view from client's object
+ pybuf.release();
+ }
}
+
+ return new PyLong(count);
}
@Override
@@ -246,28 +269,40 @@
}
@ExposedMethod(doc = write_doc)
- final PyLong FileIO_write(PyObject obj) {
-
+ final PyLong FileIO_write(PyObject buf) {
+ int count;
if (!writable) { // ... (or closed)
throw tailoredValueError("writ");
}
- // Get or synthesise a buffer API on the object to be written
- PyBuffer pybuf = readablePyBuffer(obj);
- try {
- // Access the data as a java.nio.ByteBuffer [pos:limit] within possibly larger array
- PyBuffer.Pointer bp = pybuf.getBuf();
- ByteBuffer byteBuffer = ByteBuffer.wrap(bp.storage, bp.offset, pybuf.getLen());
- int count;
- synchronized (ioDelegate) {
- count = ioDelegate.write(byteBuffer);
+ if (buf instanceof PyArray) {
+ // Special case: PyArray knows how to write itself
+ try {
+ WritableByteChannel ch = ioDelegate.getChannel();
+ OutputStream os = Channels.newOutputStream(ch);
+ count = ((PyArray)buf).toStream(os);
+ } catch (IOException ioe) {
+ throw Py.IOError(ioe);
}
- return new PyLong(count);
- } finally {
- // Even if that went badly, we should release the lock on the client buffer
- pybuf.release();
+ } else {
+ // Get or synthesise a buffer API on the object to be written
+ PyBuffer pybuf = readablePyBuffer(buf);
+
+ try {
+ // Access the data as a java.nio.ByteBuffer [pos:limit] within possibly larger array
+ PyBuffer.Pointer bp = pybuf.getBuf();
+ ByteBuffer byteBuffer = ByteBuffer.wrap(bp.storage, bp.offset, pybuf.getLen());
+ synchronized (ioDelegate) {
+ count = ioDelegate.write(byteBuffer);
+ }
+ } finally {
+ // Even if that went badly, we should release the lock on the client buffer
+ pybuf.release();
+ }
}
+
+ return new PyLong(count);
}
@Override
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list