[Jython-checkins] jython: Allow _io.FileIO to wrap more streams
jeff.allen
jython-checkins at python.org
Sat Feb 9 20:30:06 CET 2013
http://hg.python.org/jython/rev/e5ab567a191b
changeset: 7022:e5ab567a191b
parent: 6961:98b9c3a9076e
user: Jeff Allen <ja...py at farowl.co.uk>
date: Sat Feb 09 18:55:49 2013 +0000
summary:
Allow _io.FileIO to wrap more streams
The fileno() return from PyFile would sometimes be a StreamIO, notably in the
case of the streams sys.(stdin|stdout|stderr), which are now acceptable to
_io.open and _io.FileIO. Changes close semantics slightly (hence change to test_io).
files:
Lib/test/test_io.py | 4 +-
src/org/python/modules/_io/PyFileIO.java | 69 ++--
tests/java/org/python/modules/_io/_ioTest.java | 120 ++++++++-
3 files changed, 134 insertions(+), 59 deletions(-)
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -2552,8 +2552,8 @@
self.assertEqual(g.raw.mode, "wb")
self.assertEqual(g.name, f.fileno())
self.assertEqual(g.raw.name, f.fileno())
- f.close()
- g.close()
+ g.close() # Jython difference: close g first (which may flush) ...
+ f.close() # Jython difference: then close f, which closes the fd
def test_io_after_close(self):
for kwargs in [
diff --git a/src/org/python/modules/_io/PyFileIO.java b/src/org/python/modules/_io/PyFileIO.java
--- a/src/org/python/modules/_io/PyFileIO.java
+++ b/src/org/python/modules/_io/PyFileIO.java
@@ -1,15 +1,10 @@
-/* Copyright (c)2012 Jython Developers */
+/* Copyright (c)2013 Jython Developers */
package org.python.modules._io;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
-import java.nio.channels.Channel;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.channels.WritableByteChannel;
import org.python.core.ArgParser;
import org.python.core.BuiltinDocs;
@@ -26,6 +21,7 @@
import org.python.core.PyUnicode;
import org.python.core.io.FileIO;
import org.python.core.io.RawIOBase;
+import org.python.core.io.StreamIO;
import org.python.expose.ExposedGet;
import org.python.expose.ExposedMethod;
import org.python.expose.ExposedNew;
@@ -39,8 +35,8 @@
public static final PyType TYPE = PyType.fromClass(PyFileIO.class);
- /** The FileIO to which we delegate operations not complete locally. */
- private FileIO ioDelegate;
+ /** The {@link FileIO} or {@link StreamIO} to which we delegate operations not complete locally. */
+ private RawIOBase ioDelegate;
/*
* Implementation note: CPython fileio does not use the base-class, possibly overridden,
@@ -105,12 +101,15 @@
*/
public PyFileIO(PyType subtype, PyObject file, OpenMode mode, boolean closefd) {
super(subtype);
- setDelegate(file, mode, closefd);
- this.closefd = closefd;
+ // Establish the direction(s) of flow
readable = mode.reading | mode.updating;
writable = mode.writing | mode.updating | mode.appending;
+ // Assign a delegate according to the file argument
+ this.closefd = closefd;
+ setDelegate(file, mode);
+
// The mode string of a raw file always asserts it is binary: "rb", "rb+", or "wb".
if (readable) {
this.mode = new PyString(writable ? "rb+" : "rb");
@@ -132,9 +131,8 @@
*
* @param file name or descriptor
* @param mode parsed file open mode
- * @param closefd must be true if file is in fact a name (checked, not used)
*/
- private void setDelegate(PyObject file, OpenMode mode, boolean closefd) {
+ private void setDelegate(PyObject file, OpenMode mode) {
if (file instanceof PyString) {
// Open a file by name
@@ -151,23 +149,13 @@
*/
Object fd = file.__tojava__(Object.class);
- if (fd instanceof RawIOBase) {
- // It is the "Jython file descriptor" from which we can get a channel.
+ if (fd instanceof FileIO || fd instanceof StreamIO) {
/*
- * If the argument were a FileIO, could we assign it directly to ioDelegate? I think
- * not: there would be a problem with close and closefd=False since it is not the
- * PyFileIO that keeps state.
+ * It is the "Jython file descriptor", of a type suitable to be the ioDelegate. The
+ * allowed types are able to give us a non-null InputStream or OutputStream,
+ * according to direction.
*/
- Channel channel = ((RawIOBase)fd).getChannel();
- if (channel instanceof FileChannel) {
- if (channel.isOpen()) {
- FileChannel fc = (FileChannel)channel;
- ioDelegate = new FileIO(fc, mode.forFileIO());
- } else {
- // File not open (we have to check as FileIO doesn't)
- throw Py.OSError(Errno.EBADF);
- }
- }
+ ioDelegate = (RawIOBase)fd;
}
}
@@ -175,11 +163,22 @@
if (ioDelegate == null) {
// The file was a type we don't know how to use
throw Py.TypeError(String.format("invalid file: %s", file.__repr__().asString()));
+
+ } else {
+
+ if (ioDelegate.closed()) {
+ // A closed file descriptor is a "bad descriptor"
+ throw Py.OSError(Errno.EBADF);
+ }
+
+ if ((readable && !ioDelegate.readable()) || (writable && !ioDelegate.writable())) {
+ // Requested mode in conflict with underlying file or stream
+ throw tailoredValueError(readable ? "read" : "writ");
+ }
+
+ // The name is either the textual name or a file descriptor (see Python docs)
+ fastGetDict().__setitem__("name", file);
}
-
- // The name is either the textual name or a file descriptor (see Python docs)
- fastGetDict().__setitem__("name", file);
-
}
private static final String[] openArgs = {"file", "mode", "closefd"};
@@ -239,8 +238,8 @@
PyArray a = (PyArray)buf;
try {
- ReadableByteChannel ch = ioDelegate.getChannel();
- InputStream is = Channels.newInputStream(ch);
+ // The ioDelegate, if readable, can always provide an InputStream (see setDelegate)
+ InputStream is = ioDelegate.asInputStream();
count = a.fillFromStream(is);
count *= a.getItemsize();
} catch (IOException ioe) {
@@ -281,8 +280,8 @@
if (buf instanceof PyArray) {
// Special case: PyArray knows how to write itself
try {
- WritableByteChannel ch = ioDelegate.getChannel();
- OutputStream os = Channels.newOutputStream(ch);
+ // The ioDelegate, if writable, can always provide an OutputStream (see setDelegate)
+ OutputStream os = ioDelegate.asOutputStream();
count = ((PyArray)buf).toStream(os);
} catch (IOException ioe) {
throw Py.IOError(ioe);
diff --git a/tests/java/org/python/modules/_io/_ioTest.java b/tests/java/org/python/modules/_io/_ioTest.java
--- a/tests/java/org/python/modules/_io/_ioTest.java
+++ b/tests/java/org/python/modules/_io/_ioTest.java
@@ -5,9 +5,15 @@
import static org.junit.matchers.JUnitMatchers.*;
import java.io.File;
+import java.io.FileDescriptor;
import java.io.FileInputStream;
-import java.io.IOError;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
import java.util.Arrays;
import org.junit.Before;
@@ -18,6 +24,7 @@
import org.python.core.PyObject;
import org.python.core.PyStringMap;
import org.python.core.PySystemState;
+import org.python.core.imp;
import org.python.core.io.RawIOBase;
import org.python.util.PythonInterpreter;
@@ -28,7 +35,12 @@
*/
public class _ioTest {
- /** We need the interpreter to be initialised for these tests **/
+ // Some file names to use
+ private final String FILE1 = "$test_1_tmp";
+ private final String FILE2 = "$test_2_tmp";
+ private final String FILE3 = "$test_3_tmp";
+
+ // We need the interpreter to be initialised for these tests.
PySystemState systemState;
PyStringMap dict;
PythonInterpreter interp;
@@ -94,6 +106,61 @@
}
}
+ /** Check <code>PyFile().fileno()</code> is acceptable to <code>_io.open()</code> */
+ @Test
+ public void openPyFileByFileno() throws IOException {
+ PySystemState sys = Py.getSystemState();
+ PyFile file = new PyFile(FILE1, "w", 1);
+ openByFilenoTest(file, "wb");
+ }
+
+ /** Check <code>PyFile(OutputStream).fileno()</code> is acceptable to <code>_io.open()</code> */
+ @Test
+ public void openPyFileOStreamByFileno() throws IOException {
+ PySystemState sys = Py.getSystemState();
+ OutputStream ostream = new FileOutputStream(FILE1);
+ PyFile file = new PyFile(ostream);
+ openByFilenoTest(file, "wb");
+ }
+
+ /** Check <code>sys.stdin.fileno()</code> is acceptable to <code>_io.open()</code> */
+ @Test
+ public void openStdinByFileno() throws IOException {
+ PySystemState sys = Py.getSystemState();
+ openByFilenoTest(sys.stdin, "rb");
+ }
+
+ /** Check <code>sys.stdout.fileno()</code> is acceptable to <code>_io.open()</code> */
+ @Test
+ public void openStdoutByFileno() throws IOException {
+ PySystemState sys = Py.getSystemState();
+ openByFilenoTest(sys.stdout, "wb");
+ }
+
+ /** Check <code>sys.stderr.fileno()</code> is acceptable to <code>_io.open()</code> */
+ @Test
+ public void openStderrByFileno() throws IOException {
+ PySystemState sys = Py.getSystemState();
+ openByFilenoTest(sys.stderr, "wb");
+ }
+
+ /**
+ * Test opening by a "file descriptor" obtained from another file or stream. The main purpose is
+ * to test that the return from <code>fileno()</code> is acceptable to <code>_io.open()</code>.
+ *
+ * @param file anything with a "fileno" function
+ * @param mode mode string "rb" etc.
+ * @throws IOException
+ */
+ public void openByFilenoTest(PyObject file, String mode) throws IOException {
+ PyObject pyfd = file.invoke("fileno");
+ RawIOBase fd = (RawIOBase)pyfd.__tojava__(RawIOBase.class);
+ PyObject[] args = new PyObject[] {pyfd, Py.newString(mode), Py.False};
+ String[] kwds = {"closefd"};
+ PyObject file2 = _io.open(args, kwds);
+ file2.invoke("close");
+ }
+
/**
* Test automatic closing of files when the interpreter finally exits. Done correctly, text
* written to any kind of file-like object should be flushed to disk and the file closed when
@@ -106,9 +173,9 @@
public void closeNeglectedFiles() throws IOException {
// File names
- final String F = "$test_1_tmp"; // Raw file
- final String FB = "$test_2_tmp"; // Buffered file
- final String FT = "$test_3_tmp"; // Test file
+ final String F = FILE1; // Raw file
+ final String FB = FILE2; // Buffered file
+ final String FT = FILE3; // Test file
String expText = "Line 1\nLine 2\nLine 3."; // Note: all ascii, but with new lines
byte[] expBytes = expText.getBytes();
@@ -149,11 +216,11 @@
assertTrue(pyft.__closed);
// If they were not closed properly not all bytes will reach the files.
- checkFileContent(F, expBytes);
- checkFileContent(FB, expBytes);
+ checkFileContent(F, expBytes, true);
+ checkFileContent(FB, expBytes, true);
// Expect that TextIOWrapper should have adjusted the line separator
- checkFileContent(FT, newlineFix(expText));
+ checkFileContent(FT, newlineFix(expText), true);
}
/**
@@ -165,9 +232,10 @@
@Test
public void closeNeglectedPyFiles() throws IOException {
- final String F = "$test_1_tmp"; // Raw file
- final String FB = "$test_2_tmp"; // Buffered file
- final String FT = "$test_3_tmp"; // Test file
+ // File names
+ final String F = FILE1; // Raw file
+ final String FB = FILE2; // Buffered file
+ final String FT = FILE3; // Test file
String expText = "Line 1\nLine 2\nLine 3.";
byte[] expBytes = expText.getBytes();
@@ -183,19 +251,19 @@
interp.exec("f = open('" + F + "', 'wb', 0)");
PyFile pyf = (PyFile)interp.get("f");
assertNotNull(pyf);
- RawIOBase r = (RawIOBase) pyf.fileno().__tojava__(RawIOBase.class);
+ RawIOBase r = (RawIOBase)pyf.fileno().__tojava__(RawIOBase.class);
// This should get us a buffered binary PyFile called fb
interp.exec("fb = open('" + FB + "', 'wb')");
PyFile pyfb = (PyFile)interp.get("fb");
assertNotNull(pyfb);
- RawIOBase rb = (RawIOBase) pyfb.fileno().__tojava__(RawIOBase.class);
+ RawIOBase rb = (RawIOBase)pyfb.fileno().__tojava__(RawIOBase.class);
// This should get us an buffered text PyFile called ft
interp.exec("ft = open('" + FT + "', 'w')");
PyFile pyft = (PyFile)interp.get("ft");
assertNotNull(pyft);
- RawIOBase rt = (RawIOBase) pyft.fileno().__tojava__(RawIOBase.class);
+ RawIOBase rt = (RawIOBase)pyft.fileno().__tojava__(RawIOBase.class);
// Write the bytes test material to each file but don't close it
interp.exec("f.write(b)");
@@ -211,39 +279,47 @@
assertTrue(rt.closed());
// If they were not closed properly not all bytes will reach the files.
- checkFileContent(F, expBytes);
- checkFileContent(FB, expBytes);
+ checkFileContent(F, expBytes, true);
+ checkFileContent(FB, expBytes, true);
// Expect that TextIOWrapper should have adjusted the line separator
- checkFileContent(FT, newlineFix(expText));
+ checkFileContent(FT, newlineFix(expText), true);
}
/**
- * Check the file contains the bytes we expect and <b>delete the file</b>. If it was not closed
- * properly (layers in the right order and a flush) not all bytes will have reached the files.
+ * Check the file contains the bytes we expect and optionally <b>delete the file</b>. If it was
+ * not closed properly (layers in the right order and a flush) not all bytes will have reached
+ * the files.
*
* @param name of file
* @param expBytes expected
+ * @param delete the file if true
* @throws IOException if cannot open/read
*/
- private static void checkFileContent(String name, byte[] expBytes) throws IOException {
+ private static void checkFileContent(String name, byte[] expBytes, boolean delete)
+ throws IOException {
+ // Open and read
byte[] r = new byte[2 * expBytes.length];
File f = new File(name);
FileInputStream in = new FileInputStream(f);
int n = in.read(r);
in.close();
+ // Check as expected
String msg = "Bytes read from " + name;
assertEquals(msg, expBytes.length, n);
byte[] resBytes = Arrays.copyOf(r, n);
assertArrayEquals(msg, expBytes, resBytes);
- f.delete();
+ // Delete the file
+ if (delete) {
+ f.delete();
+ }
}
-
/**
* Replace "\n" characters by the system-defined newline sequence and return as bytes.
+ *
* @param expText to translate
* @return result as bytes
*/
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list