[Jython-checkins] jython: Adds various os.* functions; supports int file descriptors
jim.baker
jython-checkins at python.org
Wed Jan 28 03:51:15 CET 2015
https://hg.python.org/jython/rev/e1a246f6a178
changeset: 7561:e1a246f6a178
user: Jim Baker <jim.baker at rackspace.com>
date: Tue Jan 27 19:51:07 2015 -0700
summary:
Adds various os.* functions; supports int file descriptors
Adds os.* functions for file descriptors, including os.fstat, as well
as support int file descriptors from underlying OS. Also adds
os.times. Uses more of the Posix support now provided by Java Native
Runtime Posix 3.0.x.
Fixes http://bugs.jython.org/issue1560, http://bugs.jython.org/issue1736,
and http://bugs.jython.org/issue1948
files:
Lib/test/test_fileno.py | 2 +-
Lib/test/test_os.py | 12 +-
src/org/python/core/PyObject.java | 9 +
src/org/python/core/io/FileIO.java | 37 +
src/org/python/modules/posix/PosixModule.java | 305 ++++++++--
5 files changed, 294 insertions(+), 71 deletions(-)
diff --git a/Lib/test/test_fileno.py b/Lib/test/test_fileno.py
--- a/Lib/test/test_fileno.py
+++ b/Lib/test/test_fileno.py
@@ -32,7 +32,7 @@
self.assertEqual(os.path.getsize(self.filename), 0)
self.fp.close()
- raises(IOError, 9, os.ftruncate, self.fd, 0)
+ raises(OSError, 9, os.ftruncate, self.fd, 0)
def test_lseek(self):
self.assertEqual(os.lseek(self.fd, 0, 1), 0)
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -32,8 +32,6 @@
os.close(f)
self.assertTrue(os.access(test_support.TESTFN, os.W_OK))
- @unittest.skipIf(test_support.is_jython,
- "Jython does not yet support os.dup.")
def test_closerange(self):
first = os.open(test_support.TESTFN, os.O_CREAT|os.O_RDWR)
# We must allocate two consecutive file descriptors, otherwise
@@ -371,8 +369,6 @@
class WalkTests(unittest.TestCase):
"""Tests for os.walk()."""
- @unittest.skipIf(test_support.is_jython,
- "FIXME: investigate in Jython")
def test_traversal(self):
import os
from os.path import join
@@ -578,7 +574,8 @@
def check(self, f, *args):
try:
- f(test_support.make_bad_fd(), *args)
+ fd = test_support.make_bad_fd()
+ f(fd, *args)
except OSError as e:
self.assertEqual(e.errno, errno.EBADF)
except ValueError:
@@ -587,14 +584,13 @@
self.fail("%r didn't raise a OSError with a bad file descriptor"
% f)
- @unittest.skipIf(test_support.is_jython, "FIXME: investigate for Jython")
def test_isatty(self):
if hasattr(os, "isatty"):
self.assertEqual(os.isatty(test_support.make_bad_fd()), False)
def test_closerange(self):
if hasattr(os, "closerange"):
- fd = test_support.make_bad_fd()
+ fd = int(test_support.make_bad_fd()) # need to take an int for Jython, given this test
# Make sure none of the descriptors we are about to close are
# currently valid (issue 6542).
for i in range(10):
@@ -624,8 +620,6 @@
if hasattr(os, "fpathconf"):
self.check(os.fpathconf, "PC_NAME_MAX")
- @unittest.skipIf(test_support.is_jython,
- "ftruncate not implemented in Jython")
def test_ftruncate(self):
if hasattr(os, "ftruncate"):
self.check(os.ftruncate, 0)
diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java
--- a/src/org/python/core/PyObject.java
+++ b/src/org/python/core/PyObject.java
@@ -590,6 +590,15 @@
}
/**
+ * Determine if this object can act as an int (implements __int__).
+ *
+ * @return true if the object can act as an int
+ */
+ public boolean isInteger() {
+ return getType().lookup("__int__") != null;
+ }
+
+ /**
* Determine if this object can act as an index (implements __index__).
*
* @return true if the object can act as an index
diff --git a/src/org/python/core/io/FileIO.java b/src/org/python/core/io/FileIO.java
--- a/src/org/python/core/io/FileIO.java
+++ b/src/org/python/core/io/FileIO.java
@@ -2,19 +2,23 @@
package org.python.core.io;
import java.io.File;
+import java.io.FileDescriptor;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
+import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import jnr.constants.platform.Errno;
+import jnr.posix.util.FieldAccess;
import jnr.posix.util.Platform;
import org.python.core.Py;
+import org.python.core.PyObject;
import org.python.core.PyString;
import org.python.core.util.RelativeFile;
import org.python.modules.posix.PosixModule;
@@ -435,4 +439,37 @@
public FileChannel getChannel() {
return fileChannel;
}
+
+ public FileDescriptor getFD() {
+ if (file != null) {
+ try {
+ return file.getFD();
+ } catch (IOException ioe) {
+ throw Py.OSError(ioe);
+ }
+ } else if (fileOutputStream != null) {
+ try {
+ return fileOutputStream.getFD();
+ } catch (IOException ioe) {
+ throw Py.OSError(ioe);
+ }
+ }
+ throw Py.OSError(Errno.EBADF);
+ }
+
+ public PyObject __int__() {
+ int intFD = -1;
+ try {
+ Field fdField = FieldAccess.getProtectedField(FileDescriptor.class, "fd");
+ intFD = fdField.getInt(getFD());
+ } catch (SecurityException e) {
+ } catch (IllegalArgumentException e) {
+ } catch (IllegalAccessException e) {
+ }
+ return Py.newInteger(intFD);
+ }
+
+ public PyObject __add__(PyObject otherObj) {
+ return __int__().__add__(otherObj);
+ }
}
diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -6,10 +6,13 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
import java.nio.ByteBuffer;
import java.nio.channels.Channel;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
+import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
@@ -19,9 +22,12 @@
import jnr.constants.Constant;
import jnr.constants.platform.Errno;
+import jnr.constants.platform.Sysconf;
import jnr.posix.FileStat;
import jnr.posix.POSIX;
import jnr.posix.POSIXFactory;
+import jnr.posix.Times;
+import jnr.posix.util.FieldAccess;
import jnr.posix.util.Platform;
import org.python.core.BufferProtocol;
@@ -34,14 +40,11 @@
import org.python.core.PyException;
import org.python.core.PyFile;
import org.python.core.PyFloat;
-import org.python.core.PyInteger;
import org.python.core.PyList;
import org.python.core.PyObject;
import org.python.core.PyString;
-import org.python.core.PySystemState;
import org.python.core.PyTuple;
import org.python.core.imp;
-import org.python.core.io.FileDescriptors;
import org.python.core.io.FileIO;
import org.python.core.io.IOBase;
import org.python.core.io.RawIOBase;
@@ -114,7 +117,8 @@
dict.__setitem__("error", Py.OSError);
dict.__setitem__("stat_result", PyStatResult.TYPE);
- // Faster call paths
+ // Faster call paths, because __call__ is defined
+ dict.__setitem__("fstat", new FstatFunction());
dict.__setitem__("lstat", new LstatFunction());
dict.__setitem__("stat", new StatFunction());
@@ -141,6 +145,81 @@
dict.__setitem__("__doc__", __doc__);
}
+ // Combine Java FileDescriptor objects with
+ // Posix int file descriptors in one representation
+ private static class FDUnion {
+ volatile int intFD;
+ final FileDescriptor javaFD;
+
+ FDUnion(int fd) {
+ intFD = fd;
+ javaFD = null;
+ }
+ FDUnion(FileDescriptor fd) {
+ intFD = -1;
+ javaFD = fd;
+ }
+
+ boolean isIntFD() {
+ return intFD != -1;
+ }
+
+ // FIXME should this store this int fd -> FileDescriptor in a weak value map so it can be looked up later?
+ int getIntFD() {
+ return getIntFD(true);
+ }
+
+ int getIntFD(boolean checkFD) {
+ if (intFD == -1) {
+ if (!(javaFD instanceof FileDescriptor)) {
+ throw Py.OSError(Errno.EBADF);
+ }
+ try {
+ Field fdField = FieldAccess.getProtectedField(FileDescriptor.class, "fd");
+ intFD = fdField.getInt(javaFD);
+ } catch (SecurityException e) {
+ } catch (IllegalArgumentException e) {
+ } catch (IllegalAccessException e) {
+ }
+ }
+ if (checkFD) {
+ posix.fstat(intFD); // check if this a good FD or not
+ }
+ return intFD;
+ }
+
+ @Override
+ public String toString() {
+ return "FDUnion(int=" + intFD + ", java=" + javaFD + ")";
+ }
+
+ }
+
+ private static FDUnion getFD(PyObject fdObj) {
+ if (fdObj.isInteger()) {
+ int intFd = fdObj.asInt();
+ switch (intFd) {
+ case 0:
+ return new FDUnion(FileDescriptor.in);
+ case 1:
+ return new FDUnion(FileDescriptor.out);
+ case 2:
+ return new FDUnion(FileDescriptor.err);
+ default:
+ return new FDUnion(intFd);
+ }
+ }
+ Object tojava = fdObj.__tojava__(FileDescriptor.class);
+ if (tojava != Py.NoConversion) {
+ return new FDUnion((FileDescriptor) tojava);
+ }
+ tojava = fdObj.__tojava__(FileIO.class);
+ if (tojava != Py.NoConversion) {
+ return new FDUnion(((FileIO)tojava).getFD());
+ }
+ throw Py.TypeError("an integer or Java/Jython file descriptor is required");
+ }
+
public static PyString __doc___exit = new PyString(
"_exit(status)\n\n" +
"Exit to the system with specified status, without normal exit processing.");
@@ -226,13 +305,32 @@
"close(fd)\n\n" +
"Close a file descriptor (for low level IO).");
public static void close(PyObject fd) {
- try {
- FileDescriptors.get(fd).close();
- } catch (PyException pye) {
- throw badFD();
+ Object obj = fd.__tojava__(RawIOBase.class);
+ if (obj != Py.NoConversion) {
+ ((RawIOBase)obj).close();
+ } else {
+ posix.close(getFD(fd).getIntFD());
}
}
+ public static void closerange(PyObject fd_lowObj, PyObject fd_highObj) {
+ int fd_low = getFD(fd_lowObj).getIntFD(false);
+ int fd_high = getFD(fd_highObj).getIntFD(false);
+ for (int i = fd_low; i < fd_high; i++) {
+ try {
+ posix.close(i); // FIXME catch exceptions
+ } catch (Exception e) {}
+ }
+ }
+
+ public static PyObject dup(PyObject fd1) {
+ return Py.newInteger(posix.dup(getFD(fd1).getIntFD()));
+ }
+
+ public static PyObject dup2(PyObject fd1, PyObject fd2) {
+ return Py.newInteger(posix.dup2(getFD(fd1).getIntFD(), getFD(fd2).getIntFD()));
+ }
+
public static PyString __doc__fdopen = new PyString(
"fdopen(fd [, mode='r' [, bufsize]]) -> file_object\n\n" +
"Return an open file object connected to a file descriptor.");
@@ -249,7 +347,12 @@
if (mode.length() == 0 || !"rwa".contains("" + mode.charAt(0))) {
throw Py.ValueError(String.format("invalid file mode '%s'", mode));
}
- RawIOBase rawIO = FileDescriptors.get(fd);
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj == Py.NoConversion) {
+ getFD(fd).getIntFD();
+ throw Py.NotImplementedError("Integer file descriptors not currently supported for fdopen");
+ }
+ RawIOBase rawIO = (RawIOBase)javaobj;
if (rawIO.closed()) {
throw badFD();
}
@@ -270,21 +373,30 @@
"does not force update of metadata.");
@Hide(OS.NT)
public static void fdatasync(PyObject fd) {
- fsync(fd, false);
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj != Py.NoConversion) {
+ fsync((RawIOBase)javaobj, false);
+ } else {
+ posix.fdatasync(getFD(fd).getIntFD());
+ }
}
public static PyString __doc__fsync = new PyString(
"fsync(fildes)\n\n" +
"force write of file with filedescriptor to disk.");
public static void fsync(PyObject fd) {
- fsync(fd, true);
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj != Py.NoConversion) {
+ fsync((RawIOBase)javaobj, true);
+ } else {
+ posix.fsync(getFD(fd).getIntFD());
+ }
}
/**
* Internal fsync implementation.
*/
- private static void fsync(PyObject fd, boolean metadata) {
- RawIOBase rawIO = FileDescriptors.get(fd);
+ private static void fsync(RawIOBase rawIO, boolean metadata) {
rawIO.checkClosed();
Channel channel = rawIO.getChannel();
if (!(channel instanceof FileChannel)) {
@@ -304,11 +416,17 @@
public static PyString __doc__ftruncate = new PyString(
"ftruncate(fd, length)\n\n" +
"Truncate a file to a specified length.");
+
public static void ftruncate(PyObject fd, long length) {
- try {
- FileDescriptors.get(fd).truncate(length);
- } catch (PyException pye) {
- throw Py.IOError(Errno.EBADF);
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj != Py.NoConversion) {
+ try {
+ ((RawIOBase) javaobj).truncate(length);
+ } catch (PyException pye) {
+ throw Py.OSError(Errno.EBADF);
+ }
+ } else {
+ posix.ftruncate(getFD(fd).getIntFD(), length);
}
}
@@ -390,40 +508,40 @@
return posix.getpgrp();
}
+
+
public static PyString __doc__isatty = new PyString(
"isatty(fd) -> bool\n\n" +
"Return True if the file descriptor 'fd' is an open file descriptor\n" +
"connected to the slave end of a terminal.");
public static boolean isatty(PyObject fdObj) {
- if (fdObj instanceof PyInteger) {
- FileDescriptor fd;
- switch (fdObj.asInt()) {
- case 0:
- fd = FileDescriptor.in;
- break;
- case 1:
- fd = FileDescriptor.out;
- break;
- case 2:
- fd = FileDescriptor.err;
- break;
- default:
- throw Py.NotImplementedError("Integer file descriptor compatibility only "
- + "available for stdin, stdout and stderr (0-2)");
+ Object tojava = fdObj.__tojava__(IOBase.class);
+ if (tojava != Py.NoConversion) {
+ try {
+ return ((IOBase) tojava).isatty();
+ } catch (PyException pye) {
+ if (pye.match(Py.ValueError)) {
+ return false;
+ }
+ throw pye;
}
- return posix.isatty(fd);
}
- Object tojava = fdObj.__tojava__(FileDescriptor.class);
- if (tojava != Py.NoConversion) {
- return posix.isatty((FileDescriptor)tojava);
+ FDUnion fd = getFD(fdObj);
+ if (fd.javaFD != null) {
+ return posix.isatty(fd.javaFD);
}
-
- tojava = fdObj.__tojava__(IOBase.class);
- if (tojava == Py.NoConversion) {
- throw Py.TypeError("a file descriptor is required");
+ try {
+ fd.getIntFD(); // evaluate for side effect of checking EBADF or raising TypeError
+ } catch (PyException pye) {
+ if (pye.match(Py.OSError)) {
+ return false;
+ }
+ throw pye;
}
- return ((IOBase)tojava).isatty();
+ throw Py.NotImplementedError(
+ "Integer file descriptor compatibility only "
+ + "available for stdin, stdout and stderr (0-2)");
}
public static PyString __doc__kill = new PyString(
@@ -504,10 +622,15 @@
"lseek(fd, pos, how) -> newpos\n\n" +
"Set the current position of a file descriptor.");
public static long lseek(PyObject fd, long pos, int how) {
- try {
- return FileDescriptors.get(fd).seek(pos, how);
- } catch (PyException pye) {
- throw badFD();
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj != Py.NoConversion) {
+ try {
+ return ((RawIOBase) javaobj).seek(pos, how);
+ } catch (PyException pye) {
+ throw badFD();
+ }
+ } else {
+ return posix.lseek(getFD(fd).getIntFD(), pos, how);
}
}
@@ -610,10 +733,17 @@
"read(fd, buffersize) -> string\n\n" +
"Read a file descriptor.");
public static PyObject read(PyObject fd, int buffersize) {
- try {
- return new PyString(StringUtil.fromBytes(FileDescriptors.get(fd).read(buffersize)));
- } catch (PyException pye) {
- throw badFD();
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj != Py.NoConversion) {
+ try {
+ return new PyString(StringUtil.fromBytes(((RawIOBase) javaobj).read(buffersize)));
+ } catch (PyException pye) {
+ throw badFD();
+ }
+ } else {
+ ByteBuffer buffer = ByteBuffer.allocate(buffersize);
+ posix.read(getFD(fd).getIntFD(), buffer, buffersize);
+ return new PyString(StringUtil.fromBytes(buffer));
}
}
@@ -709,6 +839,26 @@
}
}
+ private static PyFloat ratio(long num, long div) {
+ return Py.newFloat(((double)num)/((double)div));
+ }
+
+ public static PyString __doc__times = new PyString(
+ "times() -> (utime, stime, cutime, cstime, elapsed_time)\n\n" +
+ "Return a tuple of floating point numbers indicating process times.");
+
+ public static PyTuple times() {
+ Times times = posix.times();
+ long CLK_TCK = Sysconf._SC_CLK_TCK.longValue();
+ return new PyTuple(
+ ratio(times.utime(), CLK_TCK),
+ ratio(times.stime(), CLK_TCK),
+ ratio(times.cutime(), CLK_TCK),
+ ratio(times.cstime(), CLK_TCK),
+ ratio(ManagementFactory.getRuntimeMXBean().getUptime(), 1000)
+ );
+ }
+
public static PyString __doc__umask = new PyString(
"umask(new_mask) -> old_mask\n\n" +
"Set the current numeric umask and return the previous umask.");
@@ -810,18 +960,23 @@
return new PyTuple(Py.newInteger(pid), Py.newInteger(status[0]));
}
- public static PyString __doc__write = new PyString("write(fd, string) -> byteswritten\n\n"
- + "Write a string to a file descriptor.");
+ public static PyString __doc__write = new PyString(
+ "write(fd, string) -> byteswritten\n\n" +
+ "Write a string to a file descriptor.");
public static int write(PyObject fd, BufferProtocol bytes) {
// Get a buffer view: we can cope with N-dimensional data, but not strided data.
try (PyBuffer buf = bytes.getBuffer(PyBUF.ND)) {
// Get a ByteBuffer of that data, setting the position and limit to the real data.
- ByteBuffer bb = buf.getNIOByteBuffer();
- try {
- // Write the data (returning the count of bytes).
- return FileDescriptors.get(fd).write(bb);
- } catch (PyException pye) {
- throw badFD();
+ ByteBuffer bb = buf.getNIOByteBuffer();
+ Object javaobj = fd.__tojava__(RawIOBase.class);
+ if (javaobj != Py.NoConversion) {
+ try {
+ return ((RawIOBase) javaobj).write(bb);
+ } catch (PyException pye) {
+ throw badFD();
+ }
+ } else {
+ return posix.write(getFD(fd).getIntFD(), bb, bb.position());
}
}
}
@@ -907,11 +1062,21 @@
/**
* Return the absolute form of path.
*
- * @param path a PyObject, raising a TypeError if an invalid path type
+ * @param pathObj a PyObject, raising a TypeError if an invalid path type
* @return an absolute path String
*/
- private static String absolutePath(PyObject path) {
- return PySystemState.getPathLazy(asPath(path));
+ private static String absolutePath(PyObject pathObj) {
+ String pathStr = asPath(pathObj);
+ if (pathStr.equals("")) {
+ // Otherwise this path will get prefixed with the current working directory,
+ // which is both wrong and very surprising!
+ throw Py.OSError(Errno.ENOENT, pathObj);
+ }
+ Path path = FileSystems.getDefault().getPath(pathStr);
+ if (!path.isAbsolute()) {
+ path = FileSystems.getDefault().getPath(Py.getSystemState().getCurrentWorkingDir(), pathStr);
+ }
+ return path.toAbsolutePath().normalize().toString();
}
private static PyException badFD() {
@@ -967,7 +1132,6 @@
@Override
public PyObject __call__(PyObject path) {
String absolutePath = absolutePath(path);
-
// round tripping from a string to a file to a string loses
// trailing slashes so add them back back in to get correct posix.stat
// behaviour if path is not a directory.
@@ -977,4 +1141,23 @@
return PyStatResult.fromFileStat(posix.stat(absolutePath));
}
}
+
+ static class FstatFunction extends PyBuiltinFunctionNarrow {
+ FstatFunction() {
+ super("fstat", 1, 1,
+ "fstat(fd) -> stat result\\n\\nLike stat(), but for an open file descriptor.");
+ }
+
+ @Override
+ public PyObject __call__(PyObject fdObj) {
+ FDUnion fd = getFD(fdObj);
+ FileStat stat;
+ if (fd.isIntFD()) {
+ stat = posix.fstat(fd.intFD);
+ } else {
+ stat = posix.fstat(fd.javaFD);;
+ }
+ return PyStatResult.fromFileStat(stat);
+ }
+ }
}
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list