[Python-3000-checkins] r54737 - in python/branches/p3yk: Lib/io.py Lib/test/test_io.py Modules/_fileio.c
guido.van.rossum
python-3000-checkins at python.org
Tue Apr 10 21:01:51 CEST 2007
Author: guido.van.rossum
Date: Tue Apr 10 21:01:47 2007
New Revision: 54737
Modified:
python/branches/p3yk/Lib/io.py
python/branches/p3yk/Lib/test/test_io.py
python/branches/p3yk/Modules/_fileio.c
Log:
Implement long positioning (Unix only, probably).
Etc., etc.
Modified: python/branches/p3yk/Lib/io.py
==============================================================================
--- python/branches/p3yk/Lib/io.py (original)
+++ python/branches/p3yk/Lib/io.py Tue Apr 10 21:01:47 2007
@@ -1,9 +1,15 @@
-"""New I/O library.
+"""New I/O library conforming to PEP 3116.
This is an early prototype; eventually some of this will be
reimplemented in C and the rest may be turned into a package.
-See PEP 3116.
+Conformance of alternative implementations: all arguments are intended
+to be positional-only except the arguments of the open() function.
+Argument names except those of the open() function are not part of the
+specification. Instance variables and methods whose name starts with
+a leading underscore are not part of the specification (except "magic"
+names like __iter__). Only the top-level names listed in the __all__
+variable are part of the specification.
XXX need to default buffer size to 1 if isatty()
XXX need to support 1 meaning line-buffered
@@ -142,6 +148,9 @@
This does not define read(), readinto() and write(), nor
readline() and friends, since their signatures vary per layer.
+
+ Not that calling any method (even inquiries) on a closed file is
+ undefined. Implementations may raise IOError in this case.
"""
### Internal ###
@@ -153,19 +162,20 @@
### Positioning ###
- def seek(self, pos: int, whence: int = 0) -> None:
- """seek(pos: int, whence: int = 0) -> None. Change stream position.
+ def seek(self, pos: int, whence: int = 0) -> int:
+ """seek(pos: int, whence: int = 0) -> int. Change stream position.
Seek to byte offset pos relative to position indicated by whence:
0 Start of stream (the default). pos should be >= 0;
1 Current position - whence may be negative;
2 End of stream - whence usually negative.
+ Returns the new absolute position.
"""
self._unsupported("seek")
def tell(self) -> int:
"""tell() -> int. Return current stream position."""
- self._unsupported("tell")
+ return self.seek(0, 1)
def truncate(self, pos: int = None) -> None:
"""truncate(size: int = None) -> None. Truncate file to size bytes.
@@ -432,7 +442,7 @@
### Positioning ###
def seek(self, pos, whence=0):
- self.raw.seek(pos, whence)
+ return self.raw.seek(pos, whence)
def tell(self):
return self.raw.tell()
@@ -515,6 +525,7 @@
self._pos = max(0, len(self._buffer) + pos)
else:
raise IOError("invalid whence value")
+ return self._pos
def tell(self):
return self._pos
@@ -620,8 +631,9 @@
def seek(self, pos, whence=0):
if whence == 1:
pos -= len(self._read_buf)
- self.raw.seek(pos, whence)
+ pos = self.raw.seek(pos, whence)
self._read_buf = b""
+ return pos
class BufferedWriter(_BufferedIOMixin):
@@ -679,7 +691,7 @@
def seek(self, pos, whence=0):
self.flush()
- self.raw.seek(pos, whence)
+ return self.raw.seek(pos, whence)
class BufferedRWPair(BufferedIOBase):
@@ -750,13 +762,9 @@
self.flush()
# First do the raw seek, then empty the read buffer, so that
# if the raw seek fails, we don't lose buffered data forever.
- self.raw.seek(pos, whence)
+ pos = self.raw.seek(pos, whence)
self._read_buf = b""
- # XXX I suppose we could implement some magic here to move through the
- # existing read buffer in the case of seek(<some small +ve number>, 1)
- # XXX OTOH it might be good to *guarantee* that the buffer is
- # empty after a seek or flush; for small relative forward
- # seeks one might as well use small reads instead.
+ return pos
def tell(self):
if (self._write_buf):
Modified: python/branches/p3yk/Lib/test/test_io.py
==============================================================================
--- python/branches/p3yk/Lib/test/test_io.py (original)
+++ python/branches/p3yk/Lib/test/test_io.py Tue Apr 10 21:01:47 2007
@@ -4,10 +4,10 @@
from itertools import chain
from test import test_support
-import io
+import io # The module under test
-class MockIO(io.RawIOBase):
+class MockRawIO(io.RawIOBase):
def __init__(self, read_stack=()):
self._read_stack = list(read_stack)
@@ -56,13 +56,13 @@
class MockNonBlockWriterIO(io.RawIOBase):
- def __init__(self, blockingScript):
- self.bs = list(blockingScript)
+ def __init__(self, blocking_script):
+ self._blocking_script = list(blocking_script)
self._write_stack = []
def write(self, b):
self._write_stack.append(b[:])
- n = self.bs.pop(0)
+ n = self._blocking_script.pop(0)
if (n < 0):
raise io.BlockingIOError(0, "test blocking", -n)
else:
@@ -90,6 +90,23 @@
f.seek(-2, 2)
f.truncate()
+ def large_file_ops(self, f):
+ assert f.readable()
+ assert f.writable()
+ self.assertEqual(f.seek(2**32), 2**32)
+ self.assertEqual(f.tell(), 2**32)
+ self.assertEqual(f.write(b"xxx"), 3)
+ self.assertEqual(f.tell(), 2**32 + 3)
+ self.assertEqual(f.seek(-1, 1), 2**32 + 2)
+ f.truncate()
+ self.assertEqual(f.tell(), 2**32 + 2)
+ self.assertEqual(f.seek(0, 2), 2**32 + 2)
+ f.truncate(2**32 + 1)
+ self.assertEqual(f.tell(), 2**32 + 1)
+ self.assertEqual(f.seek(0, 2), 2**32 + 1)
+ self.assertEqual(f.seek(-1, 2), 2**32)
+ self.assertEqual(f.read(2), b"x")
+
def read_ops(self, f):
data = f.read(5)
self.assertEqual(data, b"hello")
@@ -130,19 +147,9 @@
f = io.BytesIO(data)
self.read_ops(f)
- def test_fileio_FileIO(self):
- import _fileio
- f = _fileio._FileIO(test_support.TESTFN, "w")
- self.assertEqual(f.readable(), False)
- self.assertEqual(f.writable(), True)
- self.assertEqual(f.seekable(), True)
- self.write_ops(f)
- f.close()
- f = _fileio._FileIO(test_support.TESTFN, "r")
- self.assertEqual(f.readable(), True)
- self.assertEqual(f.writable(), False)
- self.assertEqual(f.seekable(), True)
- self.read_ops(f)
+ def test_large_file_ops(self):
+ f = io.open(test_support.TESTFN, "w+b", buffering=0)
+ self.large_file_ops(f)
f.close()
@@ -205,7 +212,7 @@
class BufferedReaderTest(unittest.TestCase):
def testRead(self):
- rawio = MockIO((b"abc", b"d", b"efg"))
+ rawio = MockRawIO((b"abc", b"d", b"efg"))
bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcdef", bufio.read(6))
@@ -231,7 +238,7 @@
def testReadNonBlocking(self):
# Inject some None's in there to simulate EWOULDBLOCK
- rawio = MockIO((b"abc", b"d", None, b"efg", None, None))
+ rawio = MockRawIO((b"abc", b"d", None, b"efg", None, None))
bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcd", bufio.read(6))
@@ -241,19 +248,19 @@
self.assertEquals(b"", bufio.read())
def testReadToEof(self):
- rawio = MockIO((b"abc", b"d", b"efg"))
+ rawio = MockRawIO((b"abc", b"d", b"efg"))
bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcdefg", bufio.read(9000))
def testReadNoArgs(self):
- rawio = MockIO((b"abc", b"d", b"efg"))
+ rawio = MockRawIO((b"abc", b"d", b"efg"))
bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcdefg", bufio.read())
def testFileno(self):
- rawio = MockIO((b"abc", b"d", b"efg"))
+ rawio = MockRawIO((b"abc", b"d", b"efg"))
bufio = io.BufferedReader(rawio)
self.assertEquals(42, bufio.fileno())
@@ -268,7 +275,7 @@
def testWrite(self):
# Write to the buffered IO but don't overflow the buffer.
- writer = MockIO()
+ writer = MockRawIO()
bufio = io.BufferedWriter(writer, 8)
bufio.write(b"abc")
@@ -276,7 +283,7 @@
self.assertFalse(writer._write_stack)
def testWriteOverflow(self):
- writer = MockIO()
+ writer = MockRawIO()
bufio = io.BufferedWriter(writer, 8)
bufio.write(b"abc")
@@ -305,13 +312,13 @@
# later.
def testFileno(self):
- rawio = MockIO((b"abc", b"d", b"efg"))
+ rawio = MockRawIO((b"abc", b"d", b"efg"))
bufio = io.BufferedWriter(rawio)
self.assertEquals(42, bufio.fileno())
def testFlush(self):
- writer = MockIO()
+ writer = MockRawIO()
bufio = io.BufferedWriter(writer, 8)
bufio.write(b"abc")
@@ -323,8 +330,8 @@
class BufferedRWPairTest(unittest.TestCase):
def testRWPair(self):
- r = MockIO(())
- w = MockIO()
+ r = MockRawIO(())
+ w = MockRawIO()
pair = io.BufferedRWPair(r, w)
# XXX need implementation
@@ -333,7 +340,7 @@
class BufferedRandomTest(unittest.TestCase):
def testReadAndWrite(self):
- raw = MockIO((b"asdf", b"ghjk"))
+ raw = MockRawIO((b"asdf", b"ghjk"))
rw = io.BufferedRandom(raw, 8, 12)
self.assertEqual(b"as", rw.read(2))
Modified: python/branches/p3yk/Modules/_fileio.c
==============================================================================
--- python/branches/p3yk/Modules/_fileio.c (original)
+++ python/branches/p3yk/Modules/_fileio.c Tue Apr 10 21:01:47 2007
@@ -18,12 +18,6 @@
* To Do:
*
* - autoconfify header file inclusion
- * - Make the ABC RawIO type and inherit from it.
- *
- * Unanswered questions:
- *
- * - Add mode and name properties a la Python 2 file objects?
- * - Check for readable/writable before attempting to read/write?
*/
#ifdef MS_WINDOWS
@@ -36,7 +30,6 @@
typedef struct {
PyObject_HEAD
int fd;
- int own_fd; /* 1 means we should close fd */
int readable;
int writable;
int seekable; /* -1 means unknown */
@@ -80,8 +73,6 @@
if (self != NULL) {
self->fd = -1;
self->weakreflist = NULL;
- self->own_fd = 1;
- self->seekable = -1;
}
return (PyObject *) self;
@@ -107,7 +98,7 @@
PyObject *exc;
PyObject *closeresult = fileio_close(self);
Py_DECREF(closeresult);
-
+
exc = PyObject_CallFunction(PyExc_IOError, "(is)",
EISDIR, msg);
PyErr_SetObject(PyExc_IOError, exc);
@@ -126,7 +117,7 @@
static char *kwlist[] = {"file", "mode", NULL};
char *name = NULL;
char *mode = "r";
- char *s;
+ char *s;
int wideargument = 0;
int ret = 0;
int rwa = 0, plus = 0, append = 0;
@@ -183,7 +174,8 @@
}
self->readable = self->writable = 0;
- s = mode;
+ self->seekable = -1;
+ s = mode;
while (*s) {
switch (*s++) {
case 'r':
@@ -240,7 +232,6 @@
if (fd >= 0) {
self->fd = fd;
- /* XXX Should we set self->own_fd = 0 ??? */
}
else {
Py_BEGIN_ALLOW_THREADS
@@ -257,7 +248,7 @@
error:
ret = -1;
-
+
done:
PyMem_Free(name);
return ret;
@@ -269,15 +260,16 @@
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) self);
- if (self->fd >= 0 && self->own_fd) {
+ if (self->fd >= 0) {
PyObject *closeresult = fileio_close(self);
if (closeresult == NULL) {
#ifdef HAVE_STRERROR
- PySys_WriteStderr("close failed: [Errno %d] %s\n", errno, strerror(errno));
+ PySys_WriteStderr("close failed: [Errno %d] %s\n",
+ errno, strerror(errno));
#else
PySys_WriteStderr("close failed: [Errno %d]\n", errno);
#endif
- } else
+ } else
Py_DECREF(closeresult);
}
@@ -292,6 +284,13 @@
}
static PyObject *
+err_mode(char *action)
+{
+ PyErr_Format(PyExc_ValueError, "File not open for %s", action);
+ return NULL;
+}
+
+static PyObject *
fileio_fileno(PyFileIOObject *self)
{
if (self->fd < 0)
@@ -338,9 +337,12 @@
{
char *ptr;
Py_ssize_t n;
-
+
if (self->fd < 0)
return err_closed();
+ if (!self->readable)
+ return err_mode("reading");
+
if (!PyArg_ParseTuple(args, "w#", &ptr, &n))
return NULL;
@@ -367,6 +369,8 @@
if (self->fd < 0)
return err_closed();
+ if (!self->readable)
+ return err_mode("reading");
if (!PyArg_ParseTuple(args, "i", &size))
return NULL;
@@ -391,9 +395,9 @@
if (n != size) {
if (PyBytes_Resize(bytes, n) < 0) {
Py_DECREF(bytes);
- return NULL;
+ return NULL;
}
- }
+ }
return (PyObject *) bytes;
}
@@ -406,6 +410,9 @@
if (self->fd < 0)
return err_closed();
+ if (!self->writable)
+ return err_mode("writing");
+
if (!PyArg_ParseTuple(args, "s#", &ptr, &n))
return NULL;
@@ -424,146 +431,163 @@
return PyInt_FromLong(n);
}
+/* XXX Windows support below is likely incomplete */
+
+#if defined(MS_WIN64) || defined(MS_WINDOWS)
+typedef PY_LONG_LONG Py_off_t;
+#else
+typedef off_t Py_off_t;
+#endif
+
+/* Cribbed from posix_lseek() */
static PyObject *
-fileio_seek(PyFileIOObject *self, PyObject *args)
+portable_lseek(int fd, PyObject *posobj, int whence)
{
- Py_ssize_t offset;
- Py_ssize_t whence = 0;
+ Py_off_t pos, res;
- if (self->fd < 0)
- return err_closed();
+#ifdef SEEK_SET
+ /* Turn 0, 1, 2 into SEEK_{SET,CUR,END} */
+ switch (whence) {
+#if SEEK_SET != 0
+ case 0: whence = SEEK_SET; break;
+#endif
+#if SEEK_CUR != 1
+ case 1: whence = SEEK_CUR; break;
+#endif
+#if SEEL_END != 2
+ case 2: whence = SEEK_END; break;
+#endif
+ }
+#endif /* SEEK_SET */
- if (!PyArg_ParseTuple(args, "i|i", &offset, &whence))
- return NULL;
+ if (posobj == NULL)
+ pos = 0;
+ else {
+#if !defined(HAVE_LARGEFILE_SUPPORT)
+ pos = PyInt_AsLong(posobj);
+#else
+ pos = PyLong_Check(posobj) ?
+ PyLong_AsLongLong(posobj) : PyInt_AsLong(posobj);
+#endif
+ if (PyErr_Occurred())
+ return NULL;
+ }
Py_BEGIN_ALLOW_THREADS
- errno = 0;
- offset = lseek(self->fd, offset, whence);
+#if defined(MS_WIN64) || defined(MS_WINDOWS)
+ res = _lseeki64(fd, pos, whence);
+#else
+ res = lseek(fd, pos, whence);
+#endif
Py_END_ALLOW_THREADS
+ if (res < 0)
+ return PyErr_SetFromErrno(PyExc_IOError);
- if (offset < 0) {
- PyErr_SetFromErrno(PyExc_IOError);
- return NULL;
- }
-
- Py_RETURN_NONE;
+#if !defined(HAVE_LARGEFILE_SUPPORT)
+ return PyInt_FromLong(res);
+#else
+ return PyLong_FromLongLong(res);
+#endif
}
static PyObject *
-fileio_tell(PyFileIOObject *self, PyObject *args)
+fileio_seek(PyFileIOObject *self, PyObject *args)
{
- Py_ssize_t offset;
+ PyObject *posobj;
+ int whence = 0;
if (self->fd < 0)
return err_closed();
- Py_BEGIN_ALLOW_THREADS
- errno = 0;
- offset = lseek(self->fd, 0, SEEK_CUR);
- Py_END_ALLOW_THREADS
-
- if (offset < 0) {
- PyErr_SetFromErrno(PyExc_IOError);
+ if (!PyArg_ParseTuple(args, "O|i", &posobj, &whence))
return NULL;
- }
- return PyInt_FromLong(offset);
+ return portable_lseek(self->fd, posobj, whence);
}
-#ifdef HAVE_FTRUNCATE
static PyObject *
-fileio_truncate(PyFileIOObject *self, PyObject *args)
+fileio_tell(PyFileIOObject *self, PyObject *args)
{
- Py_ssize_t length;
- int ret;
-
if (self->fd < 0)
return err_closed();
- /* Setup default value */
- Py_BEGIN_ALLOW_THREADS
- errno = 0;
- length = lseek(self->fd, 0, SEEK_CUR);
- Py_END_ALLOW_THREADS
+ return portable_lseek(self->fd, NULL, 1);
+}
- if (length < 0) {
- PyErr_SetFromErrno(PyExc_IOError);
- return NULL;
- }
-
- if (!PyArg_ParseTuple(args, "|i", &length))
+static PyObject *
+fileio_truncate(PyFileIOObject *self, PyObject *args)
+{
+ PyObject *posobj = NULL;
+ Py_off_t pos;
+ int fd, whence;
+
+ fd = self->fd;
+ if (fd < 0)
+ return err_closed();
+ if (!self->writable)
+ return err_mode("writing");
+
+ if (!PyArg_ParseTuple(args, "|O", &posobj))
return NULL;
-#ifdef MS_WINDOWS
- /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
- so don't even try using it. */
- {
- HANDLE hFile;
- Py_ssize_t initialpos;
+ if (posobj == NULL)
+ whence = 1;
+ else
+ whence = 0;
- /* Have to move current pos to desired endpoint on Windows. */
- Py_BEGIN_ALLOW_THREADS
- errno = 0;
- ret = _portable_fseek(f->f_fp, newsize, SEEK_SET) != 0;
- Py_END_ALLOW_THREADS
- if (ret)
- goto onioerror;
+ posobj = portable_lseek(fd, posobj, whence);
+ if (posobj == NULL)
+ return NULL;
+
+#if !defined(HAVE_LARGEFILE_SUPPORT)
+ pos = PyInt_AsLong(posobj);
+#else
+ pos = PyLong_Check(posobj) ?
+ PyLong_AsLongLong(posobj) : PyInt_AsLong(posobj);
+#endif
+ Py_DECREF(posobj);
+ if (PyErr_Occurred())
+ return NULL;
- /* Truncate. Note that this may grow the file! */
- Py_BEGIN_ALLOW_THREADS
- errno = 0;
- hFile = (HANDLE)_get_osfhandle(fileno(f->f_fp));
- ret = hFile == (HANDLE)-1;
- if (ret == 0) {
- ret = SetEndOfFile(hFile) == 0;
- if (ret)
- errno = EACCES;
- }
- Py_END_ALLOW_THREADS
- if (ret)
- goto onioerror;
- }
-#else
Py_BEGIN_ALLOW_THREADS
errno = 0;
- ret = ftruncate(self->fd, length);
+ pos = ftruncate(fd, pos);
Py_END_ALLOW_THREADS
-#endif /* !MS_WINDOWS */
- if (ret < 0) {
- onioerror:
+ if (errno < 0)
PyErr_SetFromErrno(PyExc_IOError);
- return NULL;
- }
- /* Return to initial position */
- Py_BEGIN_ALLOW_THREADS
- errno = 0;
- ret = lseek(self->fd, length, SEEK_SET);
- Py_END_ALLOW_THREADS
- if (ret < 0)
- goto onioerror;
-
Py_RETURN_NONE;
}
-#endif
+
+static char *
+mode_string(PyFileIOObject *self)
+{
+ if (self->readable) {
+ if (self->writable)
+ return "r+";
+ else
+ return "r";
+ }
+ else
+ return "w";
+}
static PyObject *
fileio_repr(PyFileIOObject *self)
{
- PyObject *ret = NULL;
+ if (self->fd < 0)
+ return PyString_FromFormat("_fileio._FileIO(-1)");
- ret = PyString_FromFormat("<%s file at %p>",
- self->fd < 0 ? "closed" : "open",
- self);
- return ret;
+ return PyString_FromFormat("_fileio._FileIO(%d, '%s')",
+ self->fd, mode_string(self));
}
static PyObject *
fileio_isatty(PyFileIOObject *self)
{
long res;
-
+
if (self->fd < 0)
return err_closed();
Py_BEGIN_ALLOW_THREADS
@@ -572,14 +596,6 @@
return PyBool_FromLong(res);
}
-static PyObject *
-fileio_self(PyFileIOObject *self)
-{
- if (self->fd < 0)
- return err_closed();
- Py_INCREF(self);
- return (PyObject *)self;
-}
PyDoc_STRVAR(fileio_doc,
"file(name: str[, mode: str]) -> file IO object\n"
@@ -639,12 +655,6 @@
PyDoc_STRVAR(isatty_doc,
"isatty() -> bool. True if the file is connected to a tty device.");
-PyDoc_STRVAR(enter_doc,
-"__enter__() -> self.");
-
-PyDoc_STRVAR(exit_doc,
-"__exit__(*excinfo) -> None. Closes the file.");
-
PyDoc_STRVAR(seekable_doc,
"seekable() -> bool. True if file supports random-access.");
@@ -667,20 +677,26 @@
{"writable", (PyCFunction)fileio_writable, METH_NOARGS, writable_doc},
{"fileno", (PyCFunction)fileio_fileno, METH_NOARGS, fileno_doc},
{"isatty", (PyCFunction)fileio_isatty, METH_NOARGS, isatty_doc},
- {"__enter__",(PyCFunction)fileio_self, METH_NOARGS, enter_doc},
- {"__exit__", (PyCFunction)fileio_close, METH_VARARGS, exit_doc},
{NULL, NULL} /* sentinel */
};
-/* 'closed' is an attribute for backwards compatibility reasons. */
+/* 'closed' and 'mode' are attributes for backwards compatibility reasons. */
+
+static PyObject *
+get_closed(PyFileIOObject *self, void *closure)
+{
+ return PyBool_FromLong((long)(self->fd < 0));
+}
+
static PyObject *
-get_closed(PyFileIOObject *f, void *closure)
+get_mode(PyFileIOObject *self, void *closure)
{
- return PyBool_FromLong((long)(f->fd < 0));
+ return PyString_FromString(mode_string(self));
}
static PyGetSetDef fileio_getsetlist[] = {
{"closed", (getter)get_closed, NULL, "True if the file is closed"},
+ {"mode", (getter)get_mode, NULL, "String giving the file mode"},
{0},
};
More information about the Python-3000-checkins
mailing list