[Python-3000-checkins] r59311 - in python/branches/py3k: Include/fileobject.h Lib/io.py Lib/test/test_io.py Objects/complexobject.c Objects/fileobject.c Python/import.c Python/pythonrun.c
guido.van.rossum
python-3000-checkins at python.org
Mon Dec 3 23:54:21 CET 2007
Author: guido.van.rossum
Date: Mon Dec 3 23:54:21 2007
New Revision: 59311
Modified:
python/branches/py3k/Include/fileobject.h
python/branches/py3k/Lib/io.py
python/branches/py3k/Lib/test/test_io.py
python/branches/py3k/Objects/complexobject.c
python/branches/py3k/Objects/fileobject.c
python/branches/py3k/Python/import.c
python/branches/py3k/Python/pythonrun.c
Log:
Add an errors parameter to open() and TextIOWrapper() to specify error handling.
Modified: python/branches/py3k/Include/fileobject.h
==============================================================================
--- python/branches/py3k/Include/fileobject.h (original)
+++ python/branches/py3k/Include/fileobject.h Mon Dec 3 23:54:21 2007
@@ -9,7 +9,7 @@
#define PY_STDIOTEXTMODE "b"
PyAPI_FUNC(PyObject *) PyFile_FromFd(int, char *, char *, int, char *, char *,
- int);
+ char *, int);
PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int);
PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int);
PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *);
Modified: python/branches/py3k/Lib/io.py
==============================================================================
--- python/branches/py3k/Lib/io.py (original)
+++ python/branches/py3k/Lib/io.py Mon Dec 3 23:54:21 2007
@@ -49,8 +49,8 @@
self.characters_written = characters_written
-def open(file, mode="r", buffering=None, encoding=None, newline=None,
- closefd=True):
+def open(file, mode="r", buffering=None, encoding=None, errors=None,
+ newline=None, closefd=True):
r"""Replacement for the built-in open function.
Args:
@@ -61,6 +61,7 @@
can be: 0 = unbuffered, 1 = line buffered,
larger = fully buffered.
encoding: optional string giving the text encoding.
+ errors: optional string giving the encoding error handling.
newline: optional newlines specifier; must be None, '', '\n', '\r'
or '\r\n'; all other values are illegal. It controls the
handling of line endings. It works as follows:
@@ -99,7 +100,7 @@
'U': universal newline mode (for backwards compatibility)
Constraints:
- - encoding must not be given when a binary mode is given
+ - encoding or errors must not be given when a binary mode is given
- buffering must not be zero when a text mode is given
Returns:
@@ -115,6 +116,8 @@
raise TypeError("invalid buffering: %r" % buffering)
if encoding is not None and not isinstance(encoding, str):
raise TypeError("invalid encoding: %r" % encoding)
+ if errors is not None and not isinstance(errors, str):
+ raise TypeError("invalid errors: %r" % errors)
modes = set(mode)
if modes - set("arwb+tU") or len(mode) > len(modes):
raise ValueError("invalid mode: %r" % mode)
@@ -136,6 +139,8 @@
raise ValueError("must have exactly one of read/write/append mode")
if binary and encoding is not None:
raise ValueError("binary mode doesn't take an encoding argument")
+ if binary and errors is not None:
+ raise ValueError("binary mode doesn't take an errors argument")
if binary and newline is not None:
raise ValueError("binary mode doesn't take a newline argument")
raw = FileIO(file,
@@ -177,7 +182,7 @@
buffer.name = file
buffer.mode = mode
return buffer
- text = TextIOWrapper(buffer, encoding, newline)
+ text = TextIOWrapper(buffer, encoding, errors, newline)
text.name = file
text.mode = mode
return text
@@ -1128,7 +1133,7 @@
_CHUNK_SIZE = 128
- def __init__(self, buffer, encoding=None, newline=None):
+ def __init__(self, buffer, encoding=None, errors=None, newline=None):
if newline not in (None, "", "\n", "\r", "\r\n"):
raise ValueError("illegal newline value: %r" % (newline,))
if encoding is None:
@@ -1148,8 +1153,15 @@
if not isinstance(encoding, str):
raise ValueError("invalid encoding: %r" % encoding)
+ if errors is None:
+ errors = "strict"
+ else:
+ if not isinstance(errors, str):
+ raise ValueError("invalid errors: %r" % errors)
+
self.buffer = buffer
self._encoding = encoding
+ self._errors = errors
self._readuniversal = not newline
self._readtranslate = newline is None
self._readnl = newline
@@ -1164,6 +1176,10 @@
def encoding(self):
return self._encoding
+ @property
+ def errors(self):
+ return self._errors
+
# A word about _snapshot. This attribute is either None, or a
# tuple (decoder_state, readahead, pending) where decoder_state is
# the second (integer) item of the decoder state, readahead is the
@@ -1206,7 +1222,7 @@
if haslf and self._writetranslate and self._writenl != "\n":
s = s.replace("\n", self._writenl)
# XXX What if we were just reading?
- b = s.encode(self._encoding)
+ b = s.encode(self._encoding, self._errors)
self.buffer.write(b)
if haslf and self.isatty():
self.flush()
@@ -1220,7 +1236,7 @@
if make_decoder is None:
raise IOError("Can't find an incremental decoder for encoding %s" %
self._encoding)
- decoder = make_decoder() # XXX: errors
+ decoder = make_decoder(self._errors)
if self._readuniversal:
decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
self._decoder = decoder
@@ -1447,9 +1463,11 @@
# XXX This is really slow, but fully functional
- def __init__(self, initial_value="", encoding="utf-8", newline="\n"):
+ def __init__(self, initial_value="", encoding="utf-8",
+ errors="strict", newline="\n"):
super(StringIO, self).__init__(BytesIO(),
encoding=encoding,
+ errors=errors,
newline=newline)
if initial_value:
if not isinstance(initial_value, str):
@@ -1459,4 +1477,4 @@
def getvalue(self):
self.flush()
- return self.buffer.getvalue().decode(self._encoding)
+ return self.buffer.getvalue().decode(self._encoding, self._errors)
Modified: python/branches/py3k/Lib/test/test_io.py
==============================================================================
--- python/branches/py3k/Lib/test/test_io.py (original)
+++ python/branches/py3k/Lib/test/test_io.py Mon Dec 3 23:54:21 2007
@@ -496,6 +496,46 @@
def tearDown(self):
test_support.unlink(test_support.TESTFN)
+ def testEncodingErrorsReading(self):
+ # (1) default
+ b = io.BytesIO(b"abc\n\xff\n")
+ t = io.TextIOWrapper(b, encoding="ascii")
+ self.assertRaises(UnicodeError, t.read)
+ # (2) explicit strict
+ b = io.BytesIO(b"abc\n\xff\n")
+ t = io.TextIOWrapper(b, encoding="ascii", errors="strict")
+ self.assertRaises(UnicodeError, t.read)
+ # (3) ignore
+ b = io.BytesIO(b"abc\n\xff\n")
+ t = io.TextIOWrapper(b, encoding="ascii", errors="ignore")
+ self.assertEquals(t.read(), "abc\n\n")
+ # (4) replace
+ b = io.BytesIO(b"abc\n\xff\n")
+ t = io.TextIOWrapper(b, encoding="ascii", errors="replace")
+ self.assertEquals(t.read(), "abc\n\ufffd\n")
+
+ def testEncodingErrorsWriting(self):
+ # (1) default
+ b = io.BytesIO()
+ t = io.TextIOWrapper(b, encoding="ascii")
+ self.assertRaises(UnicodeError, t.write, "\xff")
+ # (2) explicit strict
+ b = io.BytesIO()
+ t = io.TextIOWrapper(b, encoding="ascii", errors="strict")
+ self.assertRaises(UnicodeError, t.write, "\xff")
+ # (3) ignore
+ b = io.BytesIO()
+ t = io.TextIOWrapper(b, encoding="ascii", errors="ignore")
+ t.write("abc\xffdef\n")
+ t.flush()
+ self.assertEquals(b.getvalue(), b"abcdef\n")
+ # (4) replace
+ b = io.BytesIO()
+ t = io.TextIOWrapper(b, encoding="ascii", errors="replace")
+ t.write("abc\xffdef\n")
+ t.flush()
+ self.assertEquals(b.getvalue(), b"abc?def\n")
+
def testNewlinesInput(self):
testdata = b"AAA\nBBB\nCCC\rDDD\rEEE\r\nFFF\r\nGGG"
normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
Modified: python/branches/py3k/Objects/complexobject.c
==============================================================================
--- python/branches/py3k/Objects/complexobject.c (original)
+++ python/branches/py3k/Objects/complexobject.c Mon Dec 3 23:54:21 2007
@@ -915,6 +915,7 @@
return NULL;
}
cr.real = PyFloat_AsDouble(tmp);
+ cr.imag = 0.0; /* Shut up compiler warning */
Py_DECREF(tmp);
}
if (i == NULL) {
Modified: python/branches/py3k/Objects/fileobject.c
==============================================================================
--- python/branches/py3k/Objects/fileobject.c (original)
+++ python/branches/py3k/Objects/fileobject.c Mon Dec 3 23:54:21 2007
@@ -27,15 +27,16 @@
PyObject *
PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding,
- char *newline, int closefd)
+ char *errors, char *newline, int closefd)
{
PyObject *io, *stream, *nameobj = NULL;
io = PyImport_ImportModule("io");
if (io == NULL)
return NULL;
- stream = PyObject_CallMethod(io, "open", "isissi", fd, mode,
- buffering, encoding, newline, closefd);
+ stream = PyObject_CallMethod(io, "open", "isisssi", fd, mode,
+ buffering, encoding, errors,
+ newline, closefd);
Py_DECREF(io);
if (stream == NULL)
return NULL;
Modified: python/branches/py3k/Python/import.c
==============================================================================
--- python/branches/py3k/Python/import.c (original)
+++ python/branches/py3k/Python/import.c Mon Dec 3 23:54:21 2007
@@ -2602,7 +2602,7 @@
(char*)PyUnicode_GetDefaultEncoding();
}
fob = PyFile_FromFd(fd, pathname, fdp->mode, -1,
- (char*)encoding, NULL, 1);
+ (char*)encoding, NULL, NULL, 1);
if (fob == NULL) {
close(fd);
PyMem_FREE(found_encoding);
Modified: python/branches/py3k/Python/pythonrun.c
==============================================================================
--- python/branches/py3k/Python/pythonrun.c (original)
+++ python/branches/py3k/Python/pythonrun.c Mon Dec 3 23:54:21 2007
@@ -770,7 +770,7 @@
#endif
}
else {
- if (!(std = PyFile_FromFd(fd, "<stdin>", "r", -1, NULL,
+ if (!(std = PyFile_FromFd(fd, "<stdin>", "r", -1, NULL, NULL,
"\n", 0))) {
goto error;
}
@@ -790,7 +790,7 @@
#endif
}
else {
- if (!(std = PyFile_FromFd(fd, "<stdout>", "w", -1, NULL,
+ if (!(std = PyFile_FromFd(fd, "<stdout>", "w", -1, NULL, NULL,
"\n", 0))) {
goto error;
}
@@ -811,7 +811,7 @@
#endif
}
else {
- if (!(std = PyFile_FromFd(fd, "<stderr>", "w", -1, NULL,
+ if (!(std = PyFile_FromFd(fd, "<stderr>", "w", -1, NULL, NULL,
"\n", 0))) {
goto error;
}
More information about the Python-3000-checkins
mailing list