[pypy-commit] pypy py3k: Use 'surrogateescape' error handling in fsencode/fsdecode functions.

[amauryfa]

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3k
Changeset: r58339:9c5666c6a8e9
Date: 2012-10-21 09:26 +0200
http://bitbucket.org/pypy/pypy/changeset/9c5666c6a8e9/

Log:	Use 'surrogateescape' error handling in fsencode/fsdecode functions.

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -39,9 +39,16 @@
 def fsencode_w(space, w_obj):
     if space.isinstance_w(w_obj, space.w_unicode):
         w_obj = space.call_method(w_obj, 'encode',
-                                  getfilesystemencoding(space))
+                                  getfilesystemencoding(space),
+                                  space.wrap('surrogateescape'))
     return space.bytes0_w(w_obj)
 
+def fsdecode(space, w_obj):
+    w_unicode = space.call_method(w_obj, 'decode',
+                                  getfilesystemencoding(space),
+                                  space.wrap('surrogateescape'))
+    return w_unicode
+
 class FileEncoder(object):
     def __init__(self, space, w_obj):
         self.space = space
@@ -63,8 +70,7 @@
 
     def as_unicode(self):
         space = self.space
-        w_unicode = space.call_method(self.w_obj, 'decode',
-                                      getfilesystemencoding(space))
+        w_unicode = fsdecode(space, self.w_obj)
         return space.unicode0_w(w_unicode)
 
 @specialize.memo()
@@ -544,17 +550,11 @@
         if space.isinstance_w(w_dirname, space.w_unicode):
             dirname = FileEncoder(space, w_dirname)
             result = rposix.listdir(dirname)
-            w_fs_encoding = getfilesystemencoding(space)
             len_result = len(result)
             result_w = [None] * len_result
             for i in range(len_result):
                 w_bytes = space.wrapbytes(result[i])
-                try:
-                    result_w[i] = space.call_method(w_bytes,
-                                                    "decode", w_fs_encoding)
-                except OperationError, e:
-                    # fall back to the original byte string
-                    result_w[i] = w_bytes
+                result_w[i] = fsdecode(space, w_bytes)
         else:
             dirname = space.str0_w(w_dirname)
             result = rposix.listdir(dirname)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -292,10 +292,17 @@
             u = b"caf\xe9".decode(sys.getfilesystemencoding())
         except UnicodeDecodeError:
             # Could not decode, listdir returned the byte string
-            assert (bytes, b"caf\xe9") in typed_result
+            assert (str, 'caf\udce9') in typed_result
         else:
             assert (str, u) in typed_result
 
+    def test_undecodable_filename(self):
+        posix = self.posix
+        assert posix.access('caf\xe9', posix.R_OK) is False
+        assert posix.access(b'caf\xe9', posix.R_OK) is False
+        assert posix.access('caf\udcc0', posix.R_OK) is False
+        assert posix.access(b'caf\xc3', posix.R_OK) is False
+
     def test_access(self):
         pdir = self.pdir + '/file1'
         posix = self.posix