[pypy-svn] r75665 - in pypy/branch/interplevel-codecs/pypy: interpreter interpreter/pyparser module/_codecs objspace/std

[afa at codespeak.net]

Author: afa
Date: Tue Jun 29 17:46:51 2010
New Revision: 75665

Modified:
   pypy/branch/interplevel-codecs/pypy/interpreter/pyparser/parsestring.py
   pypy/branch/interplevel-codecs/pypy/interpreter/unicodehelper.py
   pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py
   pypy/branch/interplevel-codecs/pypy/objspace/std/marshal_impl.py
Log:
Simplify interpreter/unicodehelper.py, now it calls functions in interp_codecs, 
and avoids applevel code.

rlib.runicode can't be used directly, we need app-level exceptions.


Modified: pypy/branch/interplevel-codecs/pypy/interpreter/pyparser/parsestring.py
==============================================================================
--- pypy/branch/interplevel-codecs/pypy/interpreter/pyparser/parsestring.py	(original)
+++ pypy/branch/interplevel-codecs/pypy/interpreter/pyparser/parsestring.py	Tue Jun 29 17:46:51 2010
@@ -72,12 +72,12 @@
             bufp = 0
             bufq = len(buf)
         assert 0 <= bufp <= bufq
-        w_substr = space.wrap(buf[bufp : bufq])
+        substr = buf[bufp:bufq]
         if rawmode:
-            w_v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, w_substr)
+            v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, substr)
         else:
-            w_v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, w_substr)
-        return w_v
+            v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, substr)
+        return space.wrap(v)
 
     need_encoding = (encoding is not None and
                      encoding != "utf-8" and encoding != "iso-8859-1")
@@ -86,7 +86,7 @@
     substr = s[ps : q]
     if rawmode or '\\' not in s[ps:]:
         if need_encoding:
-            w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(substr))
+            w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
             #w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
             w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
             return w_v
@@ -200,7 +200,7 @@
     # while (s < end && *s != '\\') s++; */ /* inefficient for u".."
     while ps < end and ord(s[ps]) & 0x80:
         ps += 1
-    w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(s[pt : ps]))
+    w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, s[pt:ps]))
     w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
     v = space.str_w(w_v)
     return v, ps

Modified: pypy/branch/interplevel-codecs/pypy/interpreter/unicodehelper.py
==============================================================================
--- pypy/branch/interplevel-codecs/pypy/interpreter/unicodehelper.py	(original)
+++ pypy/branch/interplevel-codecs/pypy/interpreter/unicodehelper.py	Tue Jun 29 17:46:51 2010
@@ -1,30 +1,10 @@
-from pypy.interpreter import gateway
+from pypy.module._codecs import interp_codecs
 
-app = gateway.applevel(r'''
-    def PyUnicode_DecodeUnicodeEscape(data):
-        import _codecs
-        return _codecs.unicode_escape_decode(data)[0]
+def PyUnicode_AsEncodedString(space, w_data, w_encoding):
+    return interp_codecs.encode(space, w_data, w_encoding)
 
-    def PyUnicode_DecodeRawUnicodeEscape(data):
-        import _codecs
-        return _codecs.raw_unicode_escape_decode(data)[0]
-
-    def PyUnicode_DecodeUTF8(data):
-        import _codecs
-        return _codecs.utf_8_decode(data)[0]
-
-    def PyUnicode_AsEncodedString(data, encoding):
-        import _codecs
-        return _codecs.encode(data, encoding)
-
-    def PyUnicode_EncodeUTF8(data):
-        import _codecs
-        return _codecs.utf_8_encode(data)[0]
-
-''')
-
-PyUnicode_DecodeUnicodeEscape = app.interphook('PyUnicode_DecodeUnicodeEscape')
-PyUnicode_DecodeRawUnicodeEscape = app.interphook('PyUnicode_DecodeRawUnicodeEscape')
-PyUnicode_DecodeUTF8 = app.interphook('PyUnicode_DecodeUTF8')
-PyUnicode_AsEncodedString = app.interphook('PyUnicode_AsEncodedString')
-PyUnicode_EncodeUTF8 = app.interphook('PyUnicode_EncodeUTF8') 
+# These functions take and return unwrapped rpython strings and unicodes
+PyUnicode_DecodeUnicodeEscape = interp_codecs.make_raw_decoder('unicode_escape')
+PyUnicode_DecodeRawUnicodeEscape = interp_codecs.make_raw_decoder('raw_unicode_escape')
+PyUnicode_DecodeUTF8 = interp_codecs.make_raw_decoder('utf_8')
+PyUnicode_EncodeUTF8 = interp_codecs.make_raw_encoder('utf_8')

Modified: pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py
==============================================================================
--- pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py	(original)
+++ pypy/branch/interplevel-codecs/pypy/module/_codecs/interp_codecs.py	Tue Jun 29 17:46:51 2010
@@ -339,6 +339,31 @@
 
 from pypy.rlib import runicode
 
+def make_raw_encoder(name):
+    rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
+    assert hasattr(runicode, rname)
+    def raw_encoder(space, uni):
+        state = space.fromcache(CodecState)
+        func = getattr(runicode, rname)
+        errors = "strict"
+        return func(uni, len(uni), errors, state.encode_error_handler)
+    raw_encoder.func_name = rname
+    return raw_encoder
+
+def make_raw_decoder(name):
+    rname = "str_decode_%s" % (name.replace("_decode", ""), )
+    assert hasattr(runicode, rname)
+    def raw_decoder(space, string):
+        final = True
+        errors = "strict"
+        state = space.fromcache(CodecState)
+        func = getattr(runicode, rname)
+        result, consumed = func(string, len(string), errors,
+                                final, state.decode_error_handler)
+        return result
+    raw_decoder.func_name = rname
+    return raw_decoder
+
 def make_encoder_wrapper(name):
     rname = "unicode_encode_%s" % (name.replace("_encode", ""), )
     assert hasattr(runicode, rname)

Modified: pypy/branch/interplevel-codecs/pypy/objspace/std/marshal_impl.py
==============================================================================
--- pypy/branch/interplevel-codecs/pypy/objspace/std/marshal_impl.py	(original)
+++ pypy/branch/interplevel-codecs/pypy/objspace/std/marshal_impl.py	Tue Jun 29 17:46:51 2010
@@ -447,11 +447,11 @@
 register(TYPE_CODE, unmarshal_pycode)
 
 def marshal_w__Unicode(space, w_unicode, m):
-    s = space.str_w(unicodehelper.PyUnicode_EncodeUTF8(space, w_unicode))
+    s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode))
     m.atom_str(TYPE_UNICODE, s)
 
 def unmarshal_Unicode(space, u, tc):
-    return unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(u.get_str()))
+    return space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str()))
 register(TYPE_UNICODE, unmarshal_Unicode)
 
 app = gateway.applevel(r'''