[pypy-commit] pypy rpython-unicode-formatting: add support for unicode formatting in the rtyper: so far lltype only, and it probably does not work as intended when using %s on a unicode string

[antocuni]

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: rpython-unicode-formatting
Changeset: r56134:4cce4c57a4c6
Date: 2012-07-18 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/4cce4c57a4c6/

Log:	add support for unicode formatting in the rtyper: so far lltype
	only, and it probably does not work as intended when using %s on a
	unicode string

diff --git a/pypy/rpython/lltypesystem/rstr.py b/pypy/rpython/lltypesystem/rstr.py
--- a/pypy/rpython/lltypesystem/rstr.py
+++ b/pypy/rpython/lltypesystem/rstr.py
@@ -1,5 +1,6 @@
 from weakref import WeakValueDictionary
 from pypy.tool.pairtype import pairtype
+from pypy.annotation import model as annmodel
 from pypy.rpython.error import TyperError
 from pypy.rlib.objectmodel import malloc_zero_filled, we_are_translated
 from pypy.rlib.objectmodel import _hash_string, enforceargs
@@ -962,13 +963,18 @@
     def do_stringformat(cls, hop, sourcevarsrepr):
         s_str = hop.args_s[0]
         assert s_str.is_constant()
+        is_unicode = isinstance(s_str, annmodel.SomeUnicodeString)
+        if is_unicode:
+            TEMPBUF = TEMP_UNICODE
+        else:
+            TEMPBUF = TEMP
         s = s_str.const
         things = cls.parse_fmt_string(s)
         size = inputconst(Signed, len(things)) # could be unsigned?
-        cTEMP = inputconst(Void, TEMP)
+        cTEMP = inputconst(Void, TEMPBUF)
         cflags = inputconst(Void, {'flavor': 'gc'})
         vtemp = hop.genop("malloc_varsize", [cTEMP, cflags, size],
-                          resulttype=Ptr(TEMP))
+                          resulttype=Ptr(TEMPBUF))
 
         argsiter = iter(sourcevarsrepr)
 
@@ -980,6 +986,7 @@
                 if not hasattr(r_arg, 'll_str'):
                     raise TyperError("ll_str unsupported for: %r" % r_arg)
                 if code == 's' or (code == 'r' and isinstance(r_arg, InstanceRepr)):
+                    # XXX: if it's unicode we don't want to call ll_str
                     vchunk = hop.gendirectcall(r_arg.ll_str, vitem)
                 elif code == 'd':
                     assert isinstance(r_arg, IntegerRepr)
@@ -999,9 +1006,17 @@
                 else:
                     raise TyperError, "%%%s is not RPython" % (code, )
             else:
-                from pypy.rpython.lltypesystem.rstr import string_repr
-                vchunk = inputconst(string_repr, thing)
+                from pypy.rpython.lltypesystem.rstr import string_repr, unicode_repr
+                if is_unicode:
+                    vchunk = inputconst(unicode_repr, thing)
+                else:
+                    vchunk = inputconst(string_repr, thing)
             i = inputconst(Signed, i)
+            if is_unicode and vchunk.concretetype != Ptr(UNICODE):
+                # if we are here, one of the ll_str.* functions returned some
+                # STR, so we convert it to unicode. It's a bit suboptimal
+                # because we do one extra copy.
+                vchunk = hop.gendirectcall(string_repr.ll_decode_latin1, vchunk)
             hop.genop('setarrayitem', [vtemp, i, vchunk])
 
         hop.exception_cannot_occur()   # to ignore the ZeroDivisionError of '%'
@@ -1009,6 +1024,7 @@
     do_stringformat = classmethod(do_stringformat)
 
 TEMP = GcArray(Ptr(STR))
+TEMP_UNICODE = GcArray(Ptr(UNICODE))
 
 # ____________________________________________________________
 
diff --git a/pypy/rpython/test/test_runicode.py b/pypy/rpython/test/test_runicode.py
--- a/pypy/rpython/test/test_runicode.py
+++ b/pypy/rpython/test/test_runicode.py
@@ -202,12 +202,6 @@
     test_upper = unsupported
     test_lower = unsupported
     test_splitlines = unsupported
-    test_strformat = unsupported
-    test_strformat_instance = unsupported
-    test_strformat_nontuple = unsupported
-    test_percentformat_instance = unsupported
-    test_percentformat_tuple = unsupported
-    test_percentformat_list = unsupported
     test_int = unsupported
     test_int_valueerror = unsupported
     test_float = unsupported