[pypy-commit] pypy default: Issue1627. Test and fix: it's a bit strange, but we need to follow

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r67934:1dc9140b7f44
Date: 2013-11-10 23:08 +0100
http://bitbucket.org/pypy/pypy/changeset/1dc9140b7f44/

Log:	Issue1627. Test and fix: it's a bit strange, but we need to follow
	CPython's lead here and return a string as long as possible, and a
	unicode only when asked to.

diff --git a/lib-python/2.7/json/encoder.py b/lib-python/2.7/json/encoder.py
--- a/lib-python/2.7/json/encoder.py
+++ b/lib-python/2.7/json/encoder.py
@@ -4,6 +4,21 @@
 
 from __pypy__.builders import StringBuilder, UnicodeBuilder
 
+class StringOrUnicodeBuilder(object):
+    def __init__(self):
+        self._builder = StringBuilder()
+    def append(self, string):
+        try:
+            self._builder.append(string)
+        except UnicodeEncodeError:
+            ub = UnicodeBuilder()
+            ub.append(self._builder.build())
+            self._builder = ub
+            ub.append(string)
+    def build(self):
+        return self._builder.build()
+
+
 ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
 ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
 HAS_UTF8 = re.compile(r'[\x80-\xff]')
@@ -192,7 +207,7 @@
         if self.ensure_ascii:
             builder = StringBuilder()
         else:
-            builder = UnicodeBuilder()
+            builder = StringOrUnicodeBuilder()
         self.__encode(o, markers, builder, 0)
         return builder.build()
 
diff --git a/pypy/module/test_lib_pypy/test_json_extra.py b/pypy/module/test_lib_pypy/test_json_extra.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/test_json_extra.py
@@ -0,0 +1,14 @@
+import py, json
+
+def is_(x, y):
+    return type(x) is type(y) and x == y
+
+def test_no_ensure_ascii():
+    assert is_(json.dumps(u"\u1234", ensure_ascii=False), u'"\u1234"')
+    assert is_(json.dumps("\xc0", ensure_ascii=False), '"\xc0"')
+    e = py.test.raises(UnicodeDecodeError, json.dumps,
+                       (u"\u1234", "\xc0"), ensure_ascii=False)
+    assert str(e.value).startswith("'ascii' codec can't decode byte 0xc0 ")
+    e = py.test.raises(UnicodeDecodeError, json.dumps,
+                       ("\xc0", u"\u1234"), ensure_ascii=False)
+    assert str(e.value).startswith("'ascii' codec can't decode byte 0xc0 ")