[pypy-commit] pypy py3k: add a 'locale' codec for use by fsdecode/encode during interpreter bootstrap,
pjenvey
noreply at buildbot.pypy.org
Thu Feb 28 01:18:09 CET 2013
Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k
Changeset: r61871:1c77bd032140
Date: 2013-02-27 16:11 -0800
http://bitbucket.org/pypy/pypy/changeset/1c77bd032140/
Log: add a 'locale' codec for use by fsdecode/encode during interpreter
bootstrap, works via POSIX wcstombs/mbrtowc (and a chunk of C from
cpython)
diff --git a/pypy/module/_codecs/locale.c b/pypy/module/_codecs/locale.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/locale.c
@@ -0,0 +1,517 @@
+/* From CPython 3.2.3's fileutils.c, and _Py_normalize_encoding from
+ unicodeobject.c
+*/
+/*
+#include "Python.h"
+*/
+#include <stdlib.h>
+#include <string.h>
+#include <wchar.h>
+#define PyMem_Malloc malloc
+#define PyMem_Free free
+/* C99 but recent Windows has it */
+#define HAVE_MBRTOWC 1
+
+#ifdef MS_WINDOWS
+# include <windows.h>
+#endif
+
+#ifdef HAVE_LANGINFO_H
+#include <locale.h>
+#include <langinfo.h>
+#endif
+
+#if 0 && defined(__APPLE__)
+extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size);
+#endif
+
+#if !defined(__APPLE__) && !defined(MS_WINDOWS)
+#if 0
+extern int _pypy_normalize_encoding(const char *, char *, size_t);
+#endif
+
+/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale.
+ On these operating systems, nl_langinfo(CODESET) announces an alias of the
+ ASCII encoding, whereas mbstowcs() and wcstombs() functions use the
+ ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use
+ locale.getpreferredencoding() codec. For example, if command line arguments
+ are decoded by mbstowcs() and encoded back by os.fsencode(), we get a
+ UnicodeEncodeError instead of retrieving the original byte string.
+
+ The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C",
+ nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least
+ one byte in range 0x80-0xff can be decoded from the locale encoding. The
+ workaround is also enabled on error, for example if getting the locale
+ failed.
+
+ Values of locale_is_ascii:
+
+ 1: the workaround is used: _Py_wchar2char() uses
+ encode_ascii_surrogateescape() and _Py_char2wchar() uses
+ decode_ascii_surrogateescape()
+ 0: the workaround is not used: _Py_wchar2char() uses wcstombs() and
+ _Py_char2wchar() uses mbstowcs()
+ -1: unknown, need to call check_force_ascii() to get the value
+*/
+static int force_ascii = -1;
+
+static int
+_pypy_check_force_ascii(void)
+{
+ char *loc;
+#if defined(HAVE_LANGINFO_H) && defined(CODESET)
+ char *codeset, **alias;
+ char encoding[100];
+ int is_ascii;
+ unsigned int i;
+ char* ascii_aliases[] = {
+ "ascii",
+ "646",
+ "ansi-x3.4-1968",
+ "ansi-x3-4-1968",
+ "ansi-x3.4-1986",
+ "cp367",
+ "csascii",
+ "ibm367",
+ "iso646-us",
+ "iso-646.irv-1991",
+ "iso-ir-6",
+ "us",
+ "us-ascii",
+ NULL
+ };
+#endif
+
+ loc = setlocale(LC_CTYPE, NULL);
+ if (loc == NULL)
+ goto error;
+ if (strcmp(loc, "C") != 0) {
+ /* the LC_CTYPE locale is different than C */
+ return 0;
+ }
+
+#if defined(HAVE_LANGINFO_H) && defined(CODESET)
+ codeset = nl_langinfo(CODESET);
+ if (!codeset || codeset[0] == '\0') {
+ /* CODESET is not set or empty */
+ goto error;
+ }
+ if (!_pypy_normalize_encoding(codeset, encoding, sizeof(encoding)))
+ goto error;
+
+ is_ascii = 0;
+ for (alias=ascii_aliases; *alias != NULL; alias++) {
+ if (strcmp(encoding, *alias) == 0) {
+ is_ascii = 1;
+ break;
+ }
+ }
+ if (!is_ascii) {
+ /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */
+ return 0;
+ }
+
+ for (i=0x80; i<0xff; i++) {
+ unsigned char ch;
+ wchar_t wch;
+ size_t res;
+
+ ch = (unsigned char)i;
+ res = mbstowcs(&wch, (char*)&ch, 1);
+ if (res != (size_t)-1) {
+ /* decoding a non-ASCII character from the locale encoding succeed:
+ the locale encoding is not ASCII, force ASCII */
+ return 1;
+ }
+ }
+ /* None of the bytes in the range 0x80-0xff can be decoded from the locale
+ encoding: the locale encoding is really ASCII */
+ return 0;
+#else
+ /* nl_langinfo(CODESET) is not available: always force ASCII */
+ return 1;
+#endif
+
+error:
+ /* if an error occured, force the ASCII encoding */
+ return 1;
+}
+
+static char*
+_pypy_encode_ascii_surrogateescape(const wchar_t *text, size_t *error_pos)
+{
+ char *result = NULL, *out;
+ size_t len, i;
+ wchar_t ch;
+
+ if (error_pos != NULL)
+ *error_pos = (size_t)-1;
+
+ len = wcslen(text);
+
+ result = PyMem_Malloc(len + 1); /* +1 for NUL byte */
+ if (result == NULL)
+ return NULL;
+
+ out = result;
+ for (i=0; i<len; i++) {
+ ch = text[i];
+
+ if (ch <= 0x7f) {
+ /* ASCII character */
+ *out++ = (char)ch;
+ }
+ else if (0xdc80 <= ch && ch <= 0xdcff) {
+ /* UTF-8b surrogate */
+ *out++ = (char)(ch - 0xdc00);
+ }
+ else {
+ if (error_pos != NULL)
+ *error_pos = i;
+ PyMem_Free(result);
+ return NULL;
+ }
+ }
+ *out = '\0';
+ return result;
+}
+#endif /* !defined(__APPLE__) && !defined(MS_WINDOWS) */
+
+#if !defined(__APPLE__) && (!defined(MS_WINDOWS) || !defined(HAVE_MBRTOWC))
+static wchar_t*
+_pypy_decode_ascii_surrogateescape(const char *arg, size_t *size)
+{
+ wchar_t *res;
+ unsigned char *in;
+ wchar_t *out;
+
+ res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t));
+ if (!res)
+ return NULL;
+
+ in = (unsigned char*)arg;
+ out = res;
+ while(*in)
+ if(*in < 128)
+ *out++ = *in++;
+ else
+ *out++ = 0xdc00 + *in++;
+ *out = 0;
+ if (size != NULL)
+ *size = out - res;
+ return res;
+}
+#endif
+
+
+/* Decode a byte string from the locale encoding with the
+ surrogateescape error handler (undecodable bytes are decoded as characters
+ in range U+DC80..U+DCFF). If a byte sequence can be decoded as a surrogate
+ character, escape the bytes using the surrogateescape error handler instead
+ of decoding them.
+
+ Use _Py_wchar2char() to encode the character string back to a byte string.
+
+ Return a pointer to a newly allocated wide character string (use
+ PyMem_Free() to free the memory) and write the number of written wide
+ characters excluding the null character into *size if size is not NULL, or
+ NULL on error (conversion or memory allocation error).
+
+ Conversion errors should never happen, unless there is a bug in the C
+ library. */
+wchar_t*
+pypy_char2wchar(const char* arg, size_t *size)
+{
+#if 0 && defined(__APPLE__)
+ wchar_t *wstr;
+ wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg));
+ if (size != NULL) {
+ if (wstr != NULL)
+ *size = wcslen(wstr);
+ else
+ *size = (size_t)-1;
+ }
+ return wstr;
+#else
+ wchar_t *res;
+ size_t argsize;
+ size_t count;
+ unsigned char *in;
+ wchar_t *out;
+#ifdef HAVE_MBRTOWC
+ mbstate_t mbs;
+#endif
+
+#if !defined(__APPLE__) && !defined(MS_WINDOWS)
+/*#ifndef MS_WINDOWS*/
+ if (force_ascii == -1)
+ force_ascii = _pypy_check_force_ascii();
+
+ if (force_ascii) {
+ /* force ASCII encoding to workaround mbstowcs() issue */
+ res = _pypy_decode_ascii_surrogateescape(arg, size);
+ if (res == NULL)
+ goto oom;
+ return res;
+ }
+#endif
+
+#ifdef HAVE_BROKEN_MBSTOWCS
+ /* Some platforms have a broken implementation of
+ * mbstowcs which does not count the characters that
+ * would result from conversion. Use an upper bound.
+ */
+ argsize = strlen(arg);
+#else
+ argsize = mbstowcs(NULL, arg, 0);
+#endif
+ if (argsize != (size_t)-1) {
+ res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+ if (!res)
+ goto oom;
+ count = mbstowcs(res, arg, argsize+1);
+ if (count != (size_t)-1) {
+ wchar_t *tmp;
+ /* Only use the result if it contains no
+ surrogate characters. */
+ for (tmp = res; *tmp != 0 &&
+ (*tmp < 0xd800 || *tmp > 0xdfff); tmp++)
+ ;
+ if (*tmp == 0) {
+ if (size != NULL)
+ *size = count;
+ return res;
+ }
+ }
+ PyMem_Free(res);
+ }
+ /* Conversion failed. Fall back to escaping with surrogateescape. */
+#ifdef HAVE_MBRTOWC
+ /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */
+
+ /* Overallocate; as multi-byte characters are in the argument, the
+ actual output could use less memory. */
+ argsize = strlen(arg) + 1;
+ res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t));
+ if (!res)
+ goto oom;
+ in = (unsigned char*)arg;
+ out = res;
+ memset(&mbs, 0, sizeof mbs);
+ while (argsize) {
+ size_t converted = mbrtowc(out, (char*)in, argsize, &mbs);
+ if (converted == 0)
+ /* Reached end of string; null char stored. */
+ break;
+ if (converted == (size_t)-2) {
+ /* Incomplete character. This should never happen,
+ since we provide everything that we have -
+ unless there is a bug in the C library, or I
+ misunderstood how mbrtowc works. */
+ fprintf(stderr, "unexpected mbrtowc result -2\n");
+ PyMem_Free(res);
+ return NULL;
+ }
+ if (converted == (size_t)-1) {
+ /* Conversion error. Escape as UTF-8b, and start over
+ in the initial shift state. */
+ *out++ = 0xdc00 + *in++;
+ argsize--;
+ memset(&mbs, 0, sizeof mbs);
+ continue;
+ }
+ if (*out >= 0xd800 && *out <= 0xdfff) {
+ /* Surrogate character. Escape the original
+ byte sequence with surrogateescape. */
+ argsize -= converted;
+ while (converted--)
+ *out++ = 0xdc00 + *in++;
+ continue;
+ }
+ /* successfully converted some bytes */
+ in += converted;
+ argsize -= converted;
+ out++;
+ }
+ if (size != NULL)
+ *size = out - res;
+#else /* HAVE_MBRTOWC */
+ /* Cannot use C locale for escaping; manually escape as if charset
+ is ASCII (i.e. escape all bytes > 128. This will still roundtrip
+ correctly in the locale's charset, which must be an ASCII superset. */
+ res = _pypy_decode_ascii_surrogateescape(arg, size);
+ if (res == NULL)
+ goto oom;
+#endif /* HAVE_MBRTOWC */
+ return res;
+oom:
+ fprintf(stderr, "out of memory\n");
+ return NULL;
+#endif /* __APPLE__ */
+}
+
+/* Encode a (wide) character string to the locale encoding with the
+ surrogateescape error handler (characters in range U+DC80..U+DCFF are
+ converted to bytes 0x80..0xFF).
+
+ This function is the reverse of _Py_char2wchar().
+
+ Return a pointer to a newly allocated byte string (use PyMem_Free() to free
+ the memory), or NULL on conversion or memory allocation error.
+
+ If error_pos is not NULL: *error_pos is the index of the invalid character
+ on conversion error, or (size_t)-1 otherwise. */
+char*
+pypy_wchar2char(const wchar_t *text, size_t *error_pos)
+{
+#if 0 && defined(__APPLE__)
+ Py_ssize_t len;
+ PyObject *unicode, *bytes = NULL;
+ char *cpath;
+
+ unicode = PyUnicode_FromWideChar(text, wcslen(text));
+ if (unicode == NULL)
+ return NULL;
+
+ bytes = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
+ PyUnicode_GET_SIZE(unicode),
+ "surrogateescape");
+ Py_DECREF(unicode);
+ if (bytes == NULL) {
+ PyErr_Clear();
+ if (error_pos != NULL)
+ *error_pos = (size_t)-1;
+ return NULL;
+ }
+
+ len = PyBytes_GET_SIZE(bytes);
+ cpath = PyMem_Malloc(len+1);
+ if (cpath == NULL) {
+ PyErr_Clear();
+ Py_DECREF(bytes);
+ if (error_pos != NULL)
+ *error_pos = (size_t)-1;
+ return NULL;
+ }
+ memcpy(cpath, PyBytes_AsString(bytes), len + 1);
+ Py_DECREF(bytes);
+ return cpath;
+#else /* __APPLE__ */
+ const size_t len = wcslen(text);
+ char *result = NULL, *bytes = NULL;
+ size_t i, size, converted;
+ wchar_t c, buf[2];
+
+#if !defined(__APPLE__) && !defined(MS_WINDOWS)
+/*#ifndef MS_WINDOWS*/
+ if (force_ascii == -1)
+ force_ascii = _pypy_check_force_ascii();
+
+ if (force_ascii)
+ return _pypy_encode_ascii_surrogateescape(text, error_pos);
+#endif
+
+ /* The function works in two steps:
+ 1. compute the length of the output buffer in bytes (size)
+ 2. outputs the bytes */
+ size = 0;
+ buf[1] = 0;
+ while (1) {
+ for (i=0; i < len; i++) {
+ c = text[i];
+ if (c >= 0xdc80 && c <= 0xdcff) {
+ /* UTF-8b surrogate */
+ if (bytes != NULL) {
+ *bytes++ = c - 0xdc00;
+ size--;
+ }
+ else
+ size++;
+ continue;
+ }
+ else {
+ buf[0] = c;
+ if (bytes != NULL)
+ converted = wcstombs(bytes, buf, size);
+ else
+ converted = wcstombs(NULL, buf, 0);
+ if (converted == (size_t)-1) {
+ if (result != NULL)
+ PyMem_Free(result);
+ if (error_pos != NULL)
+ *error_pos = i;
+ return NULL;
+ }
+ if (bytes != NULL) {
+ bytes += converted;
+ size -= converted;
+ }
+ else
+ size += converted;
+ }
+ }
+ if (result != NULL) {
+ *bytes = '\0';
+ break;
+ }
+
+ size += 1; /* nul byte at the end */
+ result = PyMem_Malloc(size);
+ if (result == NULL) {
+ if (error_pos != NULL)
+ *error_pos = (size_t)-1;
+ return NULL;
+ }
+ bytes = result;
+ }
+ return result;
+#endif /* __APPLE__ */
+}
+
+void
+pypy_char2wchar_free(wchar_t *text)
+{
+ PyMem_Free(text);
+}
+
+void
+pypy_wchar2char_free(char *bytes)
+{
+ PyMem_Free(bytes);
+}
+
+#define Py_ISUPPER isupper
+#define Py_TOLOWER tolower
+
+/* Convert encoding to lower case and replace '_' with '-' in order to
+ catch e.g. UTF_8. Return 0 on error (encoding is longer than lower_len-1),
+ 1 on success. */
+int
+_pypy_normalize_encoding(const char *encoding,
+ char *lower,
+ size_t lower_len)
+{
+ const char *e;
+ char *l;
+ char *l_end;
+
+ e = encoding;
+ l = lower;
+ l_end = &lower[lower_len - 1];
+ while (*e) {
+ if (l == l_end)
+ return 0;
+ if (Py_ISUPPER(*e)) {
+ *l++ = Py_TOLOWER(*e++);
+ }
+ else if (*e == '_') {
+ *l++ = '-';
+ e++;
+ }
+ else {
+ *l++ = *e++;
+ }
+ }
+ *l = '\0';
+ return 1;
+}
diff --git a/pypy/module/_codecs/locale.py b/pypy/module/_codecs/locale.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/locale.py
@@ -0,0 +1,153 @@
+"""
+Provides internal 'locale' codecs (via POSIX wcstombs/mbrtowc) for use
+by PyUnicode_Decode/EncodeFSDefault during interpreter bootstrap
+"""
+import os
+import py
+import sys
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rstring import UnicodeBuilder, assert_str0
+from rpython.rlib.runicode import (code_to_unichr,
+ default_unicode_error_decode, default_unicode_error_encode)
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+if we_are_translated():
+ UNICHAR_SIZE = rffi.sizeof(lltype.UniChar)
+else:
+ UNICHAR_SIZE = 2 if sys.maxunicode == 0xFFFF else 4
+MERGE_SURROGATES = UNICHAR_SIZE == 2 and rffi.sizeof(rffi.WCHAR_T) == 4
+
+
+cwd = py.path.local(__file__).dirpath()
+eci = ExternalCompilationInfo(
+ separate_module_files=[cwd.join('locale.c')],
+ export_symbols=['pypy_char2wchar', 'pypy_char2wchar_free',
+ 'pypy_wchar2char', 'pypy_wchar2char_free'])
+
+def llexternal(*args, **kwargs):
+ kwargs.setdefault('compilation_info', eci)
+ kwargs.setdefault('sandboxsafe', True)
+ kwargs.setdefault('_nowrapper', True)
+ return rffi.llexternal(*args, **kwargs)
+
+# An actual wchar_t*, rffi.CWCHARP is an array of UniChar (possibly on a
+# narrow build)
+RAW_WCHARP = lltype.Ptr(lltype.Array(rffi.WCHAR_T, hints={'nolength': True}))
+pypy_char2wchar = llexternal('pypy_char2wchar', [rffi.CCHARP, rffi.SIZE_TP],
+ RAW_WCHARP)
+pypy_char2wchar_free = llexternal('pypy_char2wchar_free', [RAW_WCHARP],
+ lltype.Void)
+pypy_wchar2char = llexternal('pypy_wchar2char', [RAW_WCHARP, rffi.SIZE_TP],
+ rffi.CCHARP)
+pypy_wchar2char_free = llexternal('pypy_wchar2char_free', [rffi.CCHARP],
+ lltype.Void)
+
+
+def unicode_encode_locale_surrogateescape(u, errorhandler=None):
+ """Encode unicode via the locale codecs (POSIX wcstombs) with the
+ surrogateescape handler.
+
+ The optional errorhandler is only called in the case of fatal
+ errors.
+ """
+ if errorhandler is None:
+ errorhandler = default_unicode_error_encode
+
+ with lltype.scoped_alloc(rffi.SIZE_TP.TO, 1) as errorposp:
+ with scoped_unicode2rawwcharp(u) as ubuf:
+ sbuf = pypy_wchar2char(ubuf, errorposp)
+ try:
+ if sbuf is None:
+ errorpos = rffi.cast(lltype.Signed, errorposp[0])
+ if errorpos == -1:
+ raise MemoryError
+ errmsg = _errmsg("pypy_wchar2char")
+ errorhandler('strict', 'filesystemencoding', errmsg, u,
+ errorpos, errorpos + 1)
+ return rffi.charp2str(sbuf)
+ finally:
+ pypy_wchar2char_free(sbuf)
+
+
+def unicode_decode_locale_surrogateescape(s, errorhandler=None):
+ """Decode strs via the locale codecs (POSIX mrbtowc) with the
+ surrogateescape handler.
+
+ The optional errorhandler is only called in the case of fatal
+ errors.
+ """
+ if errorhandler is None:
+ errorhandler = default_unicode_error_decode
+
+ with lltype.scoped_alloc(rffi.SIZE_TP.TO, 1) as sizep:
+ with rffi.scoped_str2charp(s) as sbuf:
+ ubuf = pypy_char2wchar(sbuf, sizep)
+ try:
+ if ubuf is None:
+ errmsg = _errmsg("pypy_char2wchar")
+ errorhandler('strict', 'filesystemencoding', errmsg, s, 0, 1)
+ size = rffi.cast(lltype.Signed, sizep[0])
+ return rawwcharp2unicoden(ubuf, size)
+ finally:
+ pypy_char2wchar_free(ubuf)
+
+
+def _errmsg(what):
+ from rpython.rlib import rposix
+ errmsg = os.strerror(rposix.get_errno())
+ return "%s failed" % what if errmsg is None else errmsg
+
+
+class scoped_unicode2rawwcharp:
+ def __init__(self, value):
+ if value is not None:
+ self.buf = unicode2rawwcharp(value)
+ else:
+ self.buf = lltype.nullptr(RAW_WCHARP.TO)
+ def __enter__(self):
+ return self.buf
+ def __exit__(self, *args):
+ if self.buf:
+ lltype.free(self.buf, flavor='raw')
+
+def unicode2rawwcharp(u):
+ """unicode -> raw wchar_t*"""
+ size = _unicode2cwcharp_loop(u, None) if MERGE_SURROGATES else len(u)
+ array = lltype.malloc(RAW_WCHARP.TO, size + 1, flavor='raw')
+ array[size] = rffi.cast(rffi.WCHAR_T, u'\x00')
+ _unicode2cwcharp_loop(u, array)
+ return array
+unicode2rawwcharp._annenforceargs_ = [unicode]
+
+def _unicode2cwcharp_loop(u, array):
+ write = array is not None
+ ulen = len(u)
+ count = i = 0
+ while i < ulen:
+ oc = ord(u[i])
+ if (MERGE_SURROGATES and
+ 0xD800 <= oc <= 0xDBFF and i + 1 < ulen and
+ 0xDC00 <= ord(u[i + 1]) <= 0xDFFF):
+ if write:
+ merged = (((oc & 0x03FF) << 10) |
+ (ord(u[i + 1]) & 0x03FF)) + 0x10000
+ array[count] = rffi.cast(rffi.WCHAR_T, merged)
+ i += 2
+ else:
+ if write:
+ array[count] = rffi.cast(rffi.WCHAR_T, oc)
+ i += 1
+ count += 1
+ return count
+unicode2rawwcharp._annenforceargs_ = [unicode, None]
+
+
+def rawwcharp2unicoden(wcp, maxlen):
+ b = UnicodeBuilder(maxlen)
+ i = 0
+ while i < maxlen and wcp[i] != u'\x00':
+ b.append(code_to_unichr(wcp[i]))
+ i += 1
+ return assert_str0(b.build())
+unicode2rawwcharp._annenforceargs_ = [None, int]
diff --git a/pypy/module/_codecs/test/test_locale.py b/pypy/module/_codecs/test/test_locale.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_codecs/test/test_locale.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+import py
+from pypy.module._codecs import interp_codecs
+from pypy.module._codecs.locale import (
+ unicode_decode_locale_surrogateescape,
+ unicode_encode_locale_surrogateescape)
+from rpython.rlib import rlocale, runicode
+
+class TestLocaleCodec(object):
+
+ def setup_class(cls):
+ from rpython.rlib import rlocale
+ cls.oldlocale = rlocale.setlocale(rlocale.LC_ALL, None)
+
+ def teardown_class(cls):
+ if hasattr(cls, 'oldlocale'):
+ from rpython.rlib import rlocale
+ rlocale.setlocale(rlocale.LC_ALL, cls.oldlocale)
+
+ def getdecoder(self, encoding):
+ return getattr(runicode, "str_decode_%s" % encoding.replace("-", "_"))
+
+ def getencoder(self, encoding):
+ return getattr(runicode,
+ "unicode_encode_%s" % encoding.replace("-", "_"))
+
+ def getstate(self):
+ return self.space.fromcache(interp_codecs.CodecState)
+
+ def setlocale(self, locale):
+ from rpython.rlib import rlocale
+ try:
+ rlocale.setlocale(rlocale.LC_ALL, locale)
+ except rlocale.LocaleError:
+ py.test.skip("%s locale unsupported" % locale)
+
+ def test_encode_locale(self):
+ self.setlocale("en_US.UTF-8")
+ locale_encoder = unicode_encode_locale_surrogateescape
+ utf8_encoder = self.getencoder('utf-8')
+ for val in u'foo', u' 日本', u'\U0001320C':
+ assert (locale_encoder(val) ==
+ utf8_encoder(val, len(val), None))
+
+ def test_encode_locale_errorhandler(self):
+ self.setlocale("en_US.UTF-8")
+ locale_encoder = unicode_encode_locale_surrogateescape
+ utf8_encoder = self.getencoder('utf-8')
+ encode_error_handler = self.getstate().encode_error_handler
+ for val in u'foo\udc80bar', u'\udcff\U0001320C':
+ expected = utf8_encoder(val, len(val), 'surrogateescape',
+ encode_error_handler)
+ assert locale_encoder(val) == expected
+
+ def test_decode_locale(self):
+ self.setlocale("en_US.UTF-8")
+ locale_decoder = unicode_decode_locale_surrogateescape
+ utf8_decoder = self.getdecoder('utf-8')
+ for val in 'foo', ' \xe6\x97\xa5\xe6\x9c\xac', '\xf0\x93\x88\x8c':
+ assert (locale_decoder(val) ==
+ utf8_decoder(val, len(val), None)[0])
+
+ def test_decode_locale_errorhandler(self):
+ self.setlocale("en_US.UTF-8")
+ locale_decoder = unicode_decode_locale_surrogateescape
+ utf8_decoder = self.getdecoder('utf-8')
+ decode_error_handler = self.getstate().decode_error_handler
+ val = 'foo\xe3bar'
+ expected = utf8_decoder(val, len(val), 'surrogateescape', True,
+ decode_error_handler)[0]
+ assert locale_decoder(val) == expected
More information about the pypy-commit
mailing list