[pypy-svn] r69298 - pypy/branch/faster-raise/lib-python/modified-2.5.2/test
fijal at codespeak.net
fijal at codespeak.net
Sun Nov 15 11:33:41 CET 2009
Author: fijal
Date: Sun Nov 15 11:33:39 2009
New Revision: 69298
Added:
pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py (contents, props changed)
Log:
Modify a test, so errors while parsing arguments can be
decoding errors as well
Added: pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py
==============================================================================
--- (empty file)
+++ pypy/branch/faster-raise/lib-python/modified-2.5.2/test/test_codeccallbacks.py Sun Nov 15 11:33:39 2009
@@ -0,0 +1,801 @@
+import test.test_support, unittest
+import sys, codecs, htmlentitydefs, unicodedata
+
+class PosReturn:
+ # this can be used for configurable callbacks
+
+ def __init__(self):
+ self.pos = 0
+
+ def handle(self, exc):
+ oldpos = self.pos
+ realpos = oldpos
+ if realpos<0:
+ realpos = len(exc.object) + realpos
+ # if we don't advance this time, terminate on the next call
+ # otherwise we'd get an endless loop
+ if realpos <= exc.start:
+ self.pos = len(exc.object)
+ return (u"<?>", oldpos)
+
+# A UnicodeEncodeError object with a bad start attribute
+class BadStartUnicodeEncodeError(UnicodeEncodeError):
+ def __init__(self):
+ UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
+ self.start = []
+
+# A UnicodeEncodeError object with a bad object attribute
+class BadObjectUnicodeEncodeError(UnicodeEncodeError):
+ def __init__(self):
+ UnicodeEncodeError.__init__(self, "ascii", u"", 0, 1, "bad")
+ self.object = []
+
+# A UnicodeDecodeError object without an end attribute
+class NoEndUnicodeDecodeError(UnicodeDecodeError):
+ def __init__(self):
+ UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
+ del self.end
+
+# A UnicodeDecodeError object with a bad object attribute
+class BadObjectUnicodeDecodeError(UnicodeDecodeError):
+ def __init__(self):
+ UnicodeDecodeError.__init__(self, "ascii", "", 0, 1, "bad")
+ self.object = []
+
+# A UnicodeTranslateError object without a start attribute
+class NoStartUnicodeTranslateError(UnicodeTranslateError):
+ def __init__(self):
+ UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
+ del self.start
+
+# A UnicodeTranslateError object without an end attribute
+class NoEndUnicodeTranslateError(UnicodeTranslateError):
+ def __init__(self):
+ UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
+ del self.end
+
+# A UnicodeTranslateError object without an object attribute
+class NoObjectUnicodeTranslateError(UnicodeTranslateError):
+ def __init__(self):
+ UnicodeTranslateError.__init__(self, u"", 0, 1, "bad")
+ del self.object
+
+class CodecCallbackTest(unittest.TestCase):
+
+ def test_xmlcharrefreplace(self):
+ # replace unencodable characters which numeric character entities.
+ # For ascii, latin-1 and charmaps this is completely implemented
+ # in C and should be reasonably fast.
+ s = u"\u30b9\u30d1\u30e2 \xe4nd eggs"
+ self.assertEqual(
+ s.encode("ascii", "xmlcharrefreplace"),
+ "スパモ änd eggs"
+ )
+ self.assertEqual(
+ s.encode("latin-1", "xmlcharrefreplace"),
+ "スパモ \xe4nd eggs"
+ )
+
+ def test_xmlcharnamereplace(self):
+ # This time use a named character entity for unencodable
+ # characters, if one is available.
+
+ def xmlcharnamereplace(exc):
+ if not isinstance(exc, UnicodeEncodeError):
+ raise TypeError("don't know how to handle %r" % exc)
+ l = []
+ for c in exc.object[exc.start:exc.end]:
+ try:
+ l.append(u"&%s;" % htmlentitydefs.codepoint2name[ord(c)])
+ except KeyError:
+ l.append(u"&#%d;" % ord(c))
+ return (u"".join(l), exc.end)
+
+ codecs.register_error(
+ "test.xmlcharnamereplace", xmlcharnamereplace)
+
+ sin = u"\xab\u211c\xbb = \u2329\u1234\u20ac\u232a"
+ sout = "«ℜ» = ⟨ሴ€⟩"
+ self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout)
+ sout = "\xabℜ\xbb = ⟨ሴ€⟩"
+ self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout)
+ sout = "\xabℜ\xbb = ⟨ሴ\xa4⟩"
+ self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout)
+
+ def test_uninamereplace(self):
+ # We're using the names from the unicode database this time,
+ # and we're doing "syntax highlighting" here, i.e. we include
+ # the replaced text in ANSI escape sequences. For this it is
+ # useful that the error handler is not called for every single
+ # unencodable character, but for a complete sequence of
+ # unencodable characters, otherwise we would output many
+ # unneccessary escape sequences.
+
+ def uninamereplace(exc):
+ if not isinstance(exc, UnicodeEncodeError):
+ raise TypeError("don't know how to handle %r" % exc)
+ l = []
+ for c in exc.object[exc.start:exc.end]:
+ l.append(unicodedata.name(c, u"0x%x" % ord(c)))
+ return (u"\033[1m%s\033[0m" % u", ".join(l), exc.end)
+
+ codecs.register_error(
+ "test.uninamereplace", uninamereplace)
+
+ sin = u"\xac\u1234\u20ac\u8000"
+ sout = "\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
+ self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout)
+
+ sout = "\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
+ self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout)
+
+ sout = "\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m"
+ self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout)
+
+ def test_backslashescape(self):
+ # Does the same as the "unicode-escape" encoding, but with different
+ # base encodings.
+ sin = u"a\xac\u1234\u20ac\u8000"
+ if sys.maxunicode > 0xffff:
+ sin += unichr(sys.maxunicode)
+ sout = "a\\xac\\u1234\\u20ac\\u8000"
+ if sys.maxunicode > 0xffff:
+ sout += "\\U%08x" % sys.maxunicode
+ self.assertEqual(sin.encode("ascii", "backslashreplace"), sout)
+
+ sout = "a\xac\\u1234\\u20ac\\u8000"
+ if sys.maxunicode > 0xffff:
+ sout += "\\U%08x" % sys.maxunicode
+ self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout)
+
+ sout = "a\xac\\u1234\xa4\\u8000"
+ if sys.maxunicode > 0xffff:
+ sout += "\\U%08x" % sys.maxunicode
+ self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout)
+
+ def test_decoderelaxedutf8(self):
+ # This is the test for a decoding callback handler,
+ # that relaxes the UTF-8 minimal encoding restriction.
+ # A null byte that is encoded as "\xc0\x80" will be
+ # decoded as a null byte. All other illegal sequences
+ # will be handled strictly.
+ def relaxedutf8(exc):
+ if not isinstance(exc, UnicodeDecodeError):
+ raise TypeError("don't know how to handle %r" % exc)
+ if exc.object[exc.start:exc.end].startswith("\xc0\x80"):
+ return (u"\x00", exc.start+2) # retry after two bytes
+ else:
+ raise exc
+
+ codecs.register_error(
+ "test.relaxedutf8", relaxedutf8)
+
+ sin = "a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80"
+ sout = u"a\x00b\x00c\xfc\x00\x00"
+ self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout)
+ sin = "\xc0\x80\xc0\x81"
+ self.assertRaises(UnicodeError, sin.decode, "utf-8", "test.relaxedutf8")
+
+ def test_charmapencode(self):
+ # For charmap encodings the replacement string will be
+ # mapped through the encoding again. This means, that
+ # to be able to use e.g. the "replace" handler, the
+ # charmap has to have a mapping for "?".
+ charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
+ sin = u"abc"
+ sout = "AABBCC"
+ self.assertEquals(codecs.charmap_encode(sin, "strict", charmap)[0], sout)
+
+ sin = u"abcA"
+ self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap)
+
+ charmap[ord("?")] = "XYZ"
+ sin = u"abcDEF"
+ sout = "AABBCCXYZXYZXYZ"
+ self.assertEquals(codecs.charmap_encode(sin, "replace", charmap)[0], sout)
+
+ charmap[ord("?")] = u"XYZ"
+ self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
+
+ charmap[ord("?")] = u"XYZ"
+ self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
+
+ def test_decodeunicodeinternal(self):
+ self.assertRaises(
+ UnicodeDecodeError,
+ "\x00\x00\x00\x00\x00".decode,
+ "unicode-internal",
+ )
+ if sys.maxunicode > 0xffff:
+ def handler_unicodeinternal(exc):
+ if not isinstance(exc, UnicodeDecodeError):
+ raise TypeError("don't know how to handle %r" % exc)
+ return (u"\x01", 1)
+
+ self.assertEqual(
+ "\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"),
+ u"\u0000"
+ )
+
+ self.assertEqual(
+ "\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"),
+ u"\u0000\ufffd"
+ )
+
+ codecs.register_error("test.hui", handler_unicodeinternal)
+
+ self.assertEqual(
+ "\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"),
+ u"\u0000\u0001\u0000"
+ )
+
+ def test_callbacks(self):
+ def handler1(exc):
+ if not isinstance(exc, UnicodeEncodeError) \
+ and not isinstance(exc, UnicodeDecodeError):
+ raise TypeError("don't know how to handle %r" % exc)
+ l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
+ return (u"[%s]" % u"".join(l), exc.end)
+
+ codecs.register_error("test.handler1", handler1)
+
+ def handler2(exc):
+ if not isinstance(exc, UnicodeDecodeError):
+ raise TypeError("don't know how to handle %r" % exc)
+ l = [u"<%d>" % ord(exc.object[pos]) for pos in xrange(exc.start, exc.end)]
+ return (u"[%s]" % u"".join(l), exc.end+1) # skip one character
+
+ codecs.register_error("test.handler2", handler2)
+
+ s = "\x00\x81\x7f\x80\xff"
+
+ self.assertEqual(
+ s.decode("ascii", "test.handler1"),
+ u"\x00[<129>]\x7f[<128>][<255>]"
+ )
+ self.assertEqual(
+ s.decode("ascii", "test.handler2"),
+ u"\x00[<129>][<128>]"
+ )
+
+ self.assertEqual(
+ "\\u3042\u3xxx".decode("unicode-escape", "test.handler1"),
+ u"\u3042[<92><117><51><120>]xx"
+ )
+
+ self.assertEqual(
+ "\\u3042\u3xx".decode("unicode-escape", "test.handler1"),
+ u"\u3042[<92><117><51><120><120>]"
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("abc", "test.handler1", {ord("a"): u"z"})[0],
+ u"z[<98>][<99>]"
+ )
+
+ self.assertEqual(
+ u"g\xfc\xdfrk".encode("ascii", "test.handler1"),
+ u"g[<252><223>]rk"
+ )
+
+ self.assertEqual(
+ u"g\xfc\xdf".encode("ascii", "test.handler1"),
+ u"g[<252><223>]"
+ )
+
+ def test_longstrings(self):
+ # test long strings to check for memory overflow problems
+ errors = [ "strict", "ignore", "replace", "xmlcharrefreplace", "backslashreplace"]
+ # register the handlers under different names,
+ # to prevent the codec from recognizing the name
+ for err in errors:
+ codecs.register_error("test." + err, codecs.lookup_error(err))
+ l = 1000
+ errors += [ "test." + err for err in errors ]
+ for uni in [ s*l for s in (u"x", u"\u3042", u"a\xe4") ]:
+ for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15", "utf-8", "utf-7", "utf-16"):
+ for err in errors:
+ try:
+ uni.encode(enc, err)
+ except UnicodeError:
+ pass
+
+ def check_exceptionobjectargs(self, exctype, args, msg):
+ # Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion
+ # check with one missing argument
+ self.assertRaises(TypeError, exctype, *args[:-1])
+ # check with one argument too much
+ self.assertRaises(TypeError, exctype, *(args + ["too much"]))
+ # check with one argument of the wrong type
+ wrongargs = [ "spam", u"eggs", 42, 1.0, None ]
+ for i in xrange(len(args)):
+ for wrongarg in wrongargs:
+ if type(wrongarg) is type(args[i]):
+ continue
+ # build argument array
+ callargs = []
+ for j in xrange(len(args)):
+ if i==j:
+ callargs.append(wrongarg)
+ else:
+ callargs.append(args[i])
+ errors = (UnicodeDecodeError, UnicodeEncodeError, TypeError)
+ self.assertRaises(errors, exctype, *callargs)
+
+ # check with the correct number and type of arguments
+ exc = exctype(*args)
+ self.assertEquals(str(exc), msg)
+
+ def test_unicodeencodeerror(self):
+ self.check_exceptionobjectargs(
+ UnicodeEncodeError,
+ ["ascii", u"g\xfcrk", 1, 2, "ouch"],
+ "'ascii' codec can't encode character u'\\xfc' in position 1: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeEncodeError,
+ ["ascii", u"g\xfcrk", 1, 4, "ouch"],
+ "'ascii' codec can't encode characters in position 1-3: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeEncodeError,
+ ["ascii", u"\xfcx", 0, 1, "ouch"],
+ "'ascii' codec can't encode character u'\\xfc' in position 0: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeEncodeError,
+ ["ascii", u"\u0100x", 0, 1, "ouch"],
+ "'ascii' codec can't encode character u'\\u0100' in position 0: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeEncodeError,
+ ["ascii", u"\uffffx", 0, 1, "ouch"],
+ "'ascii' codec can't encode character u'\\uffff' in position 0: ouch"
+ )
+ if sys.maxunicode > 0xffff:
+ self.check_exceptionobjectargs(
+ UnicodeEncodeError,
+ ["ascii", u"\U00010000x", 0, 1, "ouch"],
+ "'ascii' codec can't encode character u'\\U00010000' in position 0: ouch"
+ )
+
+ def test_unicodedecodeerror(self):
+ self.check_exceptionobjectargs(
+ UnicodeDecodeError,
+ ["ascii", "g\xfcrk", 1, 2, "ouch"],
+ "'ascii' codec can't decode byte 0xfc in position 1: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeDecodeError,
+ ["ascii", "g\xfcrk", 1, 3, "ouch"],
+ "'ascii' codec can't decode bytes in position 1-2: ouch"
+ )
+
+ def test_unicodetranslateerror(self):
+ self.check_exceptionobjectargs(
+ UnicodeTranslateError,
+ [u"g\xfcrk", 1, 2, "ouch"],
+ "can't translate character u'\\xfc' in position 1: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeTranslateError,
+ [u"g\u0100rk", 1, 2, "ouch"],
+ "can't translate character u'\\u0100' in position 1: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeTranslateError,
+ [u"g\uffffrk", 1, 2, "ouch"],
+ "can't translate character u'\\uffff' in position 1: ouch"
+ )
+ if sys.maxunicode > 0xffff:
+ self.check_exceptionobjectargs(
+ UnicodeTranslateError,
+ [u"g\U00010000rk", 1, 2, "ouch"],
+ "can't translate character u'\\U00010000' in position 1: ouch"
+ )
+ self.check_exceptionobjectargs(
+ UnicodeTranslateError,
+ [u"g\xfcrk", 1, 3, "ouch"],
+ "can't translate characters in position 1-2: ouch"
+ )
+
+ def test_badandgoodstrictexceptions(self):
+ # "strict" complains about a non-exception passed in
+ self.assertRaises(
+ TypeError,
+ codecs.strict_errors,
+ 42
+ )
+ # "strict" complains about the wrong exception type
+ self.assertRaises(
+ Exception,
+ codecs.strict_errors,
+ Exception("ouch")
+ )
+
+ # If the correct exception is passed in, "strict" raises it
+ self.assertRaises(
+ UnicodeEncodeError,
+ codecs.strict_errors,
+ UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")
+ )
+
+ def test_badandgoodignoreexceptions(self):
+ # "ignore" complains about a non-exception passed in
+ self.assertRaises(
+ TypeError,
+ codecs.ignore_errors,
+ 42
+ )
+ # "ignore" complains about the wrong exception type
+ self.assertRaises(
+ TypeError,
+ codecs.ignore_errors,
+ UnicodeError("ouch")
+ )
+ # If the correct exception is passed in, "ignore" returns an empty replacement
+ self.assertEquals(
+ codecs.ignore_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
+ (u"", 1)
+ )
+ self.assertEquals(
+ codecs.ignore_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
+ (u"", 1)
+ )
+ self.assertEquals(
+ codecs.ignore_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
+ (u"", 1)
+ )
+
+ def test_badandgoodreplaceexceptions(self):
+ # "replace" complains about a non-exception passed in
+ self.assertRaises(
+ TypeError,
+ codecs.replace_errors,
+ 42
+ )
+ # "replace" complains about the wrong exception type
+ self.assertRaises(
+ TypeError,
+ codecs.replace_errors,
+ UnicodeError("ouch")
+ )
+ self.assertRaises(
+ TypeError,
+ codecs.replace_errors,
+ BadObjectUnicodeEncodeError()
+ )
+ self.assertRaises(
+ TypeError,
+ codecs.replace_errors,
+ BadObjectUnicodeDecodeError()
+ )
+ # With the correct exception, "replace" returns an "?" or u"\ufffd" replacement
+ self.assertEquals(
+ codecs.replace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
+ (u"?", 1)
+ )
+ self.assertEquals(
+ codecs.replace_errors(UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")),
+ (u"\ufffd", 1)
+ )
+ self.assertEquals(
+ codecs.replace_errors(UnicodeTranslateError(u"\u3042", 0, 1, "ouch")),
+ (u"\ufffd", 1)
+ )
+
+ def test_badandgoodxmlcharrefreplaceexceptions(self):
+ # "xmlcharrefreplace" complains about a non-exception passed in
+ self.assertRaises(
+ TypeError,
+ codecs.xmlcharrefreplace_errors,
+ 42
+ )
+ # "xmlcharrefreplace" complains about the wrong exception types
+ self.assertRaises(
+ TypeError,
+ codecs.xmlcharrefreplace_errors,
+ UnicodeError("ouch")
+ )
+ # "xmlcharrefreplace" can only be used for encoding
+ self.assertRaises(
+ TypeError,
+ codecs.xmlcharrefreplace_errors,
+ UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
+ )
+ self.assertRaises(
+ TypeError,
+ codecs.xmlcharrefreplace_errors,
+ UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
+ )
+ # Use the correct exception
+ cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 0x3042)
+ s = "".join(unichr(c) for c in cs)
+ self.assertEquals(
+ codecs.xmlcharrefreplace_errors(
+ UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
+ ),
+ (u"".join(u"&#%d;" % ord(c) for c in s), len(s))
+ )
+
+ def test_badandgoodbackslashreplaceexceptions(self):
+ # "backslashreplace" complains about a non-exception passed in
+ self.assertRaises(
+ TypeError,
+ codecs.backslashreplace_errors,
+ 42
+ )
+ # "backslashreplace" complains about the wrong exception types
+ self.assertRaises(
+ TypeError,
+ codecs.backslashreplace_errors,
+ UnicodeError("ouch")
+ )
+ # "backslashreplace" can only be used for encoding
+ self.assertRaises(
+ TypeError,
+ codecs.backslashreplace_errors,
+ UnicodeDecodeError("ascii", "\xff", 0, 1, "ouch")
+ )
+ self.assertRaises(
+ TypeError,
+ codecs.backslashreplace_errors,
+ UnicodeTranslateError(u"\u3042", 0, 1, "ouch")
+ )
+ # Use the correct exception
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u3042", 0, 1, "ouch")),
+ (u"\\u3042", 1)
+ )
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\x00", 0, 1, "ouch")),
+ (u"\\x00", 1)
+ )
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\xff", 0, 1, "ouch")),
+ (u"\\xff", 1)
+ )
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\u0100", 0, 1, "ouch")),
+ (u"\\u0100", 1)
+ )
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\uffff", 0, 1, "ouch")),
+ (u"\\uffff", 1)
+ )
+ if sys.maxunicode>0xffff:
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U00010000", 0, 1, "ouch")),
+ (u"\\U00010000", 1)
+ )
+ self.assertEquals(
+ codecs.backslashreplace_errors(UnicodeEncodeError("ascii", u"\U0010ffff", 0, 1, "ouch")),
+ (u"\\U0010ffff", 1)
+ )
+
+ def test_badhandlerresults(self):
+ results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
+ encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15")
+
+ for res in results:
+ codecs.register_error("test.badhandler", lambda: res)
+ for enc in encs:
+ self.assertRaises(
+ TypeError,
+ u"\u3042".encode,
+ enc,
+ "test.badhandler"
+ )
+ for (enc, bytes) in (
+ ("ascii", "\xff"),
+ ("utf-8", "\xff"),
+ ("utf-7", "+x-"),
+ ("unicode-internal", "\x00"),
+ ):
+ self.assertRaises(
+ TypeError,
+ bytes.decode,
+ enc,
+ "test.badhandler"
+ )
+
+ def test_lookup(self):
+ self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict"))
+ self.assertEquals(codecs.ignore_errors, codecs.lookup_error("ignore"))
+ self.assertEquals(codecs.strict_errors, codecs.lookup_error("strict"))
+ self.assertEquals(
+ codecs.xmlcharrefreplace_errors,
+ codecs.lookup_error("xmlcharrefreplace")
+ )
+ self.assertEquals(
+ codecs.backslashreplace_errors,
+ codecs.lookup_error("backslashreplace")
+ )
+
+ def test_unencodablereplacement(self):
+ def unencrepl(exc):
+ if isinstance(exc, UnicodeEncodeError):
+ return (u"\u4242", exc.end)
+ else:
+ raise TypeError("don't know how to handle %r" % exc)
+ codecs.register_error("test.unencreplhandler", unencrepl)
+ for enc in ("ascii", "iso-8859-1", "iso-8859-15"):
+ self.assertRaises(
+ UnicodeEncodeError,
+ u"\u4242".encode,
+ enc,
+ "test.unencreplhandler"
+ )
+
+ def test_badregistercall(self):
+ # enhance coverage of:
+ # Modules/_codecsmodule.c::register_error()
+ # Python/codecs.c::PyCodec_RegisterError()
+ self.assertRaises(TypeError, codecs.register_error, 42)
+ self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42)
+
+ def test_badlookupcall(self):
+ # enhance coverage of:
+ # Modules/_codecsmodule.c::lookup_error()
+ self.assertRaises(TypeError, codecs.lookup_error)
+
+ def test_unknownhandler(self):
+ # enhance coverage of:
+ # Modules/_codecsmodule.c::lookup_error()
+ self.assertRaises(LookupError, codecs.lookup_error, "test.unknown")
+
+ def test_xmlcharrefvalues(self):
+ # enhance coverage of:
+ # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
+ # and inline implementations
+ v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000)
+ if sys.maxunicode>=100000:
+ v += (100000, 500000, 1000000)
+ s = u"".join([unichr(x) for x in v])
+ codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
+ for enc in ("ascii", "iso-8859-15"):
+ for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"):
+ s.encode(enc, err)
+
+ def test_decodehelper(self):
+ # enhance coverage of:
+ # Objects/unicodeobject.c::unicode_decode_call_errorhandler()
+ # and callers
+ self.assertRaises(LookupError, "\xff".decode, "ascii", "test.unknown")
+
+ def baddecodereturn1(exc):
+ return 42
+ codecs.register_error("test.baddecodereturn1", baddecodereturn1)
+ self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn1")
+ self.assertRaises(TypeError, "\\".decode, "unicode-escape", "test.baddecodereturn1")
+ self.assertRaises(TypeError, "\\x0".decode, "unicode-escape", "test.baddecodereturn1")
+ self.assertRaises(TypeError, "\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
+ self.assertRaises(TypeError, "\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
+ self.assertRaises(TypeError, "\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
+
+ def baddecodereturn2(exc):
+ return (u"?", None)
+ codecs.register_error("test.baddecodereturn2", baddecodereturn2)
+ self.assertRaises(TypeError, "\xff".decode, "ascii", "test.baddecodereturn2")
+
+ handler = PosReturn()
+ codecs.register_error("test.posreturn", handler.handle)
+
+ # Valid negative position
+ handler.pos = -1
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
+
+ # Valid negative position
+ handler.pos = -2
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?><?>")
+
+ # Negative position out of bounds
+ handler.pos = -3
+ self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
+
+ # Valid positive position
+ handler.pos = 1
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>0")
+
+ # Largest valid positive position (one beyond end of input)
+ handler.pos = 2
+ self.assertEquals("\xff0".decode("ascii", "test.posreturn"), u"<?>")
+
+ # Invalid positive position
+ handler.pos = 3
+ self.assertRaises(IndexError, "\xff0".decode, "ascii", "test.posreturn")
+
+ # Restart at the "0"
+ handler.pos = 6
+ self.assertEquals("\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), u"<?>0")
+
+ class D(dict):
+ def __getitem__(self, key):
+ raise ValueError
+ self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None})
+ self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D())
+ self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1})
+
+ def test_encodehelper(self):
+ # enhance coverage of:
+ # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
+ # and callers
+ self.assertRaises(LookupError, u"\xff".encode, "ascii", "test.unknown")
+
+ def badencodereturn1(exc):
+ return 42
+ codecs.register_error("test.badencodereturn1", badencodereturn1)
+ self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn1")
+
+ def badencodereturn2(exc):
+ return (u"?", None)
+ codecs.register_error("test.badencodereturn2", badencodereturn2)
+ self.assertRaises(TypeError, u"\xff".encode, "ascii", "test.badencodereturn2")
+
+ handler = PosReturn()
+ codecs.register_error("test.posreturn", handler.handle)
+
+ # Valid negative position
+ handler.pos = -1
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
+
+ # Valid negative position
+ handler.pos = -2
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?><?>")
+
+ # Negative position out of bounds
+ handler.pos = -3
+ self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
+
+ # Valid positive position
+ handler.pos = 1
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>0")
+
+ # Largest valid positive position (one beyond end of input
+ handler.pos = 2
+ self.assertEquals(u"\xff0".encode("ascii", "test.posreturn"), "<?>")
+
+ # Invalid positive position
+ handler.pos = 3
+ self.assertRaises(IndexError, u"\xff0".encode, "ascii", "test.posreturn")
+
+ handler.pos = 0
+
+ class D(dict):
+ def __getitem__(self, key):
+ raise ValueError
+ for err in ("strict", "replace", "xmlcharrefreplace", "backslashreplace", "test.posreturn"):
+ self.assertRaises(UnicodeError, codecs.charmap_encode, u"\xff", err, {0xff: None})
+ self.assertRaises(ValueError, codecs.charmap_encode, u"\xff", err, D())
+ self.assertRaises(TypeError, codecs.charmap_encode, u"\xff", err, {0xff: 300})
+
+ def test_translatehelper(self):
+ # enhance coverage of:
+ # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
+ # and callers
+ # (Unfortunately the errors argument is not directly accessible
+ # from Python, so we can't test that much)
+ class D(dict):
+ def __getitem__(self, key):
+ raise ValueError
+ self.assertRaises(ValueError, u"\xff".translate, D())
+ self.assertRaises(TypeError, u"\xff".translate, {0xff: sys.maxunicode+1})
+ self.assertRaises(TypeError, u"\xff".translate, {0xff: ()})
+
+ def test_bug828737(self):
+ charmap = {
+ ord("&"): u"&",
+ ord("<"): u"<",
+ ord(">"): u">",
+ ord('"'): u""",
+ }
+
+ for n in (1, 10, 100, 1000):
+ text = u'abc<def>ghi'*n
+ text.translate(charmap)
+
+def test_main():
+ test.test_support.run_unittest(CodecCallbackTest)
+
+if __name__ == "__main__":
+ test_main()
More information about the Pypy-commit
mailing list