[Jython-checkins] jython: Revise UTF-7 codec
jeff.allen
jython-checkins at python.org
Sat Jan 5 18:56:59 CET 2013
http://hg.python.org/jython/rev/c0072e7f0c90
changeset: 6938:c0072e7f0c90
user: Jeff Allen <ja...py at farowl.co.uk>
date: Sat Jan 05 17:29:17 2013 +0000
summary:
Revise UTF-7 codec
This fixes some minor test failures in test_codecs (now passing with 49 skips)
and obviates some skips in (v2.5-ish) test_unicode (now passing with 1 skip).
files:
Lib/test/test_unicode.py | 13 +-
src/org/python/core/codecs.java | 863 ++++++++++++---
src/org/python/modules/_codecs.java | 46 +-
3 files changed, 704 insertions(+), 218 deletions(-)
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -464,7 +464,6 @@
self.assertRaises(TypeError, unicode, 42, 42, 42)
- @unittest.skip("FIXME: broken")
def test_codecs_utf7(self):
utfTests = [
(u'A\u2262\u0391.', 'A+ImIDkQ.'), # RFC2152 example
@@ -484,14 +483,17 @@
for (x, y) in utfTests:
self.assertEqual(x.encode('utf-7'), y)
- # surrogates not supported
+ # Lone/misordered surrogates are an error
self.assertRaises(UnicodeError, unicode, '+3ADYAA-', 'utf-7')
- self.assertEqual(unicode('+3ADYAA-', 'utf-7', 'replace'), u'\ufffd')
+ # Jython (and some CPython versions): two misplaced surrogates => two replacements
+ self.assertEqual(unicode('+3ADYAA-', 'utf-7', 'replace'), u'\ufffd\ufffd')
+ # self.assertEqual(unicode('+3ADYAA-', 'utf-7', 'replace'), u'\ufffd')
def test_codecs_utf8(self):
self.assertEqual(u''.encode('utf-8'), '')
self.assertEqual(u'\u20ac'.encode('utf-8'), '\xe2\x82\xac')
+ # Jython will not compile Unicode literals with surrogate units
#self.assertEqual(u'\ud800\udc02'.encode('utf-8'), '\xf0\x90\x80\x82')
#self.assertEqual(u'\ud84d\udc56'.encode('utf-8'), '\xf0\xa3\x91\x96')
#self.assertEqual(u'\ud800'.encode('utf-8'), '\xed\xa0\x80')
@@ -528,6 +530,7 @@
# * strict decoding testing for all of the
# UTF8_ERROR cases in PyUnicode_DecodeUTF8
+ @unittest.skipIf(test_support.is_jython, "IDNA codec missing in Jython (issue 1153)")
def test_codecs_idna(self):
# Test whether trailing dot is preserved
self.assertEqual(u"www.python.org.".encode("idna"), "www.python.org.")
@@ -587,7 +590,6 @@
# Error handling (PyUnicode_EncodeDecimal())
self.assertRaises(UnicodeError, int, u"\u0200")
- @unittest.skip("FIXME: broken")
def test_codecs(self):
# Encoding
self.assertEqual(u'hello'.encode('ascii'), 'hello')
@@ -714,9 +716,6 @@
self.assertEqual(x, y)
def test_main():
- if test_support.is_jython:
- # http://bugs.jython.org/issue1153
- del UnicodeTest.test_codecs_idna
test_support.run_unittest(UnicodeTest)
if __name__ == "__main__":
diff --git a/src/org/python/core/codecs.java b/src/org/python/core/codecs.java
--- a/src/org/python/core/codecs.java
+++ b/src/org/python/core/codecs.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c)2012 Jython Developers Original Java version copyright 2000 Finn Bock
+ * Copyright (c)2013 Jython Developers. Original Java version copyright 2000 Finn Bock.
*
* This program contains material copyrighted by: Copyright (c) Corporation for National Research
* Initiatives. Originally written by Marc-Andre Lemburg (mal at lemburg.com).
@@ -410,233 +410,637 @@
/* --- UTF-7 Codec -------------------------------------------------------- */
- /* see RFC2152 for details */
- public static char utf7_special[] = {//@formatter:off
- /*
- * indicate whether a UTF-7 character is special i.e. cannot be directly
- * encoded: 0 - not special 1 - special 2 - whitespace (optional) 3 -
- * RFC2152 Set O (optional)
- */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
- 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
- 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
- 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1
- }; //@formatter:on
+ /*
+ * This codec was converted to Java from the CPython v2.7.3 final. See RFC2152 for details of
+ * the encoding scheme. We encode conservatively and decode liberally.
+ */
- private static boolean SPECIAL(char c, boolean encodeO, boolean encodeWS) {
- return (c > 127 || utf7_special[(c)] == 1) || (encodeWS && (utf7_special[(c)] == 2))
- || (encodeO && (utf7_special[(c)] == 3));
+ /* //@formatter:off
+ * The UTF-7 encoder treats ASCII characters differently according to whether they are Set D,
+ * Set O, Whitespace, or special (i.e. none of the above). See RFC2152. This array identifies
+ * these different sets:
+ * 0 : "Set D"
+ * alphanumeric and '(),-./:?
+ * 1 : "Set O"
+ * !"#$%&*;<=>@[]^_`{|}
+ * 2 : "whitespace"
+ * ht nl cr sp
+ * 3 : special (must be base64 encoded)
+ * everything else (i.e. +\~ and non-printing codes 0-8 11-12 14-31 127)
+ */
+ private static final byte[] utf7_category = {
+ /* nul soh stx etx eot enq ack bel bs ht nl vt np cr so si */
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3,
+ /* dle dc1 dc2 dc3 dc4 nak syn etb can em sub esc fs gs rs us */
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ /* sp ! " # $ % & ' ( ) * + , - . / */
+ 2, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 3, 0, 0, 0, 0,
+ /* 0 1 2 3 4 5 6 7 8 9 : ; < = > ? */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,
+ /* @ A B C D E F G H I J K L M N O */
+ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ /* P Q R S T U V W X Y Z [ \ ] ^ _ */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 1, 1, 1,
+ /* ` a b c d e f g h i j k l m n o */
+ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ /* p q r s t u v w x y z { | } ~ del */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 3, 3,
+ };//@formatter:on
+
+ /**
+ * Determine whether this character should be encoded as itself. The answer depends on whether
+ * we are encoding set O (optional special characters) as itself, and also on whether we are
+ * encoding whitespace as itself. RFC2152 makes it clear that the answers to these questions
+ * vary between applications, so this code needs to be flexible.
+ *
+ * @param c code point of the character
+ * @param directO true if characters in "set O" may be encoded as themselves
+ * @param directWS true if whitespace characters may be encoded as themselves
+ * @return
+ */
+ private static boolean ENCODE_DIRECT(int c, boolean directO, boolean directWS) {
+
+ if (c >= 128 || c < 0) {
+ return false; // Character not in table is always special
+ } else {
+ switch (utf7_category[c]) {
+ case 0: // This is a regular character
+ return true;
+ case 1: // This is a whilespace character
+ return directWS;
+ case 2: // This is an optional special character
+ return directO;
+ default: // This is always a special character (including '+')
+ return false;
+ }
+ }
}
+ /** Look-up for the Base64 encoded byte [0..0x3f] */
private static final String B64_CHARS =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
- private static char B64(int n) {
+ /** What is the Base64 encoded byte for (the bottom 6 bits of) n? */
+ private static char TO_BASE64(int n) {
return B64_CHARS.charAt(n & 0x3f);
}
- private static boolean B64CHAR(char c) {
- return B64_CHARS.indexOf(c) != -1;
+ /**
+ * Is c the code point of a Base64 character? And if so, what is the 6-bit quantity to be
+ * decodec from c? Return the 6-bit equivalent of c in a Base64 segment, -1 if it cannot be used
+ * in a Base64 segment, and -2 for the special case of '-' ending the segment.
+ */
+ private static int FROM_BASE64(int c) {
+ return (c >= 128) ? -1 : BASE64_VALUE[c];
}
- private static int UB64(char c) {
- return (c == '+' ? 62 : c == '/' ? 63 : c >= 'a' ? c - 71 : c >= 'A' ? c - 65 : c + 4);
+ /**
+ * Look-up table to convert ASCII byte to 6-bit Base64 value, -1 if not Base64, and -2 if
+ * special terminator '-'.
+ */
+ private static final byte[] BASE64_VALUE = {//@formatter:off
+ // nul soh stx etx eot enq ack bel bs ht nl vt np cr so si
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ // dle dc1 dc2 dc3 dc4 nak syn etb can em sub esc fs gs rs us
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ // sp ! " # $ % & ' ( ) * + , - . /
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -2, -1, 63,
+ // 0 1 2 3 4 5 6 7 8 9 : ; < = > ?
+ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1,
+ // @ A B C D E F G H I J K L M N O
+ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+ // P Q R S T U V W X Y Z [ \ ] ^ _
+ 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1,
+ // ` a b c d e f g h i j k l m n o
+ -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+ // p q r s t u v w x y z { | } ~ del
+ 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1,
+ };//@formatter:on
+
+ /**
+ * Enumeration of the error causes during decoding of the Base64 segment of UTF-7
+ */
+ static enum UTF7Error {
+ NONE("No error"), // No error
+ PADDING("non-zero padding bits in shift sequence"), // Error when at end
+ PARTIAL("partial character in shift sequence"), // Error when at end
+ TRUNCATED("second surrogate missing at end of shift sequence"), // Error when at end
+ MISSING("second surrogate missing"), // Lead surrogate followed by another, or BMP
+ TRAIL("unexpected second surrogate"); // Trail surrogate not preceded by lead
+
+ /** Suitable error message */
+ final String msg;
+
+ private UTF7Error(String msg) {
+ this.msg = msg;
+ }
}
- /*
- * note that we follow CPython 2.5 exactly here - it does not support surrogates, but has to
- * process as-if they are there for replacement purposes fortunately no one really cares about
- * utf-7.
+ /**
+ * Decode (perhaps partially) a sequence of bytes representing the UTF-7 encoded form of a
+ * Unicode string and return the (Jython internal representation of) the unicode object, and
+ * amount of input consumed. The only state we preserve is our read position, i.e. how many
+ * characters we have consumed. So if the input ends part way through a Base64 sequence the data
+ * reported as consumed is only that up to and not including the Base64 start marker ('+').
+ * Performance will be poor (quadratic cost) on runs of Base64 data long enough to exceed the
+ * input quantum in incremental decoding. The retruned Java String is a UTF-16 representation of
+ * the Unicode result, in line with Java conventions. Unicode characters above the BMP are
+ * represented as surrogate pairs.
+ *
+ * @param bytes input represented as String (Jython PyString convention)
+ * @param errors error policy name (e.g. "ignore", "replace")
+ * @param consumed returns number of bytes consumed in element 0, or is null if a "final" call
+ * @return unicode result (as UTF-16 Java String)
*/
- public static String PyUnicode_DecodeUTF7(String str, String errors) {
- int s = 0;
- int e = str.length();
- boolean inShift = false;
- int bitsInCharsleft = 0;
- long charsleft = 0;
- boolean surrogate = false;
- StringBuilder unicode = new StringBuilder(e);
- while (s < e) {
- // restart:
- char ch = str.charAt(s);
- if (inShift) {
- if ((ch == '-') || !B64CHAR(ch)) {
- inShift = false;
- s++;
- while (bitsInCharsleft >= 16) {
- bitsInCharsleft -= 16;
- char outCh = (char)((charsleft >> bitsInCharsleft) & 0xffff);
- if (surrogate) {
- s = codecs.insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s + 1, "code pairs are not supported");
- surrogate = false;
- } else if (0xDC00 <= outCh && outCh <= 0xDFFF) {
- surrogate = true;
- } else {
- unicode.append(outCh);
+ public static String PyUnicode_DecodeUTF7Stateful(String bytes, String errors, int[] consumed) {
+ int s; // Index in the input bytes
+ boolean inBase64 = false; // Whether s is currently in a Base64 segment
+ long base64buffer = 0; // Stored bits buffer during Base64 decoding
+ int base64bits = 0; // Number of valid bits buffered during Base64 decoding
+ int startInBytes = 0; // Place in input bytes where most recent Base64 segment begins
+ int syncInBytes = 0; // Place in input bytes where stored bits buffer last empty
+ int startInUnicode = 0; // Place in output unicode where most recent Base64 segment begins
+
+ int size = bytes.length();
+ StringBuilder unicode = new StringBuilder(size);
+
+ for (s = 0; s < size; s++) { // In error cases s may skip forwards in bytes
+
+ // Next byte to process
+ int b = bytes.charAt(s);
+
+ if (b >= 128) {
+ // The input was supposed to be 7-bit clean
+ s = insertReplacementAndGetResume(unicode, errors, "utf-7", //
+ bytes, s, s + 1, "unexpected special character") - 1;
+
+ } else if (inBase64) {
+ // We are currently processing a Base64 section
+
+ if (base64bits == 0) {
+ // Mark this point as latest easy error recovery point (bits buffer empty)
+ syncInBytes = s;
+ }
+
+ int sixBits = FROM_BASE64(b); // returns -ve if not Base64
+ if (sixBits >= 0) {
+ // And we continue processing a Base64 section
+ base64buffer = (base64buffer << 6) | sixBits;
+ base64bits += 6;
+
+ if (base64bits >= 32) {
+ // We have enough bits for a code point
+ base64bits = emitCodePoints(unicode, base64buffer, base64bits);
+
+ if (base64bits >= 32) {
+ // We stopped prematurely. Why?
+ UTF7Error error = emitCodePointsDiagnosis(base64buffer, base64bits);
+ // Difficult to know exactly what input characters to blame
+ s = insertReplacementAndGetResume(unicode, errors, "utf-7", //
+ bytes, syncInBytes, s + 1, error.msg) - 1;
+ // Discard one UTF-16 output and hope for the best
+ base64bits -= 16;
}
+
}
- if (bitsInCharsleft >= 6) {
+
+ } else {
+ // We are now leaving a Base64 section
+ inBase64 = false;
+
+ // We should have a whole number of code points and < 6 bits zero padding
+ if (base64bits > 0) {
+ // Try to emit them all
+ base64bits = emitCodePoints(unicode, base64buffer, base64bits);
+ // Now check for errors
+ UTF7Error error = emitCodePointsDiagnosis(base64buffer, base64bits);
+ if (error != UTF7Error.NONE) {
+ // Difficult to know exactly what input characters to blame
+ s = insertReplacementAndGetResume(unicode, errors, "utf-7", //
+ bytes, s, s + 1, error.msg) - 1;
+ }
+ // We are, in any case, discarding whatever is in the buffer
+ base64bits = 0;
+ }
+
+ if (b == '-') {
/*
- * The shift sequence has a partial character in it. If bitsleft < 6 then we
- * could just classify it as padding but that is not the case here
+ * '-' signals the end of Base64. The byte is is simply absorbed, but in the
+ * special case where it is the first byte of the Base64 segment, the
+ * zero-length segment '+-' actually encodes "+".
*/
- s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s + 1, "partial character in shift sequence");
- }
- /*
- * According to RFC2152 the remaining bits should be zero. We choose to signal
- * an error/insert a replacement character here so indicate the potential of a
- * misencoded character.
- */
- if (bitsInCharsleft > 0 && ((charsleft << 5 - bitsInCharsleft) & 0x1f) > 0) {
- s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s + 1, "non-zero padding bits in shift sequence");
- }
- if (ch == '-') {
- if ((s < e) && (str.charAt(s) == '-')) {
- unicode.append('-');
- inShift = true;
+ if (s == startInBytes + 1) {
+ unicode.append('+');
}
- } else if (SPECIAL(ch, false, false)) {
- s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s + 1, "unexpected special character");
} else {
- unicode.append(ch);
- }
- } else {
- charsleft = (charsleft << 6) | UB64(ch);
- bitsInCharsleft += 6;
- s++;
- while (bitsInCharsleft >= 16) {
- bitsInCharsleft -= 16;
- char outCh = (char)((charsleft >> bitsInCharsleft) & 0xffff);
- if (surrogate) {
- s = codecs.insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s + 1, "code pairs are not supported");
- } else if (0xDC00 <= outCh && outCh <= 0xDFFF) {
- surrogate = true;
- } else {
- unicode.append(outCh);
- }
+ /*
+ * This b is a US-ASCII byte for some character.
+ */
+ unicode.appendCodePoint(b);
}
}
- } else if (ch == '+') {
- s++;
- if (s < e && str.charAt(s) == '-') {
- s++;
- unicode.append('+');
- } else {
- inShift = true;
- bitsInCharsleft = 0;
- }
- } else if (SPECIAL(ch, false, false)) {
- s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s + 1, "unexpected special character");
+
+ } else if (b == '+') {
+ /*
+ * We are not currently processing a Base64 section, but this starts one. Remember
+ * where it starts, in the input bytes and the output unicode so that, if we hit the
+ * end of input before it ends, we can leave it unprocessed for next time.
+ */
+ startInBytes = s;
+ startInUnicode = unicode.length();
+
+ // Initialise the Base64 decoder
+ base64bits = 0;
+ inBase64 = true;
+
} else {
- unicode.append(ch);
- s++;
- }
- if (inShift && s == e) {
- s = insertReplacementAndGetResume(unicode, errors, "utf-7", str, //
- s, s, "unterminated shift sequence");
+ /*
+ * This b is a US-ASCII byte for some character. We are permissive on decoding; the
+ * only ASCII byte not decoding to itself is the + which begins a base64 string.
+ */
+ unicode.appendCodePoint(b);
}
}
+
+ /*
+ * We hit the end of the input. If we were part way through some Base64 processing, since we
+ * don't store all that state (inBase64, base64bits, base64buffer) the strategy is to back
+ * up the input pointer to the '-' that started the current Base64 segment.
+ */
+ if (inBase64) {
+ // Restore state to beginning of last Base64 sequence
+ s = startInBytes;
+ unicode.setLength(startInUnicode);
+ }
+
+ if (consumed != null) {
+ // Not a final call, so report how much consumed in the consumed argument
+ consumed[0] = s;
+ } else if (s < size) {
+ // This was final but we didn't exhaust the input: that's an error.
+ s = insertReplacementAndGetResume(unicode, errors, "utf-7", //
+ bytes, startInBytes, size, "unterminated shift sequence");
+ }
+
return unicode.toString();
}
- public static String PyUnicode_EncodeUTF7(String str, boolean encodeSetO,
- boolean encodeWhiteSpace, String errors) {
- int size = str.length();
+ /**
+ * Decode completely a sequence of bytes representing the UTF-7 encoded form of a Unicode string
+ * and return the (Jython internal representation of) the unicode object. The retruned Java
+ * String is a UTF-16 representation of the Unicode result, in line with Java conventions.
+ * Unicode characters above the BMP are represented as surrogate pairs.
+ *
+ * @param bytes input represented as String (Jython PyString convention)
+ * @param errors error policy name (e.g. "ignore", "replace")
+ * @return unicode result (as UTF-16 Java String)
+ */
+ public static String PyUnicode_DecodeUTF7(String bytes, String errors) {
+ return PyUnicode_DecodeUTF7Stateful(bytes, errors, null);
+ }
- if (size == 0) {
- return "";
+ /**
+ * Helper for {@link #PyUnicode_DecodeUTF7Stateful(String, String, int[])} to emit characters
+ * that accumulated as UTF-16 code units in the bits of a long integer (from Base64 decoding,
+ * say). The buffer variable may hold any number of bits (up to its 64-bit capacity). The number
+ * of valid bits is given by argument <code>n</code> and they are the <code>n</code> least
+ * significant of the buffer.
+ * <p>
+ * Only complete Unicode characters are emitted, which are obtained by consuming 16 bits (when
+ * those bits identify a BMP character), or 32 bits (when those bits form a surrogate pair).
+ * Consumed bits are not cleared from the buffer (it is passed by value), and there is no need
+ * for the client to clear them, but the method returns the new number of valid bits n1, which
+ * are in the least significant positions (that is, bits <code>n1-1</code> to <code>0</code>).
+ *
+ * If the method returns with 32 or more bits unconsumed, it has encountered an invalid sequence
+ * of bits: the leading bits will then either be an "unaccompanied" trail surrogate, or a lead
+ * surrogate not followed by a trail surrogate.
+ *
+ * @param v output UTF-16 sequence
+ * @param buffer holding the bits
+ * @param n the number of bits held (<=64)
+ * @return the number of bits not emitted (<32 unless error)
+ */
+ private static int emitCodePoints(StringBuilder v, long buffer, int n) {
+
+ // Emit code points until too few in the buffer to process.
+ while (n >= 16) {
+
+ /*
+ * Get the top 16 bits of the buffer to bottom of an int. Note no 0xffff mask as bits to
+ * left of bit-15 are harmless
+ */
+ int unit = (int)(buffer >>> (n - 16));
+ boolean unitIsSurrogate = ((unit & 0xF800) == 0xD800);
+
+ if (!unitIsSurrogate) {
+ // This (or rather its bottom 16 bits) is a BMP codepoint: easy
+ v.append((char)unit);
+ n -= 16;
+
+ } else if (n >= 32) {
+ // This a surrogate unit and we have enough bits for the whole code point.
+ if ((unit & 0x0400) == 0) {
+ // This is a lead surrogate as expected ... get the trail surrogate.
+ int unit2 = (int)(buffer >>> (n - 32));
+ if ((unit2 & 0xFC00) == 0xD800) {
+ // And this is the trail surrogate we expected
+ v.appendCodePoint(0x10000 + ((unit & 0x3ff) << 10) + (unit2 & 0x3ff));
+ n -= 32;
+ } else {
+ // But this isn't a trail surrogate: jam at >=32
+ return n;
+ }
+ } else {
+ // This is an unaccompanied trail surrogate: jam at >=32
+ return n;
+ }
+
+ } else {
+ // This a non-BMP code point but we don't have enough bits to deal with it yet
+ return n;
+ }
+
}
- boolean inShift = false;
- int bitsleft = 0;
- int charsleft = 0;
- StringBuilder v = new StringBuilder();
+ return n;
+ }
- for (int i = 0; i < size; ++i) {
- char ch = str.charAt(i);
+ /**
+ * Helper for {@link #PyUnicode_DecodeUTF7Stateful(String, String, int[])} to diagnose what went
+ * wrong in {@link #emitCodePoints(StringBuilder, long, int)}. When called with fewer than 32
+ * bits in the buffer, it assumes we are in the run-down of processing at the end of the
+ * decoder, where partial output characters are an error. For 32 bits or more, It duplicates
+ * some logic, but is called only during abnormal processing. The return is:
+ * <table>
+ * <tr>
+ * <td>NONE</td>
+ * <td>No error</td>
+ * </tr>
+ * <tr>
+ * <td>PADDING</td>
+ * <td>non-zero padding bits in shift sequence</td>
+ * <td>(error if at end of shift sequence)</td>
+ * </tr>
+ * <tr>
+ * <td>PARTIAL</td>
+ * <td>partial character in shift sequence</td>
+ * <td>(error if at end of shift sequence)</td>
+ * </tr>
+ * <tr>
+ * <td>TRUNCATED</td>
+ * <td>second surrogate missing at end of shift sequence</td>
+ * </tr>
+ * <tr>
+ * <td>MISSING</td>
+ * <td>second surrogate missing</td>
+ * </tr>
+ * <tr>
+ * <td>TRAIL</td>
+ * <td>unexpected second surrogate</td>
+ * </tr>
+ * </table>
+ * <p>
+ * We are compatible with CPython in using the term "second surrogate" in error messages rather
+ * than "trail surrogate" (which is used in the code).
+ * <p>
+ * Note that CPython (see Issue13333) allows this codec to decode lone surrogates into the
+ * internal data of unicode objects. It is difficult to reconcile this with the idea that the
+ * v3.3 statement "Strings contain Unicode characters", but that reconciliation is probably to
+ * be found in PEP383, not implemented in Jython.
+ *
+ * @param buffer holding the bits
+ * @param n the number of bits held (<=64)
+ * @return the diagnosis
+ */
+ private static UTF7Error emitCodePointsDiagnosis(long buffer, int n) {
- if (!inShift) {
- if (ch == '+') {
- v.append('+');
- v.append('-');
- } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
- charsleft = ch;
- bitsleft = 16;
- v.append('+');
- while (bitsleft >= 6) {
- v.append(B64(charsleft >> (bitsleft - 6)));
- bitsleft -= 6;
+ if (n >= 16) {
+ /*
+ * Get the top 16 bits of the buffer to bottom of an int. Note no 0xffff mask as bits to
+ * left of bit-15 are harmless
+ */
+ int unit = (int)(buffer >>> (n - 16));
+ boolean unitIsSurrogate = ((unit & 0xF800) == 0xD800);
+
+ if (!unitIsSurrogate) {
+ // No problem. In practice, we should never land here.
+ return UTF7Error.NONE;
+
+ } else if (n >= 32) {
+
+ if ((unit & 0x0400) == 0) {
+ // This is a lead surrogate, which is valid: check the next 16 bits.
+ int unit2 = ((int)(buffer >>> (n - 32))) & 0xffff;
+ if ((unit2 & 0xFC00) == 0xD800) {
+ // Not trail surrogate: that's the problem
+ return UTF7Error.MISSING;
+ } else {
+ // Hmm ... why was I called?
+ return UTF7Error.NONE;
}
- inShift = bitsleft > 0;
+
} else {
- v.append(ch);
+ // This is an unexpected trail surrogate
+ return UTF7Error.TRAIL;
}
+
} else {
- if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
- v.append(B64(charsleft << (6 - bitsleft)));
- charsleft = 0;
- bitsleft = 0;
+ // Note that 32 > n >= 16, so we are at the end of decoding
+
+ if ((unit & 0x0400) == 0) {
/*
- * Characters not in the BASE64 set implicitly unshift the sequence so no '-' is
- * required, except if the character is itself a '-'
+ * This is a lead surrogate, but since decoding stopped we must have reched the
+ * end of a Base64 segment without the trail surrogate appearing.
*/
- if (B64CHAR(ch) || ch == '-') {
+ return UTF7Error.TRUNCATED;
+
+ } else {
+ // This is an unexpected trail surrogate
+ return UTF7Error.TRAIL;
+ }
+ }
+
+ } else if (n >= 6) {
+ // Fewer than 16 bits: at end of decoding with Base64 characters left over
+ return UTF7Error.PARTIAL;
+
+ } else {
+ // Fewer than 6 bits, which should all be zero. Make a mask to extract them.
+ int validBits = (1 << n) - 1;
+ int padding = ((int)buffer) & validBits;
+ if (padding != 0) {
+ // At end of decoding with non-zero padding
+ return UTF7Error.PADDING;
+ } else {
+ // Any bits left are zero: that's ok then.
+ return UTF7Error.NONE;
+ }
+ }
+ }
+
+ /**
+ * Encode a UTF-16 Java String as UTF-7 bytes represented by the low bytes of the characters in
+ * a String. (String representation for byte data is chosen so that it may immediately become a
+ * PyString.)
+ *
+ * This method differs from the CPython equivalent (in <code>Object/unicodeobject.c</code>)
+ * which works with an array of point codes that are, in a wide build, Unicode code points.
+ *
+ * @param unicode
+ * @param base64SetO
+ * @param base64WhiteSpace
+ * @param errors
+ * @return
+ */
+ public static String PyUnicode_EncodeUTF7(String unicode, boolean base64SetO,
+ boolean base64WhiteSpace, String errors) {
+ boolean inBase64 = false;
+ int base64bits = 0;
+ long base64buffer = 0;
+
+ int size = unicode.length();
+
+ // Output bytes here: sized for ASCII + a few non-BMP characters
+ // We use a StringBuilder and return a String, but we are really storing encoded bytes
+ StringBuilder v = new StringBuilder(size + size / 8 + 10);
+
+ for (int i = 0; i < size; i++) {
+
+ // Next UTF-16 code unit to process
+ int ch = unicode.charAt(i);
+
+ /*
+ * Decide what to output and prepare for it. Mainly, decide whether to represent this
+ * UTF-16 code unit in Base64 or US-ASCII, and switch modes, with output, accordingly.
+ */
+ if (inBase64) {
+ // Currently we are in Base64 encoding: should we switch out?
+ if (ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) {
+ /*
+ * The next character is one for which we do not neeed to be in Base64, so pad
+ * out to 6n the Base64 bits we currently have buffered and emit them. Then
+ * switch to US-ASCII.
+ */
+ emitBase64Padded(v, base64buffer, base64bits);
+ inBase64 = false;
+
+ if (FROM_BASE64(ch) != -1) {
+ // Character is in the Base64 set, or is a '-': must signal end explicitly.
v.append('-');
}
- inShift = false;
- v.append(ch);
- } else {
- bitsleft += 16;
- charsleft = (charsleft << 16) | ch;
- while (bitsleft >= 6) {
- v.append(B64(charsleft >> (bitsleft - 6)));
- bitsleft -= 6;
- }
+ }
+
+ } else {
+ // Not currently in Base64 encoding: should we switch in?
+ if (ch == '+') {
+ // Special case for + since it would otherwise flag a start
+ v.append('+');
+ ch = '-'; // Comes out as +-
+ } else if (!ENCODE_DIRECT(ch, !base64SetO, !base64WhiteSpace)) {
/*
- * If the next character is special then we dont' need to terminate the shift
- * sequence. If the next character is not a BASE64 character or '-' then the
- * shift sequence will be terminated implicitly and we don't have to insert a
- * '-'.
+ * The next character is one for which we neeed to be in Base64, so switch to it
+ * and emit the Base64 start marker and initialise the coder.
*/
-
- if (bitsleft == 0) {
- if (i + 1 < size) {
- char ch2 = str.charAt(i + 1);
-
- if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {
-
- } else if (B64CHAR(ch2) || ch2 == '-') {
- v.append('-');
- inShift = false;
- } else {
- inShift = false;
- }
-
- } else {
- v.append('-');
- inShift = false;
- }
- }
+ v.append('+');
+ inBase64 = true;
+ base64bits = 0;
}
}
+
+ /*
+ * We have decided what to do (US-ASCII or Base64) but we haven't done it yet.
+ */
+ if (!inBase64) {
+ // We decided to encode the current character as US-ASCII and are in that mode
+ v.append((char)ch);
+
+ } else {
+ // We decided to encode the current character as Base64 and are in that mode
+ /*
+ * In the present implementation the characters are suppplied as a UTF-16 Java
+ * String. The UTF-7 approach to characters beyond the BMP is to encode the
+ * surrogate pair as two 16-bit pseudo-characters, which is how Jython represents it
+ * already, so the first part is already done for us by accessing the internal
+ * representation.
+ */
+ // XXX see issue #2002: we should only count surrogate pairs as one character
+ // if ((ch & 0xFC00)==0xC800) { count++; }
+
+ if (base64bits > 48) {
+ // No room for the next 16 bits: emit all we have
+ base64bits = emitBase64(v, base64buffer, base64bits);
+ }
+ base64bits += 16;
+ base64buffer = (base64buffer << 16) + ch;
+ }
}
- if (bitsleft > 0) {
- v.append(B64(charsleft << (6 - bitsleft)));
+
+ /*
+ * We've run out of input to encode. If we are currently in US-ASCII mode, we can just stop.
+ * If we are in Base64 mode, we have to come to a clean stop, since there is no opportunity
+ * to store this fact as state for next time (and there may be no next time).
+ */
+ if (inBase64) {
+ /*
+ * Currently we are in Base64 encoding and must switch out. Pad out to 6n the bits we
+ * currently have buffered and emit them. We don't know what might come next so emit a
+ * '-' to round out the segment.
+ */
+ emitBase64Padded(v, base64buffer, base64bits);
v.append('-');
}
+
return v.toString();
}
+ /**
+ * Helper for {@link #PyUnicode_EncodeUTF7(String, boolean, boolean, String)} to emit 6-bit
+ * Base64 code units as bytes to the output. The buffer variable may hold any number of bits (up
+ * to its 64-bit capacity). The number of valid bits is given by argument <code>n</code> and
+ * they are the <code>n</code> least significant of the buffer. Bits will be emitted in groups
+ * of 6, represented by their Base64 character, starting with the 6 most-significant valid bits
+ * of the buffer (that is, bits <code>n-6</code> to <code>n-1</code>). The buffer is not cleared
+ * (it is passed by value), but the method returns the new number of valid bits n1, which are in
+ * the least significant positions (that is, bits <code>n1-1</code> to <code>0</code>).
+ *
+ * @param v output byte array
+ * @param buffer holding the bits
+ * @param n the number of bits held (<=64)
+ * @return the number of bits (<6) not emitted
+ */
+ private static int emitBase64(StringBuilder v, long buffer, int n) {
+ while (n >= 6) {
+ n -= 6;
+ long sixBits = buffer >>> n;
+ char b64byte = TO_BASE64((int)sixBits);
+ v.append(b64byte);
+ }
+ return n;
+ }
+
+ /**
+ * Helper for {@link #PyUnicode_EncodeUTF7(String, boolean, boolean, String)} to emit 6-bit
+ * Base64 code units as bytes to the output. The buffer variable may hold any number of bits (up
+ * to 60 bits). The number of valid bits is given by argument <code>n</code> and they are the
+ * <code>n</code> least significant of the buffer. The buffer will be padded, by shifting in
+ * zeros at the least significant end, until it the number of valid bits is a multiple of 6.
+ * Bits will then be emitted in groups of 6, represented by their Base64 character, starting
+ * with the 6 most-significant valid bits of the buffer (that is, bits <code>n-6</code> to
+ * <code>n-1</code>). The buffer is not cleared (it is passed by value), but can be considered
+ * empty.
+ *
+ * @param v output byte array
+ * @param buffer holding the bits
+ * @param n the number of bits held (<=60)
+ */
+ private static void emitBase64Padded(StringBuilder v, long buffer, int n) {
+ if (n > 0) {
+ int npad = 5 - (n + 5) % 6; // smallest such that (n+npad) mod 6 == 0
+ emitBase64(v, buffer << npad, n + npad); // == 0 as a result of the padding
+ }
+ }
+
/* --- UTF-8 Codec ---------------------------------------------------- */
private static byte utf8_code_length[] = {//@formatter:off
@@ -867,17 +1271,6 @@
return v.toString();
}
- public static int calcNewPosition(int size, PyObject errorTuple) {
- int newPosition = ((PyInteger)errorTuple.__getitem__(1)).getValue();
- if (newPosition < 0) {
- newPosition = size + newPosition;
- }
- if (newPosition > size || newPosition < 0) {
- throw Py.IndexError(newPosition + " out of bounds of encoded string");
- }
- return newPosition;
- }
-
/* --- RawUnicodeEscape Codec ---------------------------------------- */
private static char[] hexdigit = "0123456789ABCDEF".toCharArray();
@@ -1176,41 +1569,107 @@
return replacement;
}
+ /**
+ * Handler errors encountered during decoding, adjusting the output buffer contents and
+ * returning the correct position to resume decoding (if the handler does not siomply raise an
+ * exception).
+ *
+ * @param partialDecode output buffer of unicode (as UTF-16) that the codec is building
+ * @param errors name of the error policy (or null meaning "strict")
+ * @param encoding name of encoding that encountered the error
+ * @param toDecode bytes being decoded
+ * @param start index of first byte it couldn't decode
+ * @param end index+1 of last byte it couldn't decode (usually becomes the resume point)
+ * @param reason contribution to error message if any
+ * @return the resume position: index of next byte to decode
+ */
public static int insertReplacementAndGetResume(StringBuilder partialDecode, String errors,
String encoding, String toDecode, int start, int end, String reason) {
+
+ // Handle the two special cases "ignore" and "replace" locally
if (errors != null) {
if (errors.equals(IGNORE)) {
+ // Just skip to the first non-problem byte
return end;
} else if (errors.equals(REPLACE)) {
- while (start < end) {
- partialDecode.appendCodePoint(Py_UNICODE_REPLACEMENT_CHARACTER);
- start++;
- }
+ // Insert *one* Unicode replacement character and skip
+ partialDecode.appendCodePoint(Py_UNICODE_REPLACEMENT_CHARACTER);
return end;
}
}
- PyObject replacement = decoding_error(errors, encoding, toDecode, start, end, reason);
- checkErrorHandlerReturn(errors, replacement);
- partialDecode.append(replacement.__getitem__(0).toString());
- return calcNewPosition(toDecode.length(), replacement);
+
+ // If errors not one of those, invoke the generic mechanism
+ PyObject replacementSpec = decoding_error(errors, encoding, toDecode, start, end, reason);
+ checkErrorHandlerReturn(errors, replacementSpec);
+
+ // Deliver the replacement unicode text to the output buffer
+ partialDecode.append(replacementSpec.__getitem__(0).toString());
+
+ // Return the index in toDecode at which we should resume
+ return calcNewPosition(toDecode.length(), replacementSpec);
}
- public static PyObject decoding_error(String errors, String encoding, String toEncode,
+ /**
+ * Invoke a user-defined error-handling mechanism, for errors encountered during decoding, as
+ * registered through {@link #register_error(String, PyObject)}. The return value is the return
+ * from the error handler indicating the replacement codec output and the the position at which
+ * to resume decoding. invokes the mechanism described in PEP-293.
+ *
+ * @param errors name of the error policy (or null meaning "strict")
+ * @param encoding name of encoding that encountered the error
+ * @param toDecode bytes being decoded
+ * @param start index of first byte it couldn't decode
+ * @param end index+1 of last byte it couldn't decode (usually becomes the resume point)
+ * @param reason contribution to error message if any
+ * @return must be a tuple <code>(replacement_unicode, resume_index)</code>
+ */
+ public static PyObject decoding_error(String errors, String encoding, String toDecode,
int start, int end, String reason) {
+ // Retrieve handler registered through register_error(). null is equivalent to "strict".
PyObject errorHandler = lookup_error(errors);
- PyException exc = Py.UnicodeDecodeError(encoding, toEncode, start, end, reason);
+ // Construct an exception to hand to the error handler
+ PyException exc = Py.UnicodeDecodeError(encoding, toDecode, start, end, reason);
exc.normalize();
+ // And invoke the handler.
return errorHandler.__call__(new PyObject[] {exc.value});
}
- private static void checkErrorHandlerReturn(String errors, PyObject replacement) {
- if (!(replacement instanceof PyTuple) || replacement.__len__() != 2
- || !(replacement.__getitem__(0) instanceof PyBaseString)
- || !(replacement.__getitem__(1) instanceof PyInteger)) {
+ /**
+ * Check thet the error handler returned a tuple
+ * <code>(replacement_unicode, resume_index)</code>.
+ *
+ * @param errors name of the error policy (or null meaning "strict")
+ * @param replacementSpec from error handler
+ */
+ private static void checkErrorHandlerReturn(String errors, PyObject replacementSpec) {
+ if (!(replacementSpec instanceof PyTuple) || replacementSpec.__len__() != 2
+ || !(replacementSpec.__getitem__(0) instanceof PyBaseString)
+ || !(replacementSpec.__getitem__(1) instanceof PyInteger)) {
throw new PyException(Py.TypeError, "error_handler " + errors
+ " must return a tuple of (replacement, new position)");
}
}
+
+ /**
+ * Given the return from some codec error handler (invoked while decoding), which specifies a
+ * resume position, and the length of buffer being decoded, check and interpret the resume
+ * position. Negative indexes in the error handler return are interpreted as "from the end". If
+ * the result would be out of bounds in the bytes being decoded, an exception is raised.
+ *
+ * @param size of byte buffer being decoded
+ * @param errorTuple returned from error handler
+ * @return absolute resume position.
+ */
+ public static int calcNewPosition(int size, PyObject errorTuple) {
+ int newPosition = ((PyInteger)errorTuple.__getitem__(1)).getValue();
+ if (newPosition < 0) {
+ newPosition = size + newPosition;
+ }
+ if (newPosition > size || newPosition < 0) {
+ throw Py.IndexError(newPosition + " out of bounds of encoded string");
+ }
+ return newPosition;
+ }
}
diff --git a/src/org/python/modules/_codecs.java b/src/org/python/modules/_codecs.java
--- a/src/org/python/modules/_codecs.java
+++ b/src/org/python/modules/_codecs.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c)2012 Jython Developers Original Java version copyright 2000 Finn Bock
+ * Copyright (c)2013 Jython Developers. Original Java version copyright 2000 Finn Bock.
*
* This program contains material copyrighted by: Copyright (c) Corporation for National Research
* Initiatives. Originally written by Marc-Andre Lemburg (mal at lemburg.com).
@@ -52,8 +52,30 @@
return EncodingMap.buildEncodingMap(map);
}
- private static PyTuple decode_tuple(String s, int len) {
- return new PyTuple(new PyUnicode(s), Py.newInteger(len));
+ /**
+ * Convenience method to construct the return value of decoders, providing the Unicode result as
+ * a String, and the number of bytes consumed.
+ *
+ * @param u the unicode result as a UTF-16 Java String
+ * @param bytesConsumed the number of bytes consumed
+ * @return the tuple (unicode(u), bytesConsumed)
+ */
+ private static PyTuple decode_tuple(String u, int bytesConsumed) {
+ return new PyTuple(new PyUnicode(u), Py.newInteger(bytesConsumed));
+ }
+
+ /**
+ * Convenience method to construct the return value of decoders, providing the Unicode result
+ * as a String, and the number of bytes consumed in decoding as either a single-element array or
+ * an int to be used if the array argument is null.
+ *
+ * @param u the unicode result as a UTF-16 Java String
+ * @param consumed if not null, element [0] is the number of bytes consumed
+ * @param defaultBytesConsumed if consumed==null, use this as the number of bytes consumed
+ * @return the tuple (unicode(u), bytesConsumed)
+ */
+ private static PyTuple decode_tuple(String u, int[] consumed, int defaultBytesConsumed) {
+ return decode_tuple(u, consumed != null ? consumed[0] : defaultBytesConsumed);
}
private static PyTuple decode_tuple_str(String s, int len) {
@@ -89,13 +111,18 @@
}
/* --- UTF-7 Codec --------------------------------------------------- */
- public static PyTuple utf_7_decode(String str) {
- return utf_7_decode(str, null);
+ public static PyTuple utf_7_decode(String bytes) {
+ return utf_7_decode(bytes, null);
}
- public static PyTuple utf_7_decode(String str, String errors) {
- int size = str.length();
- return decode_tuple(codecs.PyUnicode_DecodeUTF7(str, errors), size);
+ public static PyTuple utf_7_decode(String bytes, String errors) {
+ return utf_7_decode(bytes, null, false);
+ }
+
+ public static PyTuple utf_7_decode(String bytes, String errors, boolean finalFlag) {
+ int[] consumed = finalFlag ? null : new int[1];
+ String decoded = codecs.PyUnicode_DecodeUTF7Stateful(bytes, errors, consumed);
+ return decode_tuple(decoded, consumed, bytes.length());
}
public static PyTuple utf_7_encode(String str) {
@@ -107,6 +134,7 @@
return encode_tuple(codecs.PyUnicode_EncodeUTF7(str, false, false, errors), size);
}
+ /* --- string-escape Codec -------------------------------------------- */
public static PyTuple escape_decode(String str) {
return escape_decode(str, null);
}
@@ -677,7 +705,7 @@
return decode_tuple(codecs.PyUnicode_DecodeRawUnicodeEscape(str, errors), str.length());
}
- /* --- UnicodeEscape Codec -------------------------------------------- */
+ /* --- unicode-escape Codec ------------------------------------------- */
public static PyTuple unicode_escape_encode(String str) {
return unicode_escape_encode(str, null);
}
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list