From jython-checkins at python.org Fri Mar 3 04:33:12 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 03 Mar 2017 09:33:12 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232101_and_=231996?= =?utf-8?q?=2E_This_required_to_rewrite_parts_of_PyType=2EcreateAllSlots?= =?utf-8?q?=2E?= Message-ID: <20170303093311.86235.58425.63096F61@psf.io> https://hg.python.org/jython/rev/4884658aa644 changeset: 8041:4884658aa644 user: Stefan Richthofer date: Fri Mar 03 10:32:43 2017 +0100 summary: Fixed #2101 and #1996. This required to rewrite parts of PyType.createAllSlots. files: Lib/test/test_slots_jy.py | 44 ++++++- NEWS | 5 +- src/org/python/core/PyType.java | 134 +++++++++++++++++-- 3 files changed, 164 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_slots_jy.py b/Lib/test/test_slots_jy.py --- a/Lib/test/test_slots_jy.py +++ b/Lib/test/test_slots_jy.py @@ -229,11 +229,53 @@ self.assertIn("__weakref__", dir(self.make_class(HashMap, "__weakref__")())) +class MultiInheritanceSlotsTestCase(unittest.TestCase): + + def test_diamond_multi_inheritance_second_branch(self): + # see issue bugs.jython.org/issue2101 + # also related: bugs.jython.org/issue1996 + result = [] + + class A(object): + def method(self): + pass + + class B(A): + __slots__ = ('b') + def method(self): + result.append('B.method begin') + super(B, self).method() + self.b = 'b' + result.append('B.method end') + + class C(A): + def method(self): + result.append('C.method begin') + super(C, self).method() + self.c = 'c' + result.append('C.method end') + + class D1(B, C): + def method(self): super(D1, self).method() + + class D2(C, B): + def method(self): super(D2, self).method() + + D1().method() + self.assertEqual(result, + ['B.method begin', 'C.method begin', 'C.method end', 'B.method end']) + result = [] + D2().method() + self.assertEqual(result, + ['C.method begin', 'B.method begin', 'B.method end', 'C.method end']) + + def test_main(): test_support.run_unittest(SlottedTestCase, SlottedWithDictTestCase, SlottedWithWeakrefTestCase, - SpecialSlotsBaseTestCase) + SpecialSlotsBaseTestCase, + MultiInheritanceSlotsTestCase) if __name__ == '__main__': diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ Jython 2.7.1rc1 Bugs fixed + - [ 1996 ] Core slots array out of bounds with multiple inheritance + - [ 2101 ] Diamond-style multiple inheritance fails when using __slots__ on the second branch - [ 2552 ] installing scandir via pip fails (breaks e.g. installing pathlib2 via pip) - [ 2534 ] os.getlogin() returns a wrong user or returns an exception - [ 2553 ] sys.getwindowsversion not implemented (breaks pathlib on Windows) @@ -34,8 +36,7 @@ - [ 2446 ] Support SNI for SSL/TLS client sockets - [ 2455 ] Java classes in packages with __init__.py not found - [ 2481 ] Update urllib2.py from 2.7.11 - - [ 2514 ] Jython Class.__subclasses__() does not match Python output - (not in load order) + - [ 2514 ] Jython Class.__subclasses__() does not match Python output (not in load order) - [ 2413 ] ElementTree.write doesn't close files if used with invalid encoding - [ 2443 ] java.util.Map derived classes lack iterkeys, itervalues methods - [ 2516 ] _get_open_ssl_key_manager tries to validate that the private and diff --git a/src/org/python/core/PyType.java b/src/org/python/core/PyType.java --- a/src/org/python/core/PyType.java +++ b/src/org/python/core/PyType.java @@ -8,6 +8,9 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.IdentityHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -90,9 +93,12 @@ /** MethodCacheEntry version tag. */ private volatile Object versionTag = new Object(); - /** The number of __slots__ defined. */ + /** The number of __slots__ defined by this type + bases. */ private int numSlots; + /** The number of __slots__ defined by this type itself. */ + private int ownSlots = 0; + private transient ReferenceQueue subclasses_refq = new ReferenceQueue(); private Set> subclasses = Generic.linkedHashSet(); @@ -218,25 +224,105 @@ } /** + * Used internally by {@link #createAllSlots()}. + * Builds a naive pseudo mro used to collect all slot names relevant for this type. + * + * @param tp type to be investigated + * @param dest list collecting all ancestors + * @param slotsMap map linking each type to its slots + * @return position of first ancestor that is not equal to or ancestor of primary base + */ + private static int findSlottedAncestors(PyType tp, List dest, + IdentityHashMap slotsMap) { + int baseEnd = 0; + if (tp.base != null && tp.base.numSlots > 0 && !slotsMap.containsKey(tp.base)) { + findSlottedAncestors(tp.base, dest, slotsMap); + } + baseEnd = dest.size(); + PyObject slots = tp.dict.__finditem__("__slots__"); + if (slots != null) { + dest.add(tp); // to keep track of order + slotsMap.put(tp, slots); + } + if (tp.bases.length > 1) { + for (PyObject base: tp.bases) { + if (base == tp.base || !(base instanceof PyType) || ((PyType) base).numSlots == 0 + || slotsMap.containsKey((PyType) base)) { + continue; + } + findSlottedAncestors((PyType) base, dest, slotsMap); + } + } + return baseEnd; + } + + /** + * Used internally by {@link #createAllSlots()}. + * Adds all names in {@code slots} to {@code dest}. + * + * @param slots names to be added as slots + * @param dest set collecting all slots + */ + private static void insertSlots(PyObject slots, Set dest) { + if (slots instanceof PyString) { + slots = new PyTuple(slots); + } + // Check for valid slot names and create them. + for (PyObject slot: slots.asIterable()) { + String slotName = confirmIdentifier(slot); + if (slotName.equals("__dict__") || slotName.equals("__weakref__")) { + continue; + } + dest.add(slotName); + } + } + + /** * Create all slots and related descriptors. * * @param mayAddDict whether a __dict__ descriptor is allowed on this type * @param mayAddWeak whether a __weakref__ descriptor is allowed on this type */ private void createAllSlots(boolean mayAddDict, boolean mayAddWeak) { - numSlots = base.numSlots; + List slottedAncestors = new ArrayList<>(base.mro.length+(bases.length-1)*3+1); + IdentityHashMap slotsMap = + new IdentityHashMap<>(slottedAncestors.size()); + /* Here we would need the mro to search for slots (also in secondary bases) properly, + but mro hasn't been set up yet. So we quickly (?) build a pseudo mro sufficient to + find all slots. */ + int baseEnd = findSlottedAncestors(this, slottedAncestors, slotsMap); + // baseEnd is the first position of an ancestor not equal to or ancestor of primary base + int slots_tmp = 0; // used for various purpose, first to accumulate maximal slot count + for (PyType anc: slottedAncestors) { + slots_tmp += anc.numSlots; + } + /* In allSlots we collect slots of primary base first, then of this type, + then of secondary bases. + At any time we prevent it from containing __dict__ or __weakref__. */ + + // we know the required capacity, so the set likely won't be resized + LinkedHashSet allSlots = new LinkedHashSet<>(2*slots_tmp); + if (baseEnd > 0) { + for (int i = 0; i < baseEnd; ++i) { + insertSlots(slotsMap.get(slottedAncestors.get(i)), allSlots); + } + } + assert allSlots.size() == base.numSlots; + boolean wantDict = false; boolean wantWeak = false; PyObject slots = dict.__finditem__("__slots__"); - + ownSlots = 0; // to keep track of slots defined by this type itself for isSolidBase + /* from now on, slots_tmp stores position where other ancestors than primary base + begin (points to this type if it defines own slots) */ if (slots == null) { wantDict = mayAddDict; wantWeak = mayAddWeak; + slots_tmp = baseEnd; } else { if (slots instanceof PyString) { slots = new PyTuple(slots); } - // Check for valid slot names and create them. Handle two special cases for (PyObject slot : slots.asIterable()) { String slotName = confirmIdentifier(slot); @@ -264,16 +350,13 @@ continue; } } - - slotName = mangleName(name, slotName); - if (dict.__finditem__(slotName) == null) { - dict.__setitem__(slotName, new PySlot(this, slotName, numSlots++)); + if (allSlots.add(slotName)) { + ++ownSlots; } } - - // Secondary bases may provide weakrefs or dict - if (bases.length > 1 - && ((mayAddDict && !wantDict) || (mayAddWeak && !wantWeak))) { + if (bases.length > 1 && + ((mayAddDict && !wantDict) || (mayAddWeak && !wantWeak))) { + // Secondary bases may provide weakrefs or dict for (PyObject base : bases) { if (base == this.base) { // Skip primary base @@ -291,7 +374,7 @@ break; } - PyType baseType = (PyType)base; + PyType baseType = (PyType) base; if (mayAddDict && !wantDict && baseType.needs_userdict) { wantDict = true; } @@ -304,7 +387,28 @@ } } } + slots_tmp = baseEnd + 1; } + for (int i = slots_tmp; i < slottedAncestors.size(); ++i) { + insertSlots(slotsMap.get(slottedAncestors.get(i)), allSlots); + } + numSlots = allSlots.size(); + int slotPos = 0; + Iterator slotIter = allSlots.iterator(); + // skip slot names belonging to primary base (i.e. first base.numSlots ones) + for (; slotPos < base.numSlots; ++slotPos) { + slotIter.next(); + } + while (slotIter.hasNext()) { + String slotName = slotIter.next(); + slotName = mangleName(name, slotName); + if (dict.__finditem__(slotName) == null) { + dict.__setitem__(slotName, new PySlot(this, slotName, slotPos++)); + } else { + --numSlots; + } + } + assert slotPos == numSlots; if (wantDict) { createDictSlot(); @@ -1127,9 +1231,7 @@ } private static boolean isSolidBase(PyType type) { - return type.underlying_class != null || - (type.numSlots - (type.base != null ? type.base.numSlots : 0) != 0 && - !type.needs_userdict); + return type.underlying_class != null || (type.ownSlots != 0 && !type.needs_userdict); } /** -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Mar 3 09:25:11 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 03 Mar 2017 14:25:11 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Merged_https=3A//github=2Ec?= =?utf-8?q?om/jythontools/jython/pull/59=3A_Update_NEWS?= Message-ID: <20170303142510.50806.833.BE82F5AD@psf.io> https://hg.python.org/jython/rev/4b96832f0b76 changeset: 8042:4b96832f0b76 user: James Mudd date: Fri Mar 03 15:23:59 2017 +0100 summary: Merged https://github.com/jythontools/jython/pull/59: Update NEWS files: NEWS | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -34,6 +34,7 @@ - [ 2533 ] Opcode.java is outdated -> breaks PyBytecode.interpret - [ 2502 ] Missing OpenFlags enum entry makes Jython clash with JRuby dependency - [ 2446 ] Support SNI for SSL/TLS client sockets + - [ PR50 ] Calling Java vararg methods with no arguments fails - [ 2455 ] Java classes in packages with __init__.py not found - [ 2481 ] Update urllib2.py from 2.7.11 - [ 2514 ] Jython Class.__subclasses__() does not match Python output (not in load order) -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Mar 3 12:44:13 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 03 Mar 2017 17:44:13 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Updated_NEWS=2E?= Message-ID: <20170303174413.30002.50175.F6BEC119@psf.io> https://hg.python.org/jython/rev/2a61883064e1 changeset: 8043:2a61883064e1 user: Stefan Richthofer date: Fri Mar 03 18:43:32 2017 +0100 summary: Updated NEWS. files: NEWS | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2521 ] Windows installation (all) fails on Windows 10 + - [ 2557 ] ongoing pain with platform detection via os.name and sys.platform - [ 1996 ] Core slots array out of bounds with multiple inheritance - [ 2101 ] Diamond-style multiple inheritance fails when using __slots__ on the second branch - [ 2552 ] installing scandir via pip fails (breaks e.g. installing pathlib2 via pip) -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Mar 3 12:50:20 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 03 Mar 2017 17:50:20 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232561=2E?= Message-ID: <20170303175020.86572.9027.4CC3BCB8@psf.io> https://hg.python.org/jython/rev/862934df72f6 changeset: 8044:862934df72f6 user: Stefan Richthofer date: Fri Mar 03 18:50:10 2017 +0100 summary: Fixed #2561. files: Lib/platform.py | 6 ++++++ NEWS | 1 + 2 files changed, 7 insertions(+), 0 deletions(-) diff --git a/Lib/platform.py b/Lib/platform.py --- a/Lib/platform.py +++ b/Lib/platform.py @@ -588,6 +588,12 @@ from win32con import HKEY_LOCAL_MACHINE, VER_PLATFORM_WIN32_NT, \ VER_PLATFORM_WIN32_WINDOWS, VER_NT_WORKSTATION except ImportError: + if sys.platform.startswith("java"): + if os._name == 'nt': + unm = os.uname() + return unm[2], unm[3], csd, ptype + else: + return release, version, csd, ptype # Emulate the win32api module using Python APIs try: sys.getwindowsversion diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2561 ] win32_ver raises exception (breaks test_platform on windows) - [ 2521 ] Windows installation (all) fails on Windows 10 - [ 2557 ] ongoing pain with platform detection via os.name and sys.platform - [ 1996 ] Core slots array out of bounds with multiple inheritance -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Mar 3 13:18:28 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 03 Mar 2017 18:18:28 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Updated_httplib=2C_urllib?= =?utf-8?q?=2C_urllib2_to_CPython_2=2E7=2E13_versions=2E?= Message-ID: <20170303181827.28518.27232.8F092ECF@psf.io> https://hg.python.org/jython/rev/16b977e954b4 changeset: 8045:16b977e954b4 user: Stefan Richthofer date: Fri Mar 03 19:17:26 2017 +0100 summary: Updated httplib, urllib, urllib2 to CPython 2.7.13 versions. files: Lib/test/test_httplib.py | 118 +++++++++++++++- Lib/urllib.py | 2 +- lib-python/2.7/httplib.py | 35 +--- lib-python/2.7/test/test_httplib.py | 118 +++++++++++++++- lib-python/2.7/urllib.py | 2 +- 5 files changed, 244 insertions(+), 31 deletions(-) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -241,6 +241,120 @@ self.assertEqual(resp.getheader('First'), 'val') self.assertEqual(resp.getheader('Second'), 'val') + def test_malformed_truncation(self): + # Other malformed header lines, especially without colons, used to + # cause the rest of the header section to be truncated + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'Public-Key-Pins: \n' + b'pin-sha256="xxx=";\n' + b'report-uri="https://..."\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertIsNotNone(resp.getheader('Public-Key-Pins')) + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + def test_blank_line_forms(self): + # Test that both CRLF and LF blank lines can terminate the header + # section and start the body + for blank in (b'\r\n', b'\n'): + resp = b'HTTP/1.1 200 OK\r\n' b'Transfer-Encoding: chunked\r\n' + resp += blank + resp += b'4\r\nbody\r\n0\r\n\r\n' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + resp = b'HTTP/1.0 200 OK\r\n' + blank + b'body' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'body') + + # A blank line ending in CR is not treated as the end of the HTTP + # header section, therefore header fields following it should be + # parsed if possible + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'\r' + b'Name: value\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + # No header fields nor blank line + resp = b'HTTP/1.0 200 OK\r\n' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'') + + def test_from_line(self): + # The parser handles "From" lines specially, so test this does not + # affect parsing the rest of the header section + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'From start\r\n' + b' continued\r\n' + b'Name: value\r\n' + b'From middle\r\n' + b' continued\r\n' + b'Transfer-Encoding: chunked\r\n' + b'From end\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertIsNotNone(resp.getheader('Name')) + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + resp = ( + b'HTTP/1.0 200 OK\r\n' + b'From alone\r\n' + b'\r\n' + b'body' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'body') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytearray(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytearray(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + folded = resp.getheader('obs-fold') + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + def test_invalid_headers(self): conn = httplib.HTTPConnection('example.com') conn.sock = FakeSocket('') @@ -525,7 +639,7 @@ self.assertTrue(hasattr(resp,'fileno'), 'HTTPResponse should expose a fileno attribute') - # Test lines overflowing the max line size (_MAXLINE in http.client) + # Test lines overflowing the max line size (_MAXLINE in httplib) def test_overflowing_status_line(self): self.skipTest("disabled for HTTP 0.9 support") @@ -624,7 +738,7 @@ def testHTTPSConnectionSourceAddress(self): self.conn = httplib.HTTPSConnection(HOST, self.port, source_address=('', self.source_port)) - # We don't test anything here other the constructor not barfing as + # We don't test anything here other than the constructor not barfing as # this code doesn't deal with setting up an active running SSL server # for an ssl_wrapped connect() to actually return from. diff --git a/Lib/urllib.py b/Lib/urllib.py --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -142,7 +142,7 @@ self.key_file = x509.get('key_file') self.cert_file = x509.get('cert_file') self.context = context - self.addheaders = [('User-Agent', self.version)] + self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')] self.__tempfiles = [] self.__unlink = os.unlink # See cleanup() self.tempcache = None diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -242,7 +242,7 @@ # # VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1 -# the patterns for both name and value are more leniant than RFC +# the patterns for both name and value are more lenient than RFC # definitions to allow for backwards compatibility _is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match _is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search @@ -273,9 +273,8 @@ Read header lines up to the entirely blank line that terminates them. The (normally blank) line that ends the headers is skipped, but not - included in the returned list. If a non-header line ends the headers, - (which is an error), an attempt is made to backspace over it; it is - never included in the returned list. + included in the returned list. If an invalid line is found in the + header section, it is skipped, and further lines are processed. The variable self.status is set to the empty string if all went well, otherwise it is an error message. The variable self.headers is a @@ -302,19 +301,17 @@ self.status = '' headerseen = "" firstline = 1 - startofline = unread = tell = None - if hasattr(self.fp, 'unread'): - unread = self.fp.unread - elif self.seekable: + tell = None + if not hasattr(self.fp, 'unread') and self.seekable: tell = self.fp.tell while True: if len(hlist) > _MAXHEADERS: raise HTTPException("got more than %d headers" % _MAXHEADERS) if tell: try: - startofline = tell() + tell() except IOError: - startofline = tell = None + tell = None self.seekable = 0 line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -345,26 +342,14 @@ # It's a legal header line, save it. hlist.append(line) self.addheader(headerseen, line[len(headerseen)+1:].strip()) - continue elif headerseen is not None: # An empty header name. These aren't allowed in HTTP, but it's # probably a benign mistake. Don't add the header, just keep # going. - continue + pass else: - # It's not a header line; throw it back and stop here. - if not self.dict: - self.status = 'No headers' - else: - self.status = 'Non-header line where header expected' - # Try to undo the read. - if unread: - unread(line) - elif tell: - self.fp.seek(startofline) - else: - self.status = self.status + '; bad seek' - break + # It's not a header line; skip it and try the next line. + self.status = 'Non-header line where header expected' class HTTPResponse: diff --git a/lib-python/2.7/test/test_httplib.py b/lib-python/2.7/test/test_httplib.py --- a/lib-python/2.7/test/test_httplib.py +++ b/lib-python/2.7/test/test_httplib.py @@ -241,6 +241,120 @@ self.assertEqual(resp.getheader('First'), 'val') self.assertEqual(resp.getheader('Second'), 'val') + def test_malformed_truncation(self): + # Other malformed header lines, especially without colons, used to + # cause the rest of the header section to be truncated + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'Public-Key-Pins: \n' + b'pin-sha256="xxx=";\n' + b'report-uri="https://..."\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertIsNotNone(resp.getheader('Public-Key-Pins')) + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + def test_blank_line_forms(self): + # Test that both CRLF and LF blank lines can terminate the header + # section and start the body + for blank in (b'\r\n', b'\n'): + resp = b'HTTP/1.1 200 OK\r\n' b'Transfer-Encoding: chunked\r\n' + resp += blank + resp += b'4\r\nbody\r\n0\r\n\r\n' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + resp = b'HTTP/1.0 200 OK\r\n' + blank + b'body' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'body') + + # A blank line ending in CR is not treated as the end of the HTTP + # header section, therefore header fields following it should be + # parsed if possible + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'\r' + b'Name: value\r\n' + b'Transfer-Encoding: chunked\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + # No header fields nor blank line + resp = b'HTTP/1.0 200 OK\r\n' + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'') + + def test_from_line(self): + # The parser handles "From" lines specially, so test this does not + # affect parsing the rest of the header section + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'From start\r\n' + b' continued\r\n' + b'Name: value\r\n' + b'From middle\r\n' + b' continued\r\n' + b'Transfer-Encoding: chunked\r\n' + b'From end\r\n' + b'\r\n' + b'4\r\nbody\r\n0\r\n\r\n' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertIsNotNone(resp.getheader('Name')) + self.assertEqual(resp.getheader('Transfer-Encoding'), 'chunked') + self.assertEqual(resp.read(), b'body') + + resp = ( + b'HTTP/1.0 200 OK\r\n' + b'From alone\r\n' + b'\r\n' + b'body' + ) + resp = httplib.HTTPResponse(FakeSocket(resp)) + resp.begin() + self.assertEqual(resp.read(), b'body') + + def test_parse_all_octets(self): + # Ensure no valid header field octet breaks the parser + body = ( + b'HTTP/1.1 200 OK\r\n' + b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters + b'VCHAR: ' + bytearray(range(0x21, 0x7E + 1)) + b'\r\n' + b'obs-text: ' + bytearray(range(0x80, 0xFF + 1)) + b'\r\n' + b'obs-fold: text\r\n' + b' folded with space\r\n' + b'\tfolded with tab\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + sock = FakeSocket(body) + resp = httplib.HTTPResponse(sock) + resp.begin() + self.assertEqual(resp.getheader('Content-Length'), '0') + self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value') + vchar = ''.join(map(chr, range(0x21, 0x7E + 1))) + self.assertEqual(resp.getheader('VCHAR'), vchar) + self.assertIsNotNone(resp.getheader('obs-text')) + folded = resp.getheader('obs-fold') + self.assertTrue(folded.startswith('text')) + self.assertIn(' folded with space', folded) + self.assertTrue(folded.endswith('folded with tab')) + def test_invalid_headers(self): conn = httplib.HTTPConnection('example.com') conn.sock = FakeSocket('') @@ -525,7 +639,7 @@ self.assertTrue(hasattr(resp,'fileno'), 'HTTPResponse should expose a fileno attribute') - # Test lines overflowing the max line size (_MAXLINE in http.client) + # Test lines overflowing the max line size (_MAXLINE in httplib) def test_overflowing_status_line(self): self.skipTest("disabled for HTTP 0.9 support") @@ -624,7 +738,7 @@ def testHTTPSConnectionSourceAddress(self): self.conn = httplib.HTTPSConnection(HOST, self.port, source_address=('', self.source_port)) - # We don't test anything here other the constructor not barfing as + # We don't test anything here other than the constructor not barfing as # this code doesn't deal with setting up an active running SSL server # for an ssl_wrapped connect() to actually return from. diff --git a/lib-python/2.7/urllib.py b/lib-python/2.7/urllib.py --- a/lib-python/2.7/urllib.py +++ b/lib-python/2.7/urllib.py @@ -138,7 +138,7 @@ self.key_file = x509.get('key_file') self.cert_file = x509.get('cert_file') self.context = context - self.addheaders = [('User-Agent', self.version)] + self.addheaders = [('User-Agent', self.version), ('Accept', '*/*')] self.__tempfiles = [] self.__unlink = os.unlink # See cleanup() self.tempcache = None -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sat Mar 4 16:03:22 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sat, 04 Mar 2017 21:03:22 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Added_openssl=5Fmd=5Fmeth?= =?utf-8?q?=5Fnames_to_=5Fhashlib=2Ejava=2E_This_is_a_crucial_prerequisite?= Message-ID: <20170304210321.28155.27547.88E476ED@psf.io> https://hg.python.org/jython/rev/9ff3b171b9b3 changeset: 8046:9ff3b171b9b3 user: Stefan Richthofer date: Sat Mar 04 22:03:08 2017 +0100 summary: Added openssl_md_meth_names to _hashlib.java. This is a crucial prerequisite for std-lib update. Without it, regrtests won't even start running properly after a would-be std-lib update to 2.7.13. files: src/org/python/modules/_hashlib.java | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diff --git a/src/org/python/modules/_hashlib.java b/src/org/python/modules/_hashlib.java --- a/src/org/python/modules/_hashlib.java +++ b/src/org/python/modules/_hashlib.java @@ -9,8 +9,10 @@ import org.python.core.ClassDictInit; import org.python.core.Py; import org.python.core.PyArray; +import org.python.core.PyFrozenSet; import org.python.core.PyObject; import org.python.core.PyString; +import org.python.core.PyTuple; import org.python.core.PyType; import org.python.core.PyUnicode; import org.python.core.Untraversable; @@ -37,6 +39,11 @@ put("sha512", "sha-512"); }}; + public static final PyFrozenSet openssl_md_meth_names = + new PyFrozenSet(new PyTuple(Py.newString("md5"), Py.newString("sha1"), + Py.newString("sha224"), Py.newString("sha256"), Py.newString("sha384"), + Py.newString("sha512"))); + public static void classDictInit(PyObject dict) { dict.__setitem__("__name__", Py.newString("_hashlib")); dict.__setitem__("algorithmMap", null); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sat Mar 4 18:04:55 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Sat, 04 Mar 2017 23:04:55 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232563_by_adding_som?= =?utf-8?q?e_Java_integration_to_locale=2Epy=2E?= Message-ID: <20170304230455.27924.41649.BE989FBB@psf.io> https://hg.python.org/jython/rev/19080570ee62 changeset: 8047:19080570ee62 user: Stefan Richthofer date: Sun Mar 05 00:04:39 2017 +0100 summary: Fixed #2563 by adding some Java integration to locale.py. files: Lib/locale.py | 1891 +++++++++++++++++++++++++++++++++++++ 1 files changed, 1891 insertions(+), 0 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py new file mode 100644 --- /dev/null +++ b/Lib/locale.py @@ -0,0 +1,1891 @@ +""" Locale support. + + The module provides low-level access to the C lib's locale APIs + and adds high level number formatting APIs as well as a locale + aliasing engine to complement these. + + The aliasing engine includes support for many commonly used locale + names and maps them to values suitable for passing to the C lib's + setlocale() function. It also includes default encodings for all + supported locale names. + +""" + +import sys +import encodings +import encodings.aliases +import re +import operator +import functools + +try: + _unicode = unicode +except NameError: + # If Python is built without Unicode support, the unicode type + # will not exist. Fake one. + class _unicode(object): + pass + +# Try importing the _locale module. +# +# If this fails, fall back on a basic 'C' locale emulation. + +# Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before +# trying the import. So __all__ is also fiddled at the end of the file. +__all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", + "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm", + "str", "atof", "atoi", "format", "format_string", "currency", + "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", + "LC_NUMERIC", "LC_ALL", "CHAR_MAX"] + +try: + + from _locale import * + +except ImportError: + + # Locale emulation + + CHAR_MAX = 127 + LC_ALL = 6 + LC_COLLATE = 3 + LC_CTYPE = 0 + LC_MESSAGES = 5 + LC_MONETARY = 4 + LC_NUMERIC = 1 + LC_TIME = 2 + Error = ValueError + + def localeconv(): + """ localeconv() -> dict. + Returns numeric and monetary locale-specific parameters. + """ + # 'C' locale default values + return {'grouping': [127], + 'currency_symbol': '', + 'n_sign_posn': 127, + 'p_cs_precedes': 127, + 'n_cs_precedes': 127, + 'mon_grouping': [], + 'n_sep_by_space': 127, + 'decimal_point': '.', + 'negative_sign': '', + 'positive_sign': '', + 'p_sep_by_space': 127, + 'int_curr_symbol': '', + 'p_sign_posn': 127, + 'thousands_sep': '', + 'mon_thousands_sep': '', + 'frac_digits': 127, + 'mon_decimal_point': '', + 'int_frac_digits': 127} + + def setlocale(category, value=None): + """ setlocale(integer,string=None) -> string. + Activates/queries locale processing. + """ + if value not in (None, '', 'C'): + raise Error, '_locale emulation only supports "C" locale' + return 'C' + + def strcoll(a,b): + """ strcoll(string,string) -> int. + Compares two strings according to the locale. + """ + return cmp(a,b) + + def strxfrm(s): + """ strxfrm(string) -> string. + Returns a string that behaves for cmp locale-aware. + """ + return s + + +_localeconv = localeconv + +# With this dict, you can override some items of localeconv's return value. +# This is useful for testing purposes. +_override_localeconv = {} + + at functools.wraps(_localeconv) +def localeconv(): + d = _localeconv() + if _override_localeconv: + d.update(_override_localeconv) + return d + + +### Number formatting APIs + +# Author: Martin von Loewis +# improved by Georg Brandl + +# Iterate over grouping intervals +def _grouping_intervals(grouping): + last_interval = None + for interval in grouping: + # if grouping is -1, we are done + if interval == CHAR_MAX: + return + # 0: re-use last group ad infinitum + if interval == 0: + if last_interval is None: + raise ValueError("invalid grouping") + while True: + yield last_interval + yield interval + last_interval = interval + +#perform the grouping from right to left +def _group(s, monetary=False): + conv = localeconv() + thousands_sep = conv[monetary and 'mon_thousands_sep' or 'thousands_sep'] + grouping = conv[monetary and 'mon_grouping' or 'grouping'] + if not grouping: + return (s, 0) + if s[-1] == ' ': + stripped = s.rstrip() + right_spaces = s[len(stripped):] + s = stripped + else: + right_spaces = '' + left_spaces = '' + groups = [] + for interval in _grouping_intervals(grouping): + if not s or s[-1] not in "0123456789": + # only non-digit characters remain (sign, spaces) + left_spaces = s + s = '' + break + groups.append(s[-interval:]) + s = s[:-interval] + if s: + groups.append(s) + groups.reverse() + return ( + left_spaces + thousands_sep.join(groups) + right_spaces, + len(thousands_sep) * (len(groups) - 1) + ) + +# Strip a given amount of excess padding from the given string +def _strip_padding(s, amount): + lpos = 0 + while amount and s[lpos] == ' ': + lpos += 1 + amount -= 1 + rpos = len(s) - 1 + while amount and s[rpos] == ' ': + rpos -= 1 + amount -= 1 + return s[lpos:rpos+1] + +_percent_re = re.compile(r'%(?:\((?P.*?)\))?' + r'(?P[-#0-9 +*.hlL]*?)[eEfFgGdiouxXcrs%]') + +def format(percent, value, grouping=False, monetary=False, *additional): + """Returns the locale-aware substitution of a %? specifier + (percent). + + additional is for format strings which contain one or more + '*' modifiers.""" + # this is only for one-percent-specifier strings and this should be checked + match = _percent_re.match(percent) + if not match or len(match.group())!= len(percent): + raise ValueError(("format() must be given exactly one %%char " + "format specifier, %s not valid") % repr(percent)) + return _format(percent, value, grouping, monetary, *additional) + +def _format(percent, value, grouping=False, monetary=False, *additional): + if additional: + formatted = percent % ((value,) + additional) + else: + formatted = percent % value + # floats and decimal ints need special action! + if percent[-1] in 'eEfFgG': + seps = 0 + parts = formatted.split('.') + if grouping: + parts[0], seps = _group(parts[0], monetary=monetary) + decimal_point = localeconv()[monetary and 'mon_decimal_point' + or 'decimal_point'] + formatted = decimal_point.join(parts) + if seps: + formatted = _strip_padding(formatted, seps) + elif percent[-1] in 'diu': + seps = 0 + if grouping: + formatted, seps = _group(formatted, monetary=monetary) + if seps: + formatted = _strip_padding(formatted, seps) + return formatted + +def format_string(f, val, grouping=False): + """Formats a string in the same way that the % formatting would use, + but takes the current locale into account. + Grouping is applied if the third parameter is true.""" + percents = list(_percent_re.finditer(f)) + new_f = _percent_re.sub('%s', f) + + if operator.isMappingType(val): + new_val = [] + for perc in percents: + if perc.group()[-1]=='%': + new_val.append('%') + else: + new_val.append(format(perc.group(), val, grouping)) + else: + if not isinstance(val, tuple): + val = (val,) + new_val = [] + i = 0 + for perc in percents: + if perc.group()[-1]=='%': + new_val.append('%') + else: + starcount = perc.group('modifiers').count('*') + new_val.append(_format(perc.group(), + val[i], + grouping, + False, + *val[i+1:i+1+starcount])) + i += (1 + starcount) + val = tuple(new_val) + + return new_f % val + +def currency(val, symbol=True, grouping=False, international=False): + """Formats val according to the currency settings + in the current locale.""" + conv = localeconv() + + # check for illegal values + digits = conv[international and 'int_frac_digits' or 'frac_digits'] + if digits == 127: + raise ValueError("Currency formatting is not possible using " + "the 'C' locale.") + + s = format('%%.%if' % digits, abs(val), grouping, monetary=True) + # '<' and '>' are markers if the sign must be inserted between symbol and value + s = '<' + s + '>' + + if symbol: + smb = conv[international and 'int_curr_symbol' or 'currency_symbol'] + precedes = conv[val<0 and 'n_cs_precedes' or 'p_cs_precedes'] + separated = conv[val<0 and 'n_sep_by_space' or 'p_sep_by_space'] + + if precedes: + s = smb + (separated and ' ' or '') + s + else: + s = s + (separated and ' ' or '') + smb + + sign_pos = conv[val<0 and 'n_sign_posn' or 'p_sign_posn'] + sign = conv[val<0 and 'negative_sign' or 'positive_sign'] + + if sign_pos == 0: + s = '(' + s + ')' + elif sign_pos == 1: + s = sign + s + elif sign_pos == 2: + s = s + sign + elif sign_pos == 3: + s = s.replace('<', sign) + elif sign_pos == 4: + s = s.replace('>', sign) + else: + # the default if nothing specified; + # this should be the most fitting sign position + s = sign + s + + return s.replace('<', '').replace('>', '') + +def str(val): + """Convert float to integer, taking the locale into account.""" + return format("%.12g", val) + +def atof(string, func=float): + "Parses a string as a float according to the locale settings." + #First, get rid of the grouping + ts = localeconv()['thousands_sep'] + if ts: + string = string.replace(ts, '') + #next, replace the decimal point with a dot + dd = localeconv()['decimal_point'] + if dd: + string = string.replace(dd, '.') + #finally, parse the string + return func(string) + +def atoi(str): + "Converts a string to an integer according to the locale settings." + return atof(str, int) + +def _test(): + setlocale(LC_ALL, "") + #do grouping + s1 = format("%d", 123456789,1) + print s1, "is", atoi(s1) + #standard formatting + s1 = str(3.14) + print s1, "is", atof(s1) + +### Locale name aliasing engine + +# Author: Marc-Andre Lemburg, mal at lemburg.com +# Various tweaks by Fredrik Lundh + +# store away the low-level version of setlocale (it's +# overridden below) +_setlocale = setlocale + +# Avoid relying on the locale-dependent .lower() method +# (see issue #1813). +_ascii_lower_map = ''.join( + chr(x + 32 if x >= ord('A') and x <= ord('Z') else x) + for x in range(256) +) + +def normalize(localename): + + """ Returns a normalized locale code for the given locale + name. + + The returned locale code is formatted for use with + setlocale(). + + If normalization fails, the original name is returned + unchanged. + + If the given encoding is not known, the function defaults to + the default encoding for the locale code just like setlocale() + does. + + """ + # Normalize the locale name and extract the encoding + if isinstance(localename, _unicode): + localename = localename.encode('ascii') + fullname = localename.translate(_ascii_lower_map) + if ':' in fullname: + # ':' is sometimes used as encoding delimiter. + fullname = fullname.replace(':', '.') + if '.' in fullname: + langname, encoding = fullname.split('.')[:2] + fullname = langname + '.' + encoding + else: + langname = fullname + encoding = '' + + # First lookup: fullname (possibly with encoding) + norm_encoding = encoding.replace('-', '') + norm_encoding = norm_encoding.replace('_', '') + lookup_name = langname + '.' + encoding + code = locale_alias.get(lookup_name, None) + if code is not None: + return code + #print 'first lookup failed' + + # Second try: langname (without encoding) + code = locale_alias.get(langname, None) + if code is not None: + #print 'langname lookup succeeded' + if '.' in code: + langname, defenc = code.split('.') + else: + langname = code + defenc = '' + if encoding: + # Convert the encoding to a C lib compatible encoding string + norm_encoding = encodings.normalize_encoding(encoding) + #print 'norm encoding: %r' % norm_encoding + norm_encoding = encodings.aliases.aliases.get(norm_encoding, + norm_encoding) + #print 'aliased encoding: %r' % norm_encoding + encoding = locale_encoding_alias.get(norm_encoding, + norm_encoding) + else: + encoding = defenc + #print 'found encoding %r' % encoding + if encoding: + return langname + '.' + encoding + else: + return langname + + else: + return localename + +def _parse_localename(localename): + + """ Parses the locale code for localename and returns the + result as tuple (language code, encoding). + + The localename is normalized and passed through the locale + alias engine. A ValueError is raised in case the locale name + cannot be parsed. + + The language code corresponds to RFC 1766. code and encoding + can be None in case the values cannot be determined or are + unknown to this implementation. + + """ + code = normalize(localename) + if '@' in code: + # Deal with locale modifiers + code, modifier = code.split('@') + if modifier == 'euro' and '.' not in code: + # Assume Latin-9 for @euro locales. This is bogus, + # since some systems may use other encodings for these + # locales. Also, we ignore other modifiers. + return code, 'iso-8859-15' + + if '.' in code: + return tuple(code.split('.')[:2]) + elif code == 'C': + return None, None + raise ValueError, 'unknown locale: %s' % localename + +def _build_localename(localetuple): + + """ Builds a locale code from the given tuple (language code, + encoding). + + No aliasing or normalizing takes place. + + """ + language, encoding = localetuple + if language is None: + language = 'C' + if encoding is None: + return language + else: + return language + '.' + encoding + +def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): + + """ Tries to determine the default locale settings and returns + them as tuple (language code, encoding). + + According to POSIX, a program which has not called + setlocale(LC_ALL, "") runs using the portable 'C' locale. + Calling setlocale(LC_ALL, "") lets it use the default locale as + defined by the LANG variable. Since we don't want to interfere + with the current locale setting we thus emulate the behavior + in the way described above. + + To maintain compatibility with other platforms, not only the + LANG variable is tested, but a list of variables given as + envvars parameter. The first found to be defined will be + used. envvars defaults to the search path used in GNU gettext; + it must always contain the variable name 'LANG'. + + Except for the code 'C', the language code corresponds to RFC + 1766. code and encoding can be None in case the values cannot + be determined. + + """ + + if sys.platform.startswith("java"): + from java.util import Locale + from java.nio.charset import Charset + return Locale.getDefault().toString().__str__(), \ + Charset.defaultCharset().name().__str__() + try: + # check if it's supported by the _locale module + import _locale + code, encoding = _locale._getdefaultlocale() + except (ImportError, AttributeError): + pass + else: + # make sure the code/encoding values are valid + if sys.platform == "win32" and code and code[:2] == "0x": + # map windows language identifier to language name + code = windows_locale.get(int(code, 0)) + # ...add other platform-specific processing here, if + # necessary... + return code, encoding + + # fall back on POSIX behaviour + import os + lookup = os.environ.get + for variable in envvars: + localename = lookup(variable,None) + if localename: + if variable == 'LANGUAGE': + localename = localename.split(':')[0] + break + else: + localename = 'C' + return _parse_localename(localename) + + +def getlocale(category=LC_CTYPE): + + """ Returns the current setting for the given locale category as + tuple (language code, encoding). + + category may be one of the LC_* value except LC_ALL. It + defaults to LC_CTYPE. + + Except for the code 'C', the language code corresponds to RFC + 1766. code and encoding can be None in case the values cannot + be determined. + + """ + localename = _setlocale(category) + if category == LC_ALL and ';' in localename: + raise TypeError, 'category LC_ALL is not supported' + return _parse_localename(localename) + +def setlocale(category, locale=None): + + """ Set the locale for the given category. The locale can be + a string, an iterable of two strings (language code and encoding), + or None. + + Iterables are converted to strings using the locale aliasing + engine. Locale strings are passed directly to the C lib. + + category may be given as one of the LC_* values. + + """ + if locale and type(locale) is not type(""): + # convert to string + locale = normalize(_build_localename(locale)) + return _setlocale(category, locale) + +def resetlocale(category=LC_ALL): + + """ Sets the locale for category to the default setting. + + The default setting is determined by calling + getdefaultlocale(). category defaults to LC_ALL. + + """ + _setlocale(category, _build_localename(getdefaultlocale())) + +if sys.platform.startswith("java"): + from java.nio.charset import Charset + def getpreferredencoding(do_setlocale = True): + return Charset.defaultCharset().name().__str__() +elif sys.platform.startswith("win"): + # On Win32, this will return the ANSI code page + def getpreferredencoding(do_setlocale = True): + """Return the charset that the user is likely using.""" + import _locale + return _locale._getdefaultlocale()[1] +else: + # On Unix, if CODESET is available, use that. + try: + CODESET + except NameError: + # Fall back to parsing environment variables :-( + def getpreferredencoding(do_setlocale = True): + """Return the charset that the user is likely using, + by looking at environment variables.""" + return getdefaultlocale()[1] + else: + def getpreferredencoding(do_setlocale = True): + """Return the charset that the user is likely using, + according to the system configuration.""" + if do_setlocale: + oldloc = setlocale(LC_CTYPE) + try: + setlocale(LC_CTYPE, "") + except Error: + pass + result = nl_langinfo(CODESET) + setlocale(LC_CTYPE, oldloc) + return result + else: + return nl_langinfo(CODESET) + + +### Database +# +# The following data was extracted from the locale.alias file which +# comes with X11 and then hand edited removing the explicit encoding +# definitions and adding some more aliases. The file is usually +# available as /usr/lib/X11/locale/locale.alias. +# + +# +# The local_encoding_alias table maps lowercase encoding alias names +# to C locale encoding names (case-sensitive). Note that normalize() +# first looks up the encoding in the encodings.aliases dictionary and +# then applies this mapping to find the correct C lib name for the +# encoding. +# +locale_encoding_alias = { + + # Mappings for non-standard encoding names used in locale names + '437': 'C', + 'c': 'C', + 'en': 'ISO8859-1', + 'jis': 'JIS7', + 'jis7': 'JIS7', + 'ajec': 'eucJP', + + # Mappings from Python codec names to C lib encoding names + 'ascii': 'ISO8859-1', + 'latin_1': 'ISO8859-1', + 'iso8859_1': 'ISO8859-1', + 'iso8859_10': 'ISO8859-10', + 'iso8859_11': 'ISO8859-11', + 'iso8859_13': 'ISO8859-13', + 'iso8859_14': 'ISO8859-14', + 'iso8859_15': 'ISO8859-15', + 'iso8859_16': 'ISO8859-16', + 'iso8859_2': 'ISO8859-2', + 'iso8859_3': 'ISO8859-3', + 'iso8859_4': 'ISO8859-4', + 'iso8859_5': 'ISO8859-5', + 'iso8859_6': 'ISO8859-6', + 'iso8859_7': 'ISO8859-7', + 'iso8859_8': 'ISO8859-8', + 'iso8859_9': 'ISO8859-9', + 'iso2022_jp': 'JIS7', + 'shift_jis': 'SJIS', + 'tactis': 'TACTIS', + 'euc_jp': 'eucJP', + 'euc_kr': 'eucKR', + 'utf_8': 'UTF-8', + 'koi8_r': 'KOI8-R', + 'koi8_u': 'KOI8-U', + # XXX This list is still incomplete. If you know more + # mappings, please file a bug report. Thanks. +} + +# +# The locale_alias table maps lowercase alias names to C locale names +# (case-sensitive). Encodings are always separated from the locale +# name using a dot ('.'); they should only be given in case the +# language name is needed to interpret the given encoding alias +# correctly (CJK codes often have this need). +# +# Note that the normalize() function which uses this tables +# removes '_' and '-' characters from the encoding part of the +# locale name before doing the lookup. This saves a lot of +# space in the table. +# +# MAL 2004-12-10: +# Updated alias mapping to most recent locale.alias file +# from X.org distribution using makelocalealias.py. +# +# These are the differences compared to the old mapping (Python 2.4 +# and older): +# +# updated 'bg' -> 'bg_BG.ISO8859-5' to 'bg_BG.CP1251' +# updated 'bg_bg' -> 'bg_BG.ISO8859-5' to 'bg_BG.CP1251' +# updated 'bulgarian' -> 'bg_BG.ISO8859-5' to 'bg_BG.CP1251' +# updated 'cz' -> 'cz_CZ.ISO8859-2' to 'cs_CZ.ISO8859-2' +# updated 'cz_cz' -> 'cz_CZ.ISO8859-2' to 'cs_CZ.ISO8859-2' +# updated 'czech' -> 'cs_CS.ISO8859-2' to 'cs_CZ.ISO8859-2' +# updated 'dutch' -> 'nl_BE.ISO8859-1' to 'nl_NL.ISO8859-1' +# updated 'et' -> 'et_EE.ISO8859-4' to 'et_EE.ISO8859-15' +# updated 'et_ee' -> 'et_EE.ISO8859-4' to 'et_EE.ISO8859-15' +# updated 'fi' -> 'fi_FI.ISO8859-1' to 'fi_FI.ISO8859-15' +# updated 'fi_fi' -> 'fi_FI.ISO8859-1' to 'fi_FI.ISO8859-15' +# updated 'iw' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' +# updated 'iw_il' -> 'iw_IL.ISO8859-8' to 'he_IL.ISO8859-8' +# updated 'japanese' -> 'ja_JP.SJIS' to 'ja_JP.eucJP' +# updated 'lt' -> 'lt_LT.ISO8859-4' to 'lt_LT.ISO8859-13' +# updated 'lv' -> 'lv_LV.ISO8859-4' to 'lv_LV.ISO8859-13' +# updated 'sl' -> 'sl_CS.ISO8859-2' to 'sl_SI.ISO8859-2' +# updated 'slovene' -> 'sl_CS.ISO8859-2' to 'sl_SI.ISO8859-2' +# updated 'th_th' -> 'th_TH.TACTIS' to 'th_TH.ISO8859-11' +# updated 'zh_cn' -> 'zh_CN.eucCN' to 'zh_CN.gb2312' +# updated 'zh_cn.big5' -> 'zh_TW.eucTW' to 'zh_TW.big5' +# updated 'zh_tw' -> 'zh_TW.eucTW' to 'zh_TW.big5' +# +# MAL 2008-05-30: +# Updated alias mapping to most recent locale.alias file +# from X.org distribution using makelocalealias.py. +# +# These are the differences compared to the old mapping (Python 2.5 +# and older): +# +# updated 'cs_cs.iso88592' -> 'cs_CZ.ISO8859-2' to 'cs_CS.ISO8859-2' +# updated 'serbocroatian' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' +# updated 'sh' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' +# updated 'sh_hr.iso88592' -> 'sh_HR.ISO8859-2' to 'hr_HR.ISO8859-2' +# updated 'sh_sp' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' +# updated 'sh_yu' -> 'sh_YU.ISO8859-2' to 'sr_CS.ISO8859-2' +# updated 'sp' -> 'sp_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sp_yu' -> 'sp_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sr' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sr at cyrillic' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sr_sp' -> 'sr_SP.ISO8859-2' to 'sr_CS.ISO8859-2' +# updated 'sr_yu' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sr_yu.cp1251 at cyrillic' -> 'sr_YU.CP1251' to 'sr_CS.CP1251' +# updated 'sr_yu.iso88592' -> 'sr_YU.ISO8859-2' to 'sr_CS.ISO8859-2' +# updated 'sr_yu.iso88595' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sr_yu.iso88595 at cyrillic' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# updated 'sr_yu.microsoftcp1251 at cyrillic' -> 'sr_YU.CP1251' to 'sr_CS.CP1251' +# updated 'sr_yu.utf8 at cyrillic' -> 'sr_YU.UTF-8' to 'sr_CS.UTF-8' +# updated 'sr_yu at cyrillic' -> 'sr_YU.ISO8859-5' to 'sr_CS.ISO8859-5' +# +# AP 2010-04-12: +# Updated alias mapping to most recent locale.alias file +# from X.org distribution using makelocalealias.py. +# +# These are the differences compared to the old mapping (Python 2.6.5 +# and older): +# +# updated 'ru' -> 'ru_RU.ISO8859-5' to 'ru_RU.UTF-8' +# updated 'ru_ru' -> 'ru_RU.ISO8859-5' to 'ru_RU.UTF-8' +# updated 'serbocroatian' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8 at latin' +# updated 'sh' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8 at latin' +# updated 'sh_yu' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8 at latin' +# updated 'sr' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' +# updated 'sr at cyrillic' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' +# updated 'sr at latn' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8 at latin' +# updated 'sr_cs.utf8 at latn' -> 'sr_CS.UTF-8' to 'sr_RS.UTF-8 at latin' +# updated 'sr_cs at latn' -> 'sr_CS.ISO8859-2' to 'sr_RS.UTF-8 at latin' +# updated 'sr_yu' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8 at latin' +# updated 'sr_yu.utf8 at cyrillic' -> 'sr_CS.UTF-8' to 'sr_RS.UTF-8' +# updated 'sr_yu at cyrillic' -> 'sr_CS.ISO8859-5' to 'sr_RS.UTF-8' +# + +locale_alias = { + 'a3': 'a3_AZ.KOI8-C', + 'a3_az': 'a3_AZ.KOI8-C', + 'a3_az.koi8c': 'a3_AZ.KOI8-C', + 'af': 'af_ZA.ISO8859-1', + 'af_za': 'af_ZA.ISO8859-1', + 'af_za.iso88591': 'af_ZA.ISO8859-1', + 'am': 'am_ET.UTF-8', + 'am_et': 'am_ET.UTF-8', + 'american': 'en_US.ISO8859-1', + 'american.iso88591': 'en_US.ISO8859-1', + 'ar': 'ar_AA.ISO8859-6', + 'ar_aa': 'ar_AA.ISO8859-6', + 'ar_aa.iso88596': 'ar_AA.ISO8859-6', + 'ar_ae': 'ar_AE.ISO8859-6', + 'ar_ae.iso88596': 'ar_AE.ISO8859-6', + 'ar_bh': 'ar_BH.ISO8859-6', + 'ar_bh.iso88596': 'ar_BH.ISO8859-6', + 'ar_dz': 'ar_DZ.ISO8859-6', + 'ar_dz.iso88596': 'ar_DZ.ISO8859-6', + 'ar_eg': 'ar_EG.ISO8859-6', + 'ar_eg.iso88596': 'ar_EG.ISO8859-6', + 'ar_iq': 'ar_IQ.ISO8859-6', + 'ar_iq.iso88596': 'ar_IQ.ISO8859-6', + 'ar_jo': 'ar_JO.ISO8859-6', + 'ar_jo.iso88596': 'ar_JO.ISO8859-6', + 'ar_kw': 'ar_KW.ISO8859-6', + 'ar_kw.iso88596': 'ar_KW.ISO8859-6', + 'ar_lb': 'ar_LB.ISO8859-6', + 'ar_lb.iso88596': 'ar_LB.ISO8859-6', + 'ar_ly': 'ar_LY.ISO8859-6', + 'ar_ly.iso88596': 'ar_LY.ISO8859-6', + 'ar_ma': 'ar_MA.ISO8859-6', + 'ar_ma.iso88596': 'ar_MA.ISO8859-6', + 'ar_om': 'ar_OM.ISO8859-6', + 'ar_om.iso88596': 'ar_OM.ISO8859-6', + 'ar_qa': 'ar_QA.ISO8859-6', + 'ar_qa.iso88596': 'ar_QA.ISO8859-6', + 'ar_sa': 'ar_SA.ISO8859-6', + 'ar_sa.iso88596': 'ar_SA.ISO8859-6', + 'ar_sd': 'ar_SD.ISO8859-6', + 'ar_sd.iso88596': 'ar_SD.ISO8859-6', + 'ar_sy': 'ar_SY.ISO8859-6', + 'ar_sy.iso88596': 'ar_SY.ISO8859-6', + 'ar_tn': 'ar_TN.ISO8859-6', + 'ar_tn.iso88596': 'ar_TN.ISO8859-6', + 'ar_ye': 'ar_YE.ISO8859-6', + 'ar_ye.iso88596': 'ar_YE.ISO8859-6', + 'arabic': 'ar_AA.ISO8859-6', + 'arabic.iso88596': 'ar_AA.ISO8859-6', + 'as': 'as_IN.UTF-8', + 'az': 'az_AZ.ISO8859-9E', + 'az_az': 'az_AZ.ISO8859-9E', + 'az_az.iso88599e': 'az_AZ.ISO8859-9E', + 'be': 'be_BY.CP1251', + 'be at latin': 'be_BY.UTF-8 at latin', + 'be_by': 'be_BY.CP1251', + 'be_by.cp1251': 'be_BY.CP1251', + 'be_by.microsoftcp1251': 'be_BY.CP1251', + 'be_by.utf8 at latin': 'be_BY.UTF-8 at latin', + 'be_by at latin': 'be_BY.UTF-8 at latin', + 'bg': 'bg_BG.CP1251', + 'bg_bg': 'bg_BG.CP1251', + 'bg_bg.cp1251': 'bg_BG.CP1251', + 'bg_bg.iso88595': 'bg_BG.ISO8859-5', + 'bg_bg.koi8r': 'bg_BG.KOI8-R', + 'bg_bg.microsoftcp1251': 'bg_BG.CP1251', + 'bn_in': 'bn_IN.UTF-8', + 'bokmal': 'nb_NO.ISO8859-1', + 'bokm\xe5l': 'nb_NO.ISO8859-1', + 'br': 'br_FR.ISO8859-1', + 'br_fr': 'br_FR.ISO8859-1', + 'br_fr.iso88591': 'br_FR.ISO8859-1', + 'br_fr.iso885914': 'br_FR.ISO8859-14', + 'br_fr.iso885915': 'br_FR.ISO8859-15', + 'br_fr.iso885915 at euro': 'br_FR.ISO8859-15', + 'br_fr.utf8 at euro': 'br_FR.UTF-8', + 'br_fr at euro': 'br_FR.ISO8859-15', + 'bs': 'bs_BA.ISO8859-2', + 'bs_ba': 'bs_BA.ISO8859-2', + 'bs_ba.iso88592': 'bs_BA.ISO8859-2', + 'bulgarian': 'bg_BG.CP1251', + 'c': 'C', + 'c-french': 'fr_CA.ISO8859-1', + 'c-french.iso88591': 'fr_CA.ISO8859-1', + 'c.en': 'C', + 'c.iso88591': 'en_US.ISO8859-1', + 'c_c': 'C', + 'c_c.c': 'C', + 'ca': 'ca_ES.ISO8859-1', + 'ca_ad': 'ca_AD.ISO8859-1', + 'ca_ad.iso88591': 'ca_AD.ISO8859-1', + 'ca_ad.iso885915': 'ca_AD.ISO8859-15', + 'ca_ad.iso885915 at euro': 'ca_AD.ISO8859-15', + 'ca_ad.utf8 at euro': 'ca_AD.UTF-8', + 'ca_ad at euro': 'ca_AD.ISO8859-15', + 'ca_es': 'ca_ES.ISO8859-1', + 'ca_es.iso88591': 'ca_ES.ISO8859-1', + 'ca_es.iso885915': 'ca_ES.ISO8859-15', + 'ca_es.iso885915 at euro': 'ca_ES.ISO8859-15', + 'ca_es.utf8 at euro': 'ca_ES.UTF-8', + 'ca_es at euro': 'ca_ES.ISO8859-15', + 'ca_fr': 'ca_FR.ISO8859-1', + 'ca_fr.iso88591': 'ca_FR.ISO8859-1', + 'ca_fr.iso885915': 'ca_FR.ISO8859-15', + 'ca_fr.iso885915 at euro': 'ca_FR.ISO8859-15', + 'ca_fr.utf8 at euro': 'ca_FR.UTF-8', + 'ca_fr at euro': 'ca_FR.ISO8859-15', + 'ca_it': 'ca_IT.ISO8859-1', + 'ca_it.iso88591': 'ca_IT.ISO8859-1', + 'ca_it.iso885915': 'ca_IT.ISO8859-15', + 'ca_it.iso885915 at euro': 'ca_IT.ISO8859-15', + 'ca_it.utf8 at euro': 'ca_IT.UTF-8', + 'ca_it at euro': 'ca_IT.ISO8859-15', + 'catalan': 'ca_ES.ISO8859-1', + 'cextend': 'en_US.ISO8859-1', + 'cextend.en': 'en_US.ISO8859-1', + 'chinese-s': 'zh_CN.eucCN', + 'chinese-t': 'zh_TW.eucTW', + 'croatian': 'hr_HR.ISO8859-2', + 'cs': 'cs_CZ.ISO8859-2', + 'cs_cs': 'cs_CZ.ISO8859-2', + 'cs_cs.iso88592': 'cs_CS.ISO8859-2', + 'cs_cz': 'cs_CZ.ISO8859-2', + 'cs_cz.iso88592': 'cs_CZ.ISO8859-2', + 'cy': 'cy_GB.ISO8859-1', + 'cy_gb': 'cy_GB.ISO8859-1', + 'cy_gb.iso88591': 'cy_GB.ISO8859-1', + 'cy_gb.iso885914': 'cy_GB.ISO8859-14', + 'cy_gb.iso885915': 'cy_GB.ISO8859-15', + 'cy_gb at euro': 'cy_GB.ISO8859-15', + 'cz': 'cs_CZ.ISO8859-2', + 'cz_cz': 'cs_CZ.ISO8859-2', + 'czech': 'cs_CZ.ISO8859-2', + 'da': 'da_DK.ISO8859-1', + 'da.iso885915': 'da_DK.ISO8859-15', + 'da_dk': 'da_DK.ISO8859-1', + 'da_dk.88591': 'da_DK.ISO8859-1', + 'da_dk.885915': 'da_DK.ISO8859-15', + 'da_dk.iso88591': 'da_DK.ISO8859-1', + 'da_dk.iso885915': 'da_DK.ISO8859-15', + 'da_dk at euro': 'da_DK.ISO8859-15', + 'danish': 'da_DK.ISO8859-1', + 'danish.iso88591': 'da_DK.ISO8859-1', + 'dansk': 'da_DK.ISO8859-1', + 'de': 'de_DE.ISO8859-1', + 'de.iso885915': 'de_DE.ISO8859-15', + 'de_at': 'de_AT.ISO8859-1', + 'de_at.iso88591': 'de_AT.ISO8859-1', + 'de_at.iso885915': 'de_AT.ISO8859-15', + 'de_at.iso885915 at euro': 'de_AT.ISO8859-15', + 'de_at.utf8 at euro': 'de_AT.UTF-8', + 'de_at at euro': 'de_AT.ISO8859-15', + 'de_be': 'de_BE.ISO8859-1', + 'de_be.iso88591': 'de_BE.ISO8859-1', + 'de_be.iso885915': 'de_BE.ISO8859-15', + 'de_be.iso885915 at euro': 'de_BE.ISO8859-15', + 'de_be.utf8 at euro': 'de_BE.UTF-8', + 'de_be at euro': 'de_BE.ISO8859-15', + 'de_ch': 'de_CH.ISO8859-1', + 'de_ch.iso88591': 'de_CH.ISO8859-1', + 'de_ch.iso885915': 'de_CH.ISO8859-15', + 'de_ch at euro': 'de_CH.ISO8859-15', + 'de_de': 'de_DE.ISO8859-1', + 'de_de.88591': 'de_DE.ISO8859-1', + 'de_de.885915': 'de_DE.ISO8859-15', + 'de_de.885915 at euro': 'de_DE.ISO8859-15', + 'de_de.iso88591': 'de_DE.ISO8859-1', + 'de_de.iso885915': 'de_DE.ISO8859-15', + 'de_de.iso885915 at euro': 'de_DE.ISO8859-15', + 'de_de.utf8 at euro': 'de_DE.UTF-8', + 'de_de at euro': 'de_DE.ISO8859-15', + 'de_lu': 'de_LU.ISO8859-1', + 'de_lu.iso88591': 'de_LU.ISO8859-1', + 'de_lu.iso885915': 'de_LU.ISO8859-15', + 'de_lu.iso885915 at euro': 'de_LU.ISO8859-15', + 'de_lu.utf8 at euro': 'de_LU.UTF-8', + 'de_lu at euro': 'de_LU.ISO8859-15', + 'deutsch': 'de_DE.ISO8859-1', + 'dutch': 'nl_NL.ISO8859-1', + 'dutch.iso88591': 'nl_BE.ISO8859-1', + 'ee': 'ee_EE.ISO8859-4', + 'ee_ee': 'ee_EE.ISO8859-4', + 'ee_ee.iso88594': 'ee_EE.ISO8859-4', + 'eesti': 'et_EE.ISO8859-1', + 'el': 'el_GR.ISO8859-7', + 'el_gr': 'el_GR.ISO8859-7', + 'el_gr.iso88597': 'el_GR.ISO8859-7', + 'el_gr at euro': 'el_GR.ISO8859-15', + 'en': 'en_US.ISO8859-1', + 'en.iso88591': 'en_US.ISO8859-1', + 'en_au': 'en_AU.ISO8859-1', + 'en_au.iso88591': 'en_AU.ISO8859-1', + 'en_be': 'en_BE.ISO8859-1', + 'en_be at euro': 'en_BE.ISO8859-15', + 'en_bw': 'en_BW.ISO8859-1', + 'en_bw.iso88591': 'en_BW.ISO8859-1', + 'en_ca': 'en_CA.ISO8859-1', + 'en_ca.iso88591': 'en_CA.ISO8859-1', + 'en_gb': 'en_GB.ISO8859-1', + 'en_gb.88591': 'en_GB.ISO8859-1', + 'en_gb.iso88591': 'en_GB.ISO8859-1', + 'en_gb.iso885915': 'en_GB.ISO8859-15', + 'en_gb at euro': 'en_GB.ISO8859-15', + 'en_hk': 'en_HK.ISO8859-1', + 'en_hk.iso88591': 'en_HK.ISO8859-1', + 'en_ie': 'en_IE.ISO8859-1', + 'en_ie.iso88591': 'en_IE.ISO8859-1', + 'en_ie.iso885915': 'en_IE.ISO8859-15', + 'en_ie.iso885915 at euro': 'en_IE.ISO8859-15', + 'en_ie.utf8 at euro': 'en_IE.UTF-8', + 'en_ie at euro': 'en_IE.ISO8859-15', + 'en_in': 'en_IN.ISO8859-1', + 'en_nz': 'en_NZ.ISO8859-1', + 'en_nz.iso88591': 'en_NZ.ISO8859-1', + 'en_ph': 'en_PH.ISO8859-1', + 'en_ph.iso88591': 'en_PH.ISO8859-1', + 'en_sg': 'en_SG.ISO8859-1', + 'en_sg.iso88591': 'en_SG.ISO8859-1', + 'en_uk': 'en_GB.ISO8859-1', + 'en_us': 'en_US.ISO8859-1', + 'en_us.88591': 'en_US.ISO8859-1', + 'en_us.885915': 'en_US.ISO8859-15', + 'en_us.iso88591': 'en_US.ISO8859-1', + 'en_us.iso885915': 'en_US.ISO8859-15', + 'en_us.iso885915 at euro': 'en_US.ISO8859-15', + 'en_us at euro': 'en_US.ISO8859-15', + 'en_us at euro@euro': 'en_US.ISO8859-15', + 'en_za': 'en_ZA.ISO8859-1', + 'en_za.88591': 'en_ZA.ISO8859-1', + 'en_za.iso88591': 'en_ZA.ISO8859-1', + 'en_za.iso885915': 'en_ZA.ISO8859-15', + 'en_za at euro': 'en_ZA.ISO8859-15', + 'en_zw': 'en_ZW.ISO8859-1', + 'en_zw.iso88591': 'en_ZW.ISO8859-1', + 'eng_gb': 'en_GB.ISO8859-1', + 'eng_gb.8859': 'en_GB.ISO8859-1', + 'english': 'en_EN.ISO8859-1', + 'english.iso88591': 'en_EN.ISO8859-1', + 'english_uk': 'en_GB.ISO8859-1', + 'english_uk.8859': 'en_GB.ISO8859-1', + 'english_united-states': 'en_US.ISO8859-1', + 'english_united-states.437': 'C', + 'english_us': 'en_US.ISO8859-1', + 'english_us.8859': 'en_US.ISO8859-1', + 'english_us.ascii': 'en_US.ISO8859-1', + 'eo': 'eo_XX.ISO8859-3', + 'eo_eo': 'eo_EO.ISO8859-3', + 'eo_eo.iso88593': 'eo_EO.ISO8859-3', + 'eo_xx': 'eo_XX.ISO8859-3', + 'eo_xx.iso88593': 'eo_XX.ISO8859-3', + 'es': 'es_ES.ISO8859-1', + 'es_ar': 'es_AR.ISO8859-1', + 'es_ar.iso88591': 'es_AR.ISO8859-1', + 'es_bo': 'es_BO.ISO8859-1', + 'es_bo.iso88591': 'es_BO.ISO8859-1', + 'es_cl': 'es_CL.ISO8859-1', + 'es_cl.iso88591': 'es_CL.ISO8859-1', + 'es_co': 'es_CO.ISO8859-1', + 'es_co.iso88591': 'es_CO.ISO8859-1', + 'es_cr': 'es_CR.ISO8859-1', + 'es_cr.iso88591': 'es_CR.ISO8859-1', + 'es_do': 'es_DO.ISO8859-1', + 'es_do.iso88591': 'es_DO.ISO8859-1', + 'es_ec': 'es_EC.ISO8859-1', + 'es_ec.iso88591': 'es_EC.ISO8859-1', + 'es_es': 'es_ES.ISO8859-1', + 'es_es.88591': 'es_ES.ISO8859-1', + 'es_es.iso88591': 'es_ES.ISO8859-1', + 'es_es.iso885915': 'es_ES.ISO8859-15', + 'es_es.iso885915 at euro': 'es_ES.ISO8859-15', + 'es_es.utf8 at euro': 'es_ES.UTF-8', + 'es_es at euro': 'es_ES.ISO8859-15', + 'es_gt': 'es_GT.ISO8859-1', + 'es_gt.iso88591': 'es_GT.ISO8859-1', + 'es_hn': 'es_HN.ISO8859-1', + 'es_hn.iso88591': 'es_HN.ISO8859-1', + 'es_mx': 'es_MX.ISO8859-1', + 'es_mx.iso88591': 'es_MX.ISO8859-1', + 'es_ni': 'es_NI.ISO8859-1', + 'es_ni.iso88591': 'es_NI.ISO8859-1', + 'es_pa': 'es_PA.ISO8859-1', + 'es_pa.iso88591': 'es_PA.ISO8859-1', + 'es_pa.iso885915': 'es_PA.ISO8859-15', + 'es_pa at euro': 'es_PA.ISO8859-15', + 'es_pe': 'es_PE.ISO8859-1', + 'es_pe.iso88591': 'es_PE.ISO8859-1', + 'es_pe.iso885915': 'es_PE.ISO8859-15', + 'es_pe at euro': 'es_PE.ISO8859-15', + 'es_pr': 'es_PR.ISO8859-1', + 'es_pr.iso88591': 'es_PR.ISO8859-1', + 'es_py': 'es_PY.ISO8859-1', + 'es_py.iso88591': 'es_PY.ISO8859-1', + 'es_py.iso885915': 'es_PY.ISO8859-15', + 'es_py at euro': 'es_PY.ISO8859-15', + 'es_sv': 'es_SV.ISO8859-1', + 'es_sv.iso88591': 'es_SV.ISO8859-1', + 'es_sv.iso885915': 'es_SV.ISO8859-15', + 'es_sv at euro': 'es_SV.ISO8859-15', + 'es_us': 'es_US.ISO8859-1', + 'es_us.iso88591': 'es_US.ISO8859-1', + 'es_uy': 'es_UY.ISO8859-1', + 'es_uy.iso88591': 'es_UY.ISO8859-1', + 'es_uy.iso885915': 'es_UY.ISO8859-15', + 'es_uy at euro': 'es_UY.ISO8859-15', + 'es_ve': 'es_VE.ISO8859-1', + 'es_ve.iso88591': 'es_VE.ISO8859-1', + 'es_ve.iso885915': 'es_VE.ISO8859-15', + 'es_ve at euro': 'es_VE.ISO8859-15', + 'estonian': 'et_EE.ISO8859-1', + 'et': 'et_EE.ISO8859-15', + 'et_ee': 'et_EE.ISO8859-15', + 'et_ee.iso88591': 'et_EE.ISO8859-1', + 'et_ee.iso885913': 'et_EE.ISO8859-13', + 'et_ee.iso885915': 'et_EE.ISO8859-15', + 'et_ee.iso88594': 'et_EE.ISO8859-4', + 'et_ee at euro': 'et_EE.ISO8859-15', + 'eu': 'eu_ES.ISO8859-1', + 'eu_es': 'eu_ES.ISO8859-1', + 'eu_es.iso88591': 'eu_ES.ISO8859-1', + 'eu_es.iso885915': 'eu_ES.ISO8859-15', + 'eu_es.iso885915 at euro': 'eu_ES.ISO8859-15', + 'eu_es.utf8 at euro': 'eu_ES.UTF-8', + 'eu_es at euro': 'eu_ES.ISO8859-15', + 'fa': 'fa_IR.UTF-8', + 'fa_ir': 'fa_IR.UTF-8', + 'fa_ir.isiri3342': 'fa_IR.ISIRI-3342', + 'fi': 'fi_FI.ISO8859-15', + 'fi.iso885915': 'fi_FI.ISO8859-15', + 'fi_fi': 'fi_FI.ISO8859-15', + 'fi_fi.88591': 'fi_FI.ISO8859-1', + 'fi_fi.iso88591': 'fi_FI.ISO8859-1', + 'fi_fi.iso885915': 'fi_FI.ISO8859-15', + 'fi_fi.iso885915 at euro': 'fi_FI.ISO8859-15', + 'fi_fi.utf8 at euro': 'fi_FI.UTF-8', + 'fi_fi at euro': 'fi_FI.ISO8859-15', + 'finnish': 'fi_FI.ISO8859-1', + 'finnish.iso88591': 'fi_FI.ISO8859-1', + 'fo': 'fo_FO.ISO8859-1', + 'fo_fo': 'fo_FO.ISO8859-1', + 'fo_fo.iso88591': 'fo_FO.ISO8859-1', + 'fo_fo.iso885915': 'fo_FO.ISO8859-15', + 'fo_fo at euro': 'fo_FO.ISO8859-15', + 'fr': 'fr_FR.ISO8859-1', + 'fr.iso885915': 'fr_FR.ISO8859-15', + 'fr_be': 'fr_BE.ISO8859-1', + 'fr_be.88591': 'fr_BE.ISO8859-1', + 'fr_be.iso88591': 'fr_BE.ISO8859-1', + 'fr_be.iso885915': 'fr_BE.ISO8859-15', + 'fr_be.iso885915 at euro': 'fr_BE.ISO8859-15', + 'fr_be.utf8 at euro': 'fr_BE.UTF-8', + 'fr_be at euro': 'fr_BE.ISO8859-15', + 'fr_ca': 'fr_CA.ISO8859-1', + 'fr_ca.88591': 'fr_CA.ISO8859-1', + 'fr_ca.iso88591': 'fr_CA.ISO8859-1', + 'fr_ca.iso885915': 'fr_CA.ISO8859-15', + 'fr_ca at euro': 'fr_CA.ISO8859-15', + 'fr_ch': 'fr_CH.ISO8859-1', + 'fr_ch.88591': 'fr_CH.ISO8859-1', + 'fr_ch.iso88591': 'fr_CH.ISO8859-1', + 'fr_ch.iso885915': 'fr_CH.ISO8859-15', + 'fr_ch at euro': 'fr_CH.ISO8859-15', + 'fr_fr': 'fr_FR.ISO8859-1', + 'fr_fr.88591': 'fr_FR.ISO8859-1', + 'fr_fr.iso88591': 'fr_FR.ISO8859-1', + 'fr_fr.iso885915': 'fr_FR.ISO8859-15', + 'fr_fr.iso885915 at euro': 'fr_FR.ISO8859-15', + 'fr_fr.utf8 at euro': 'fr_FR.UTF-8', + 'fr_fr at euro': 'fr_FR.ISO8859-15', + 'fr_lu': 'fr_LU.ISO8859-1', + 'fr_lu.88591': 'fr_LU.ISO8859-1', + 'fr_lu.iso88591': 'fr_LU.ISO8859-1', + 'fr_lu.iso885915': 'fr_LU.ISO8859-15', + 'fr_lu.iso885915 at euro': 'fr_LU.ISO8859-15', + 'fr_lu.utf8 at euro': 'fr_LU.UTF-8', + 'fr_lu at euro': 'fr_LU.ISO8859-15', + 'fran\xe7ais': 'fr_FR.ISO8859-1', + 'fre_fr': 'fr_FR.ISO8859-1', + 'fre_fr.8859': 'fr_FR.ISO8859-1', + 'french': 'fr_FR.ISO8859-1', + 'french.iso88591': 'fr_CH.ISO8859-1', + 'french_france': 'fr_FR.ISO8859-1', + 'french_france.8859': 'fr_FR.ISO8859-1', + 'ga': 'ga_IE.ISO8859-1', + 'ga_ie': 'ga_IE.ISO8859-1', + 'ga_ie.iso88591': 'ga_IE.ISO8859-1', + 'ga_ie.iso885914': 'ga_IE.ISO8859-14', + 'ga_ie.iso885915': 'ga_IE.ISO8859-15', + 'ga_ie.iso885915 at euro': 'ga_IE.ISO8859-15', + 'ga_ie.utf8 at euro': 'ga_IE.UTF-8', + 'ga_ie at euro': 'ga_IE.ISO8859-15', + 'galego': 'gl_ES.ISO8859-1', + 'galician': 'gl_ES.ISO8859-1', + 'gd': 'gd_GB.ISO8859-1', + 'gd_gb': 'gd_GB.ISO8859-1', + 'gd_gb.iso88591': 'gd_GB.ISO8859-1', + 'gd_gb.iso885914': 'gd_GB.ISO8859-14', + 'gd_gb.iso885915': 'gd_GB.ISO8859-15', + 'gd_gb at euro': 'gd_GB.ISO8859-15', + 'ger_de': 'de_DE.ISO8859-1', + 'ger_de.8859': 'de_DE.ISO8859-1', + 'german': 'de_DE.ISO8859-1', + 'german.iso88591': 'de_CH.ISO8859-1', + 'german_germany': 'de_DE.ISO8859-1', + 'german_germany.8859': 'de_DE.ISO8859-1', + 'gl': 'gl_ES.ISO8859-1', + 'gl_es': 'gl_ES.ISO8859-1', + 'gl_es.iso88591': 'gl_ES.ISO8859-1', + 'gl_es.iso885915': 'gl_ES.ISO8859-15', + 'gl_es.iso885915 at euro': 'gl_ES.ISO8859-15', + 'gl_es.utf8 at euro': 'gl_ES.UTF-8', + 'gl_es at euro': 'gl_ES.ISO8859-15', + 'greek': 'el_GR.ISO8859-7', + 'greek.iso88597': 'el_GR.ISO8859-7', + 'gu_in': 'gu_IN.UTF-8', + 'gv': 'gv_GB.ISO8859-1', + 'gv_gb': 'gv_GB.ISO8859-1', + 'gv_gb.iso88591': 'gv_GB.ISO8859-1', + 'gv_gb.iso885914': 'gv_GB.ISO8859-14', + 'gv_gb.iso885915': 'gv_GB.ISO8859-15', + 'gv_gb at euro': 'gv_GB.ISO8859-15', + 'he': 'he_IL.ISO8859-8', + 'he_il': 'he_IL.ISO8859-8', + 'he_il.cp1255': 'he_IL.CP1255', + 'he_il.iso88598': 'he_IL.ISO8859-8', + 'he_il.microsoftcp1255': 'he_IL.CP1255', + 'hebrew': 'iw_IL.ISO8859-8', + 'hebrew.iso88598': 'iw_IL.ISO8859-8', + 'hi': 'hi_IN.ISCII-DEV', + 'hi_in': 'hi_IN.ISCII-DEV', + 'hi_in.isciidev': 'hi_IN.ISCII-DEV', + 'hne': 'hne_IN.UTF-8', + 'hr': 'hr_HR.ISO8859-2', + 'hr_hr': 'hr_HR.ISO8859-2', + 'hr_hr.iso88592': 'hr_HR.ISO8859-2', + 'hrvatski': 'hr_HR.ISO8859-2', + 'hu': 'hu_HU.ISO8859-2', + 'hu_hu': 'hu_HU.ISO8859-2', + 'hu_hu.iso88592': 'hu_HU.ISO8859-2', + 'hungarian': 'hu_HU.ISO8859-2', + 'icelandic': 'is_IS.ISO8859-1', + 'icelandic.iso88591': 'is_IS.ISO8859-1', + 'id': 'id_ID.ISO8859-1', + 'id_id': 'id_ID.ISO8859-1', + 'in': 'id_ID.ISO8859-1', + 'in_id': 'id_ID.ISO8859-1', + 'is': 'is_IS.ISO8859-1', + 'is_is': 'is_IS.ISO8859-1', + 'is_is.iso88591': 'is_IS.ISO8859-1', + 'is_is.iso885915': 'is_IS.ISO8859-15', + 'is_is at euro': 'is_IS.ISO8859-15', + 'iso-8859-1': 'en_US.ISO8859-1', + 'iso-8859-15': 'en_US.ISO8859-15', + 'iso8859-1': 'en_US.ISO8859-1', + 'iso8859-15': 'en_US.ISO8859-15', + 'iso_8859_1': 'en_US.ISO8859-1', + 'iso_8859_15': 'en_US.ISO8859-15', + 'it': 'it_IT.ISO8859-1', + 'it.iso885915': 'it_IT.ISO8859-15', + 'it_ch': 'it_CH.ISO8859-1', + 'it_ch.iso88591': 'it_CH.ISO8859-1', + 'it_ch.iso885915': 'it_CH.ISO8859-15', + 'it_ch at euro': 'it_CH.ISO8859-15', + 'it_it': 'it_IT.ISO8859-1', + 'it_it.88591': 'it_IT.ISO8859-1', + 'it_it.iso88591': 'it_IT.ISO8859-1', + 'it_it.iso885915': 'it_IT.ISO8859-15', + 'it_it.iso885915 at euro': 'it_IT.ISO8859-15', + 'it_it.utf8 at euro': 'it_IT.UTF-8', + 'it_it at euro': 'it_IT.ISO8859-15', + 'italian': 'it_IT.ISO8859-1', + 'italian.iso88591': 'it_IT.ISO8859-1', + 'iu': 'iu_CA.NUNACOM-8', + 'iu_ca': 'iu_CA.NUNACOM-8', + 'iu_ca.nunacom8': 'iu_CA.NUNACOM-8', + 'iw': 'he_IL.ISO8859-8', + 'iw_il': 'he_IL.ISO8859-8', + 'iw_il.iso88598': 'he_IL.ISO8859-8', + 'ja': 'ja_JP.eucJP', + 'ja.jis': 'ja_JP.JIS7', + 'ja.sjis': 'ja_JP.SJIS', + 'ja_jp': 'ja_JP.eucJP', + 'ja_jp.ajec': 'ja_JP.eucJP', + 'ja_jp.euc': 'ja_JP.eucJP', + 'ja_jp.eucjp': 'ja_JP.eucJP', + 'ja_jp.iso-2022-jp': 'ja_JP.JIS7', + 'ja_jp.iso2022jp': 'ja_JP.JIS7', + 'ja_jp.jis': 'ja_JP.JIS7', + 'ja_jp.jis7': 'ja_JP.JIS7', + 'ja_jp.mscode': 'ja_JP.SJIS', + 'ja_jp.pck': 'ja_JP.SJIS', + 'ja_jp.sjis': 'ja_JP.SJIS', + 'ja_jp.ujis': 'ja_JP.eucJP', + 'japan': 'ja_JP.eucJP', + 'japanese': 'ja_JP.eucJP', + 'japanese-euc': 'ja_JP.eucJP', + 'japanese.euc': 'ja_JP.eucJP', + 'japanese.sjis': 'ja_JP.SJIS', + 'jp_jp': 'ja_JP.eucJP', + 'ka': 'ka_GE.GEORGIAN-ACADEMY', + 'ka_ge': 'ka_GE.GEORGIAN-ACADEMY', + 'ka_ge.georgianacademy': 'ka_GE.GEORGIAN-ACADEMY', + 'ka_ge.georgianps': 'ka_GE.GEORGIAN-PS', + 'ka_ge.georgianrs': 'ka_GE.GEORGIAN-ACADEMY', + 'kl': 'kl_GL.ISO8859-1', + 'kl_gl': 'kl_GL.ISO8859-1', + 'kl_gl.iso88591': 'kl_GL.ISO8859-1', + 'kl_gl.iso885915': 'kl_GL.ISO8859-15', + 'kl_gl at euro': 'kl_GL.ISO8859-15', + 'km_kh': 'km_KH.UTF-8', + 'kn': 'kn_IN.UTF-8', + 'kn_in': 'kn_IN.UTF-8', + 'ko': 'ko_KR.eucKR', + 'ko_kr': 'ko_KR.eucKR', + 'ko_kr.euc': 'ko_KR.eucKR', + 'ko_kr.euckr': 'ko_KR.eucKR', + 'korean': 'ko_KR.eucKR', + 'korean.euc': 'ko_KR.eucKR', + 'ks': 'ks_IN.UTF-8', + 'ks_in at devanagari': 'ks_IN at devanagari.UTF-8', + 'kw': 'kw_GB.ISO8859-1', + 'kw_gb': 'kw_GB.ISO8859-1', + 'kw_gb.iso88591': 'kw_GB.ISO8859-1', + 'kw_gb.iso885914': 'kw_GB.ISO8859-14', + 'kw_gb.iso885915': 'kw_GB.ISO8859-15', + 'kw_gb at euro': 'kw_GB.ISO8859-15', + 'ky': 'ky_KG.UTF-8', + 'ky_kg': 'ky_KG.UTF-8', + 'lithuanian': 'lt_LT.ISO8859-13', + 'lo': 'lo_LA.MULELAO-1', + 'lo_la': 'lo_LA.MULELAO-1', + 'lo_la.cp1133': 'lo_LA.IBM-CP1133', + 'lo_la.ibmcp1133': 'lo_LA.IBM-CP1133', + 'lo_la.mulelao1': 'lo_LA.MULELAO-1', + 'lt': 'lt_LT.ISO8859-13', + 'lt_lt': 'lt_LT.ISO8859-13', + 'lt_lt.iso885913': 'lt_LT.ISO8859-13', + 'lt_lt.iso88594': 'lt_LT.ISO8859-4', + 'lv': 'lv_LV.ISO8859-13', + 'lv_lv': 'lv_LV.ISO8859-13', + 'lv_lv.iso885913': 'lv_LV.ISO8859-13', + 'lv_lv.iso88594': 'lv_LV.ISO8859-4', + 'mai': 'mai_IN.UTF-8', + 'mi': 'mi_NZ.ISO8859-1', + 'mi_nz': 'mi_NZ.ISO8859-1', + 'mi_nz.iso88591': 'mi_NZ.ISO8859-1', + 'mk': 'mk_MK.ISO8859-5', + 'mk_mk': 'mk_MK.ISO8859-5', + 'mk_mk.cp1251': 'mk_MK.CP1251', + 'mk_mk.iso88595': 'mk_MK.ISO8859-5', + 'mk_mk.microsoftcp1251': 'mk_MK.CP1251', + 'ml': 'ml_IN.UTF-8', + 'mr': 'mr_IN.UTF-8', + 'mr_in': 'mr_IN.UTF-8', + 'ms': 'ms_MY.ISO8859-1', + 'ms_my': 'ms_MY.ISO8859-1', + 'ms_my.iso88591': 'ms_MY.ISO8859-1', + 'mt': 'mt_MT.ISO8859-3', + 'mt_mt': 'mt_MT.ISO8859-3', + 'mt_mt.iso88593': 'mt_MT.ISO8859-3', + 'nb': 'nb_NO.ISO8859-1', + 'nb_no': 'nb_NO.ISO8859-1', + 'nb_no.88591': 'nb_NO.ISO8859-1', + 'nb_no.iso88591': 'nb_NO.ISO8859-1', + 'nb_no.iso885915': 'nb_NO.ISO8859-15', + 'nb_no at euro': 'nb_NO.ISO8859-15', + 'nl': 'nl_NL.ISO8859-1', + 'nl.iso885915': 'nl_NL.ISO8859-15', + 'nl_be': 'nl_BE.ISO8859-1', + 'nl_be.88591': 'nl_BE.ISO8859-1', + 'nl_be.iso88591': 'nl_BE.ISO8859-1', + 'nl_be.iso885915': 'nl_BE.ISO8859-15', + 'nl_be.iso885915 at euro': 'nl_BE.ISO8859-15', + 'nl_be.utf8 at euro': 'nl_BE.UTF-8', + 'nl_be at euro': 'nl_BE.ISO8859-15', + 'nl_nl': 'nl_NL.ISO8859-1', + 'nl_nl.88591': 'nl_NL.ISO8859-1', + 'nl_nl.iso88591': 'nl_NL.ISO8859-1', + 'nl_nl.iso885915': 'nl_NL.ISO8859-15', + 'nl_nl.iso885915 at euro': 'nl_NL.ISO8859-15', + 'nl_nl.utf8 at euro': 'nl_NL.UTF-8', + 'nl_nl at euro': 'nl_NL.ISO8859-15', + 'nn': 'nn_NO.ISO8859-1', + 'nn_no': 'nn_NO.ISO8859-1', + 'nn_no.88591': 'nn_NO.ISO8859-1', + 'nn_no.iso88591': 'nn_NO.ISO8859-1', + 'nn_no.iso885915': 'nn_NO.ISO8859-15', + 'nn_no at euro': 'nn_NO.ISO8859-15', + 'no': 'no_NO.ISO8859-1', + 'no at nynorsk': 'ny_NO.ISO8859-1', + 'no_no': 'no_NO.ISO8859-1', + 'no_no.88591': 'no_NO.ISO8859-1', + 'no_no.iso88591': 'no_NO.ISO8859-1', + 'no_no.iso885915': 'no_NO.ISO8859-15', + 'no_no.iso88591 at bokmal': 'no_NO.ISO8859-1', + 'no_no.iso88591 at nynorsk': 'no_NO.ISO8859-1', + 'no_no at euro': 'no_NO.ISO8859-15', + 'norwegian': 'no_NO.ISO8859-1', + 'norwegian.iso88591': 'no_NO.ISO8859-1', + 'nr': 'nr_ZA.ISO8859-1', + 'nr_za': 'nr_ZA.ISO8859-1', + 'nr_za.iso88591': 'nr_ZA.ISO8859-1', + 'nso': 'nso_ZA.ISO8859-15', + 'nso_za': 'nso_ZA.ISO8859-15', + 'nso_za.iso885915': 'nso_ZA.ISO8859-15', + 'ny': 'ny_NO.ISO8859-1', + 'ny_no': 'ny_NO.ISO8859-1', + 'ny_no.88591': 'ny_NO.ISO8859-1', + 'ny_no.iso88591': 'ny_NO.ISO8859-1', + 'ny_no.iso885915': 'ny_NO.ISO8859-15', + 'ny_no at euro': 'ny_NO.ISO8859-15', + 'nynorsk': 'nn_NO.ISO8859-1', + 'oc': 'oc_FR.ISO8859-1', + 'oc_fr': 'oc_FR.ISO8859-1', + 'oc_fr.iso88591': 'oc_FR.ISO8859-1', + 'oc_fr.iso885915': 'oc_FR.ISO8859-15', + 'oc_fr at euro': 'oc_FR.ISO8859-15', + 'or': 'or_IN.UTF-8', + 'pa': 'pa_IN.UTF-8', + 'pa_in': 'pa_IN.UTF-8', + 'pd': 'pd_US.ISO8859-1', + 'pd_de': 'pd_DE.ISO8859-1', + 'pd_de.iso88591': 'pd_DE.ISO8859-1', + 'pd_de.iso885915': 'pd_DE.ISO8859-15', + 'pd_de at euro': 'pd_DE.ISO8859-15', + 'pd_us': 'pd_US.ISO8859-1', + 'pd_us.iso88591': 'pd_US.ISO8859-1', + 'pd_us.iso885915': 'pd_US.ISO8859-15', + 'pd_us at euro': 'pd_US.ISO8859-15', + 'ph': 'ph_PH.ISO8859-1', + 'ph_ph': 'ph_PH.ISO8859-1', + 'ph_ph.iso88591': 'ph_PH.ISO8859-1', + 'pl': 'pl_PL.ISO8859-2', + 'pl_pl': 'pl_PL.ISO8859-2', + 'pl_pl.iso88592': 'pl_PL.ISO8859-2', + 'polish': 'pl_PL.ISO8859-2', + 'portuguese': 'pt_PT.ISO8859-1', + 'portuguese.iso88591': 'pt_PT.ISO8859-1', + 'portuguese_brazil': 'pt_BR.ISO8859-1', + 'portuguese_brazil.8859': 'pt_BR.ISO8859-1', + 'posix': 'C', + 'posix-utf2': 'C', + 'pp': 'pp_AN.ISO8859-1', + 'pp_an': 'pp_AN.ISO8859-1', + 'pp_an.iso88591': 'pp_AN.ISO8859-1', + 'pt': 'pt_PT.ISO8859-1', + 'pt.iso885915': 'pt_PT.ISO8859-15', + 'pt_br': 'pt_BR.ISO8859-1', + 'pt_br.88591': 'pt_BR.ISO8859-1', + 'pt_br.iso88591': 'pt_BR.ISO8859-1', + 'pt_br.iso885915': 'pt_BR.ISO8859-15', + 'pt_br at euro': 'pt_BR.ISO8859-15', + 'pt_pt': 'pt_PT.ISO8859-1', + 'pt_pt.88591': 'pt_PT.ISO8859-1', + 'pt_pt.iso88591': 'pt_PT.ISO8859-1', + 'pt_pt.iso885915': 'pt_PT.ISO8859-15', + 'pt_pt.iso885915 at euro': 'pt_PT.ISO8859-15', + 'pt_pt.utf8 at euro': 'pt_PT.UTF-8', + 'pt_pt at euro': 'pt_PT.ISO8859-15', + 'ro': 'ro_RO.ISO8859-2', + 'ro_ro': 'ro_RO.ISO8859-2', + 'ro_ro.iso88592': 'ro_RO.ISO8859-2', + 'romanian': 'ro_RO.ISO8859-2', + 'ru': 'ru_RU.UTF-8', + 'ru.koi8r': 'ru_RU.KOI8-R', + 'ru_ru': 'ru_RU.UTF-8', + 'ru_ru.cp1251': 'ru_RU.CP1251', + 'ru_ru.iso88595': 'ru_RU.ISO8859-5', + 'ru_ru.koi8r': 'ru_RU.KOI8-R', + 'ru_ru.microsoftcp1251': 'ru_RU.CP1251', + 'ru_ua': 'ru_UA.KOI8-U', + 'ru_ua.cp1251': 'ru_UA.CP1251', + 'ru_ua.koi8u': 'ru_UA.KOI8-U', + 'ru_ua.microsoftcp1251': 'ru_UA.CP1251', + 'rumanian': 'ro_RO.ISO8859-2', + 'russian': 'ru_RU.ISO8859-5', + 'rw': 'rw_RW.ISO8859-1', + 'rw_rw': 'rw_RW.ISO8859-1', + 'rw_rw.iso88591': 'rw_RW.ISO8859-1', + 'sd': 'sd_IN at devanagari.UTF-8', + 'se_no': 'se_NO.UTF-8', + 'serbocroatian': 'sr_RS.UTF-8 at latin', + 'sh': 'sr_RS.UTF-8 at latin', + 'sh_ba.iso88592 at bosnia': 'sr_CS.ISO8859-2', + 'sh_hr': 'sh_HR.ISO8859-2', + 'sh_hr.iso88592': 'hr_HR.ISO8859-2', + 'sh_sp': 'sr_CS.ISO8859-2', + 'sh_yu': 'sr_RS.UTF-8 at latin', + 'si': 'si_LK.UTF-8', + 'si_lk': 'si_LK.UTF-8', + 'sinhala': 'si_LK.UTF-8', + 'sk': 'sk_SK.ISO8859-2', + 'sk_sk': 'sk_SK.ISO8859-2', + 'sk_sk.iso88592': 'sk_SK.ISO8859-2', + 'sl': 'sl_SI.ISO8859-2', + 'sl_cs': 'sl_CS.ISO8859-2', + 'sl_si': 'sl_SI.ISO8859-2', + 'sl_si.iso88592': 'sl_SI.ISO8859-2', + 'slovak': 'sk_SK.ISO8859-2', + 'slovene': 'sl_SI.ISO8859-2', + 'slovenian': 'sl_SI.ISO8859-2', + 'sp': 'sr_CS.ISO8859-5', + 'sp_yu': 'sr_CS.ISO8859-5', + 'spanish': 'es_ES.ISO8859-1', + 'spanish.iso88591': 'es_ES.ISO8859-1', + 'spanish_spain': 'es_ES.ISO8859-1', + 'spanish_spain.8859': 'es_ES.ISO8859-1', + 'sq': 'sq_AL.ISO8859-2', + 'sq_al': 'sq_AL.ISO8859-2', + 'sq_al.iso88592': 'sq_AL.ISO8859-2', + 'sr': 'sr_RS.UTF-8', + 'sr at cyrillic': 'sr_RS.UTF-8', + 'sr at latin': 'sr_RS.UTF-8 at latin', + 'sr at latn': 'sr_RS.UTF-8 at latin', + 'sr_cs': 'sr_RS.UTF-8', + 'sr_cs.iso88592': 'sr_CS.ISO8859-2', + 'sr_cs.iso88592 at latn': 'sr_CS.ISO8859-2', + 'sr_cs.iso88595': 'sr_CS.ISO8859-5', + 'sr_cs.utf8 at latn': 'sr_RS.UTF-8 at latin', + 'sr_cs at latn': 'sr_RS.UTF-8 at latin', + 'sr_me': 'sr_ME.UTF-8', + 'sr_rs': 'sr_RS.UTF-8', + 'sr_rs.utf8 at latn': 'sr_RS.UTF-8 at latin', + 'sr_rs at latin': 'sr_RS.UTF-8 at latin', + 'sr_rs at latn': 'sr_RS.UTF-8 at latin', + 'sr_sp': 'sr_CS.ISO8859-2', + 'sr_yu': 'sr_RS.UTF-8 at latin', + 'sr_yu.cp1251 at cyrillic': 'sr_CS.CP1251', + 'sr_yu.iso88592': 'sr_CS.ISO8859-2', + 'sr_yu.iso88595': 'sr_CS.ISO8859-5', + 'sr_yu.iso88595 at cyrillic': 'sr_CS.ISO8859-5', + 'sr_yu.microsoftcp1251 at cyrillic': 'sr_CS.CP1251', + 'sr_yu.utf8 at cyrillic': 'sr_RS.UTF-8', + 'sr_yu at cyrillic': 'sr_RS.UTF-8', + 'ss': 'ss_ZA.ISO8859-1', + 'ss_za': 'ss_ZA.ISO8859-1', + 'ss_za.iso88591': 'ss_ZA.ISO8859-1', + 'st': 'st_ZA.ISO8859-1', + 'st_za': 'st_ZA.ISO8859-1', + 'st_za.iso88591': 'st_ZA.ISO8859-1', + 'sv': 'sv_SE.ISO8859-1', + 'sv.iso885915': 'sv_SE.ISO8859-15', + 'sv_fi': 'sv_FI.ISO8859-1', + 'sv_fi.iso88591': 'sv_FI.ISO8859-1', + 'sv_fi.iso885915': 'sv_FI.ISO8859-15', + 'sv_fi.iso885915 at euro': 'sv_FI.ISO8859-15', + 'sv_fi.utf8 at euro': 'sv_FI.UTF-8', + 'sv_fi at euro': 'sv_FI.ISO8859-15', + 'sv_se': 'sv_SE.ISO8859-1', + 'sv_se.88591': 'sv_SE.ISO8859-1', + 'sv_se.iso88591': 'sv_SE.ISO8859-1', + 'sv_se.iso885915': 'sv_SE.ISO8859-15', + 'sv_se at euro': 'sv_SE.ISO8859-15', + 'swedish': 'sv_SE.ISO8859-1', + 'swedish.iso88591': 'sv_SE.ISO8859-1', + 'ta': 'ta_IN.TSCII-0', + 'ta_in': 'ta_IN.TSCII-0', + 'ta_in.tscii': 'ta_IN.TSCII-0', + 'ta_in.tscii0': 'ta_IN.TSCII-0', + 'te': 'te_IN.UTF-8', + 'tg': 'tg_TJ.KOI8-C', + 'tg_tj': 'tg_TJ.KOI8-C', + 'tg_tj.koi8c': 'tg_TJ.KOI8-C', + 'th': 'th_TH.ISO8859-11', + 'th_th': 'th_TH.ISO8859-11', + 'th_th.iso885911': 'th_TH.ISO8859-11', + 'th_th.tactis': 'th_TH.TIS620', + 'th_th.tis620': 'th_TH.TIS620', + 'thai': 'th_TH.ISO8859-11', + 'tl': 'tl_PH.ISO8859-1', + 'tl_ph': 'tl_PH.ISO8859-1', + 'tl_ph.iso88591': 'tl_PH.ISO8859-1', + 'tn': 'tn_ZA.ISO8859-15', + 'tn_za': 'tn_ZA.ISO8859-15', + 'tn_za.iso885915': 'tn_ZA.ISO8859-15', + 'tr': 'tr_TR.ISO8859-9', + 'tr_tr': 'tr_TR.ISO8859-9', + 'tr_tr.iso88599': 'tr_TR.ISO8859-9', + 'ts': 'ts_ZA.ISO8859-1', + 'ts_za': 'ts_ZA.ISO8859-1', + 'ts_za.iso88591': 'ts_ZA.ISO8859-1', + 'tt': 'tt_RU.TATAR-CYR', + 'tt_ru': 'tt_RU.TATAR-CYR', + 'tt_ru.koi8c': 'tt_RU.KOI8-C', + 'tt_ru.tatarcyr': 'tt_RU.TATAR-CYR', + 'turkish': 'tr_TR.ISO8859-9', + 'turkish.iso88599': 'tr_TR.ISO8859-9', + 'uk': 'uk_UA.KOI8-U', + 'uk_ua': 'uk_UA.KOI8-U', + 'uk_ua.cp1251': 'uk_UA.CP1251', + 'uk_ua.iso88595': 'uk_UA.ISO8859-5', + 'uk_ua.koi8u': 'uk_UA.KOI8-U', + 'uk_ua.microsoftcp1251': 'uk_UA.CP1251', + 'univ': 'en_US.utf', + 'universal': 'en_US.utf', + 'universal.utf8 at ucs4': 'en_US.UTF-8', + 'ur': 'ur_PK.CP1256', + 'ur_pk': 'ur_PK.CP1256', + 'ur_pk.cp1256': 'ur_PK.CP1256', + 'ur_pk.microsoftcp1256': 'ur_PK.CP1256', + 'uz': 'uz_UZ.UTF-8', + 'uz_uz': 'uz_UZ.UTF-8', + 'uz_uz.iso88591': 'uz_UZ.ISO8859-1', + 'uz_uz.utf8 at cyrillic': 'uz_UZ.UTF-8', + 'uz_uz at cyrillic': 'uz_UZ.UTF-8', + 've': 've_ZA.UTF-8', + 've_za': 've_ZA.UTF-8', + 'vi': 'vi_VN.TCVN', + 'vi_vn': 'vi_VN.TCVN', + 'vi_vn.tcvn': 'vi_VN.TCVN', + 'vi_vn.tcvn5712': 'vi_VN.TCVN', + 'vi_vn.viscii': 'vi_VN.VISCII', + 'vi_vn.viscii111': 'vi_VN.VISCII', + 'wa': 'wa_BE.ISO8859-1', + 'wa_be': 'wa_BE.ISO8859-1', + 'wa_be.iso88591': 'wa_BE.ISO8859-1', + 'wa_be.iso885915': 'wa_BE.ISO8859-15', + 'wa_be.iso885915 at euro': 'wa_BE.ISO8859-15', + 'wa_be at euro': 'wa_BE.ISO8859-15', + 'xh': 'xh_ZA.ISO8859-1', + 'xh_za': 'xh_ZA.ISO8859-1', + 'xh_za.iso88591': 'xh_ZA.ISO8859-1', + 'yi': 'yi_US.CP1255', + 'yi_us': 'yi_US.CP1255', + 'yi_us.cp1255': 'yi_US.CP1255', + 'yi_us.microsoftcp1255': 'yi_US.CP1255', + 'zh': 'zh_CN.eucCN', + 'zh_cn': 'zh_CN.gb2312', + 'zh_cn.big5': 'zh_TW.big5', + 'zh_cn.euc': 'zh_CN.eucCN', + 'zh_cn.gb18030': 'zh_CN.gb18030', + 'zh_cn.gb2312': 'zh_CN.gb2312', + 'zh_cn.gbk': 'zh_CN.gbk', + 'zh_hk': 'zh_HK.big5hkscs', + 'zh_hk.big5': 'zh_HK.big5', + 'zh_hk.big5hk': 'zh_HK.big5hkscs', + 'zh_hk.big5hkscs': 'zh_HK.big5hkscs', + 'zh_tw': 'zh_TW.big5', + 'zh_tw.big5': 'zh_TW.big5', + 'zh_tw.euc': 'zh_TW.eucTW', + 'zh_tw.euctw': 'zh_TW.eucTW', + 'zu': 'zu_ZA.ISO8859-1', + 'zu_za': 'zu_ZA.ISO8859-1', + 'zu_za.iso88591': 'zu_ZA.ISO8859-1', +} + +# +# This maps Windows language identifiers to locale strings. +# +# This list has been updated from +# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp +# to include every locale up to Windows Vista. +# +# NOTE: this mapping is incomplete. If your language is missing, please +# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# Make sure you include the missing language identifier and the suggested +# locale code. +# + +windows_locale = { + 0x0436: "af_ZA", # Afrikaans + 0x041c: "sq_AL", # Albanian + 0x0484: "gsw_FR",# Alsatian - France + 0x045e: "am_ET", # Amharic - Ethiopia + 0x0401: "ar_SA", # Arabic - Saudi Arabia + 0x0801: "ar_IQ", # Arabic - Iraq + 0x0c01: "ar_EG", # Arabic - Egypt + 0x1001: "ar_LY", # Arabic - Libya + 0x1401: "ar_DZ", # Arabic - Algeria + 0x1801: "ar_MA", # Arabic - Morocco + 0x1c01: "ar_TN", # Arabic - Tunisia + 0x2001: "ar_OM", # Arabic - Oman + 0x2401: "ar_YE", # Arabic - Yemen + 0x2801: "ar_SY", # Arabic - Syria + 0x2c01: "ar_JO", # Arabic - Jordan + 0x3001: "ar_LB", # Arabic - Lebanon + 0x3401: "ar_KW", # Arabic - Kuwait + 0x3801: "ar_AE", # Arabic - United Arab Emirates + 0x3c01: "ar_BH", # Arabic - Bahrain + 0x4001: "ar_QA", # Arabic - Qatar + 0x042b: "hy_AM", # Armenian + 0x044d: "as_IN", # Assamese - India + 0x042c: "az_AZ", # Azeri - Latin + 0x082c: "az_AZ", # Azeri - Cyrillic + 0x046d: "ba_RU", # Bashkir + 0x042d: "eu_ES", # Basque - Russia + 0x0423: "be_BY", # Belarusian + 0x0445: "bn_IN", # Begali + 0x201a: "bs_BA", # Bosnian - Cyrillic + 0x141a: "bs_BA", # Bosnian - Latin + 0x047e: "br_FR", # Breton - France + 0x0402: "bg_BG", # Bulgarian +# 0x0455: "my_MM", # Burmese - Not supported + 0x0403: "ca_ES", # Catalan + 0x0004: "zh_CHS",# Chinese - Simplified + 0x0404: "zh_TW", # Chinese - Taiwan + 0x0804: "zh_CN", # Chinese - PRC + 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. + 0x1004: "zh_SG", # Chinese - Singapore + 0x1404: "zh_MO", # Chinese - Macao S.A.R. + 0x7c04: "zh_CHT",# Chinese - Traditional + 0x0483: "co_FR", # Corsican - France + 0x041a: "hr_HR", # Croatian + 0x101a: "hr_BA", # Croatian - Bosnia + 0x0405: "cs_CZ", # Czech + 0x0406: "da_DK", # Danish + 0x048c: "gbz_AF",# Dari - Afghanistan + 0x0465: "div_MV",# Divehi - Maldives + 0x0413: "nl_NL", # Dutch - The Netherlands + 0x0813: "nl_BE", # Dutch - Belgium + 0x0409: "en_US", # English - United States + 0x0809: "en_GB", # English - United Kingdom + 0x0c09: "en_AU", # English - Australia + 0x1009: "en_CA", # English - Canada + 0x1409: "en_NZ", # English - New Zealand + 0x1809: "en_IE", # English - Ireland + 0x1c09: "en_ZA", # English - South Africa + 0x2009: "en_JA", # English - Jamaica + 0x2409: "en_CB", # English - Carribbean + 0x2809: "en_BZ", # English - Belize + 0x2c09: "en_TT", # English - Trinidad + 0x3009: "en_ZW", # English - Zimbabwe + 0x3409: "en_PH", # English - Philippines + 0x4009: "en_IN", # English - India + 0x4409: "en_MY", # English - Malaysia + 0x4809: "en_IN", # English - Singapore + 0x0425: "et_EE", # Estonian + 0x0438: "fo_FO", # Faroese + 0x0464: "fil_PH",# Filipino + 0x040b: "fi_FI", # Finnish + 0x040c: "fr_FR", # French - France + 0x080c: "fr_BE", # French - Belgium + 0x0c0c: "fr_CA", # French - Canada + 0x100c: "fr_CH", # French - Switzerland + 0x140c: "fr_LU", # French - Luxembourg + 0x180c: "fr_MC", # French - Monaco + 0x0462: "fy_NL", # Frisian - Netherlands + 0x0456: "gl_ES", # Galician + 0x0437: "ka_GE", # Georgian + 0x0407: "de_DE", # German - Germany + 0x0807: "de_CH", # German - Switzerland + 0x0c07: "de_AT", # German - Austria + 0x1007: "de_LU", # German - Luxembourg + 0x1407: "de_LI", # German - Liechtenstein + 0x0408: "el_GR", # Greek + 0x046f: "kl_GL", # Greenlandic - Greenland + 0x0447: "gu_IN", # Gujarati + 0x0468: "ha_NG", # Hausa - Latin + 0x040d: "he_IL", # Hebrew + 0x0439: "hi_IN", # Hindi + 0x040e: "hu_HU", # Hungarian + 0x040f: "is_IS", # Icelandic + 0x0421: "id_ID", # Indonesian + 0x045d: "iu_CA", # Inuktitut - Syllabics + 0x085d: "iu_CA", # Inuktitut - Latin + 0x083c: "ga_IE", # Irish - Ireland + 0x0410: "it_IT", # Italian - Italy + 0x0810: "it_CH", # Italian - Switzerland + 0x0411: "ja_JP", # Japanese + 0x044b: "kn_IN", # Kannada - India + 0x043f: "kk_KZ", # Kazakh + 0x0453: "kh_KH", # Khmer - Cambodia + 0x0486: "qut_GT",# K'iche - Guatemala + 0x0487: "rw_RW", # Kinyarwanda - Rwanda + 0x0457: "kok_IN",# Konkani + 0x0412: "ko_KR", # Korean + 0x0440: "ky_KG", # Kyrgyz + 0x0454: "lo_LA", # Lao - Lao PDR + 0x0426: "lv_LV", # Latvian + 0x0427: "lt_LT", # Lithuanian + 0x082e: "dsb_DE",# Lower Sorbian - Germany + 0x046e: "lb_LU", # Luxembourgish + 0x042f: "mk_MK", # FYROM Macedonian + 0x043e: "ms_MY", # Malay - Malaysia + 0x083e: "ms_BN", # Malay - Brunei Darussalam + 0x044c: "ml_IN", # Malayalam - India + 0x043a: "mt_MT", # Maltese + 0x0481: "mi_NZ", # Maori + 0x047a: "arn_CL",# Mapudungun + 0x044e: "mr_IN", # Marathi + 0x047c: "moh_CA",# Mohawk - Canada + 0x0450: "mn_MN", # Mongolian - Cyrillic + 0x0850: "mn_CN", # Mongolian - PRC + 0x0461: "ne_NP", # Nepali + 0x0414: "nb_NO", # Norwegian - Bokmal + 0x0814: "nn_NO", # Norwegian - Nynorsk + 0x0482: "oc_FR", # Occitan - France + 0x0448: "or_IN", # Oriya - India + 0x0463: "ps_AF", # Pashto - Afghanistan + 0x0429: "fa_IR", # Persian + 0x0415: "pl_PL", # Polish + 0x0416: "pt_BR", # Portuguese - Brazil + 0x0816: "pt_PT", # Portuguese - Portugal + 0x0446: "pa_IN", # Punjabi + 0x046b: "quz_BO",# Quechua (Bolivia) + 0x086b: "quz_EC",# Quechua (Ecuador) + 0x0c6b: "quz_PE",# Quechua (Peru) + 0x0418: "ro_RO", # Romanian - Romania + 0x0417: "rm_CH", # Romansh + 0x0419: "ru_RU", # Russian + 0x243b: "smn_FI",# Sami Finland + 0x103b: "smj_NO",# Sami Norway + 0x143b: "smj_SE",# Sami Sweden + 0x043b: "se_NO", # Sami Northern Norway + 0x083b: "se_SE", # Sami Northern Sweden + 0x0c3b: "se_FI", # Sami Northern Finland + 0x203b: "sms_FI",# Sami Skolt + 0x183b: "sma_NO",# Sami Southern Norway + 0x1c3b: "sma_SE",# Sami Southern Sweden + 0x044f: "sa_IN", # Sanskrit + 0x0c1a: "sr_SP", # Serbian - Cyrillic + 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic + 0x081a: "sr_SP", # Serbian - Latin + 0x181a: "sr_BA", # Serbian - Bosnia Latin + 0x045b: "si_LK", # Sinhala - Sri Lanka + 0x046c: "ns_ZA", # Northern Sotho + 0x0432: "tn_ZA", # Setswana - Southern Africa + 0x041b: "sk_SK", # Slovak + 0x0424: "sl_SI", # Slovenian + 0x040a: "es_ES", # Spanish - Spain + 0x080a: "es_MX", # Spanish - Mexico + 0x0c0a: "es_ES", # Spanish - Spain (Modern) + 0x100a: "es_GT", # Spanish - Guatemala + 0x140a: "es_CR", # Spanish - Costa Rica + 0x180a: "es_PA", # Spanish - Panama + 0x1c0a: "es_DO", # Spanish - Dominican Republic + 0x200a: "es_VE", # Spanish - Venezuela + 0x240a: "es_CO", # Spanish - Colombia + 0x280a: "es_PE", # Spanish - Peru + 0x2c0a: "es_AR", # Spanish - Argentina + 0x300a: "es_EC", # Spanish - Ecuador + 0x340a: "es_CL", # Spanish - Chile + 0x380a: "es_UR", # Spanish - Uruguay + 0x3c0a: "es_PY", # Spanish - Paraguay + 0x400a: "es_BO", # Spanish - Bolivia + 0x440a: "es_SV", # Spanish - El Salvador + 0x480a: "es_HN", # Spanish - Honduras + 0x4c0a: "es_NI", # Spanish - Nicaragua + 0x500a: "es_PR", # Spanish - Puerto Rico + 0x540a: "es_US", # Spanish - United States +# 0x0430: "", # Sutu - Not supported + 0x0441: "sw_KE", # Swahili + 0x041d: "sv_SE", # Swedish - Sweden + 0x081d: "sv_FI", # Swedish - Finland + 0x045a: "syr_SY",# Syriac + 0x0428: "tg_TJ", # Tajik - Cyrillic + 0x085f: "tmz_DZ",# Tamazight - Latin + 0x0449: "ta_IN", # Tamil + 0x0444: "tt_RU", # Tatar + 0x044a: "te_IN", # Telugu + 0x041e: "th_TH", # Thai + 0x0851: "bo_BT", # Tibetan - Bhutan + 0x0451: "bo_CN", # Tibetan - PRC + 0x041f: "tr_TR", # Turkish + 0x0442: "tk_TM", # Turkmen - Cyrillic + 0x0480: "ug_CN", # Uighur - Arabic + 0x0422: "uk_UA", # Ukrainian + 0x042e: "wen_DE",# Upper Sorbian - Germany + 0x0420: "ur_PK", # Urdu + 0x0820: "ur_IN", # Urdu - India + 0x0443: "uz_UZ", # Uzbek - Latin + 0x0843: "uz_UZ", # Uzbek - Cyrillic + 0x042a: "vi_VN", # Vietnamese + 0x0452: "cy_GB", # Welsh + 0x0488: "wo_SN", # Wolof - Senegal + 0x0434: "xh_ZA", # Xhosa - South Africa + 0x0485: "sah_RU",# Yakut - Cyrillic + 0x0478: "ii_CN", # Yi - PRC + 0x046a: "yo_NG", # Yoruba - Nigeria + 0x0435: "zu_ZA", # Zulu +} + +def _print_locale(): + + """ Test function. + """ + categories = {} + def _init_categories(categories=categories): + for k,v in globals().items(): + if k[:3] == 'LC_': + categories[k] = v + _init_categories() + del categories['LC_ALL'] + + print 'Locale defaults as determined by getdefaultlocale():' + print '-'*72 + lang, enc = getdefaultlocale() + print 'Language: ', lang or '(undefined)' + print 'Encoding: ', enc or '(undefined)' + print + + print 'Locale settings on startup:' + print '-'*72 + for name,category in categories.items(): + print name, '...' + lang, enc = getlocale(category) + print ' Language: ', lang or '(undefined)' + print ' Encoding: ', enc or '(undefined)' + print + + print + print 'Locale settings after calling resetlocale():' + print '-'*72 + resetlocale() + for name,category in categories.items(): + print name, '...' + lang, enc = getlocale(category) + print ' Language: ', lang or '(undefined)' + print ' Encoding: ', enc or '(undefined)' + print + + try: + setlocale(LC_ALL, "") + except: + print 'NOTE:' + print 'setlocale(LC_ALL, "") does not support the default locale' + print 'given in the OS environment variables.' + else: + print + print 'Locale settings after calling setlocale(LC_ALL, ""):' + print '-'*72 + for name,category in categories.items(): + print name, '...' + lang, enc = getlocale(category) + print ' Language: ', lang or '(undefined)' + print ' Encoding: ', enc or '(undefined)' + print + +### + +try: + LC_MESSAGES +except NameError: + pass +else: + __all__.append("LC_MESSAGES") + +if __name__=='__main__': + print 'Locale aliasing:' + print + _print_locale() + print + print 'Number formatting:' + print + _test() -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 5 19:46:02 2017 From: jython-checkins at python.org (darjus.loktevic) Date: Mon, 06 Mar 2017 00:46:02 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Don=27t_set_a_capath_if_a_c?= =?utf-8?q?acert_is_specified?= Message-ID: <20170306004601.106719.57919.3E1FCCD4@psf.io> https://hg.python.org/jython/rev/eb676b781ab2 changeset: 8048:eb676b781ab2 user: Tom Alexander date: Mon Mar 06 11:38:34 2017 +1100 summary: Don't set a capath if a cacert is specified files: Lib/_socket.py | 4 ++-- Lib/ssl.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Lib/_socket.py b/Lib/_socket.py --- a/Lib/_socket.py +++ b/Lib/_socket.py @@ -803,13 +803,13 @@ selector.notify(self, exception=exception, hangup=hangup) @raises_java_exception - def _handle_channel_future(self, future, reason): + def _handle_channel_future(self, future, reason, wait=False): # All differences between nonblocking vs blocking with optional timeouts # is managed by this method. # # All sockets can be selected on, regardless of blocking/nonblocking state. future.addListener(self._notify_selectors) - if self.timeout is None: + if self.timeout is None or wait: log.debug("Syncing on future %s for %s", future, reason, extra={"sock": self}) return future.sync() elif self.timeout: diff --git a/Lib/ssl.py b/Lib/ssl.py --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -306,7 +306,6 @@ if java_cert_file is not None and os.path.isfile(java_cert_file): cafile = java_cert_file - capath = os.path.dirname(java_cert_file) else: if default_cert_dir_env is not None: capath = default_cert_dir_env if os.path.isdir(default_cert_dir_env) else None @@ -323,7 +322,7 @@ capath = os.path.dirname(cafile) return DefaultVerifyPaths(cafile if os.path.isfile(cafile) else None, - capath if os.path.isdir(capath) else None, + capath if capath and os.path.isdir(capath) else None, 'SSL_CERT_FILE', default_cert_file_env, 'SSL_CERT_DIR', default_cert_dir_env) @@ -689,7 +688,7 @@ # http://stackoverflow.com/questions/24628271/exception-in-netty-io-netty-util-concurrent-blockingoperationexception # - handshake in the child thread pool else: - self._sock._handle_channel_future(self._handshake_future, "SSL handshake") + self._sock._handle_channel_future(self._handshake_future, "SSL handshake", wait=True) def dup(self): raise NotImplemented("Can't dup() %s instances" % @@ -1154,9 +1153,13 @@ if os.path.isfile(possible_cafile): cafiles.append(possible_cafile) elif os.path.isfile(possible_cafile): - with open(possible_cafile) as f: - if PEM_HEADER in f.read(): - cafiles.append(possible_cafile) + try: + with open(possible_cafile) as f: + if PEM_HEADER in f.read(): + cafiles.append(possible_cafile) + except IOError: + log.debug("Not including %s file as a possible cafile due to permissions error" % possible_cafile) + pass # Probably permissions related...ignore certs = [] private_key = None -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Mar 6 11:18:13 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 06 Mar 2017 16:18:13 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232504=2E?= Message-ID: <20170306161812.29652.32730.0F3C6EE3@psf.io> https://hg.python.org/jython/rev/e5f34158a59c changeset: 8049:e5f34158a59c user: Stefan Richthofer date: Mon Mar 06 12:18:02 2017 -0400 summary: Fixed #2504. files: Lib/datetime.py | 9 +++++++-- Lib/test/test_datetime_jy.py | 6 ++++-- NEWS | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -43,6 +43,9 @@ cal.clear() return cal + def _make_java_default_calendar(): + return GregorianCalendar(0, 0, 0, 0, 0, 0) + def _make_java_calendar(d): tzinfo = d.tzinfo if tzinfo == None: @@ -1073,11 +1076,13 @@ def __tojava__(self, java_class): if java_class not in (Calendar, Date, Object): return Py.NoConversion - calendar = _make_java_utc_calendar() - calendar.set(self.year, self.month - 1, self.day) if java_class == Calendar: + calendar = _make_java_utc_calendar() + calendar.set(self.year, self.month - 1, self.day) return calendar else: + calendar = _make_java_default_calendar() + calendar.set(self.year, self.month - 1, self.day) return Date(calendar.getTimeInMillis()) diff --git a/Lib/test/test_datetime_jy.py b/Lib/test/test_datetime_jy.py --- a/Lib/test/test_datetime_jy.py +++ b/Lib/test/test_datetime_jy.py @@ -6,7 +6,7 @@ from datetime import time from datetime import date, datetime -from java.util import Calendar, GregorianCalendar +from java.util import Calendar, GregorianCalendar, TimeZone from java.sql import Date, Time, Timestamp @@ -109,7 +109,9 @@ x = date(2007, 1, 3) y = x.__tojava__(Date) self.assertIsInstance(y, Date) - self.assertEqual(y.getTime(), (x - date(1970, 1, 1)).total_seconds() * 1000) + # Note that java.sql.Date operates regarding to default timezone, so adjust offset + off = TimeZone.getDefault().getRawOffset() + self.assertEqual(y.getTime()+off, (x - date(1970, 1, 1)).total_seconds() * 1000) def test_time(self): self.assertTrue(hasattr(time, "__tojava__")) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2504 ] datetime.date.__tojava__ returns incorrect dates in non-UTC timezones with + negative offset (Jython 2.7.0) - [ 2561 ] win32_ver raises exception (breaks test_platform on windows) - [ 2521 ] Windows installation (all) fails on Windows 10 - [ 2557 ] ongoing pain with platform detection via os.name and sys.platform -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Tue Mar 7 19:27:17 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Wed, 08 Mar 2017 00:27:17 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232524_by_applying_t?= =?utf-8?q?he_suggested_patch=2E?= Message-ID: <20170308002716.51229.36126.1994A67C@psf.io> https://hg.python.org/jython/rev/0522ab9e72e9 changeset: 8050:0522ab9e72e9 user: Stefan Richthofer date: Tue Mar 07 20:26:48 2017 -0400 summary: Fixed #2524 by applying the suggested patch. files: Lib/datetime.py | 12 ++++++++++-- NEWS | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1498,8 +1498,16 @@ calendar = _make_java_calendar(self) if calendar == Py.NoConversion: return Py.NoConversion - epoch_ms = (self.hour * 3600 + self.minute * 60 + self.second) * 1000 + self.microsecond // 1000 - calendar.setTimeInMillis(epoch_ms) + + #initialize to epoch time - effectively clear out the current date from the calendar. + calendar.setTimeInMillis(0); + + #now setup the calendar to have the details populated from this time. + calendar.set(Calendar.HOUR_OF_DAY, self.hour) + calendar.set(Calendar.MINUTE, self.minute) + calendar.set(Calendar.SECOND, self.second) + calendar.set(Calendar.MILLISECOND, self.microsecond // 1000) + if java_class == Calendar: return calendar else: diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2524 ] datetime <-> time conversion incorrect in non UTC times - [ 2504 ] datetime.date.__tojava__ returns incorrect dates in non-UTC timezones with negative offset (Jython 2.7.0) - [ 2561 ] win32_ver raises exception (breaks test_platform on windows) -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Mar 8 00:20:34 2017 From: jython-checkins at python.org (darjus.loktevic) Date: Wed, 08 Mar 2017 05:20:34 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Backout_synchronization_par?= =?utf-8?q?t_of_=232500_as_it_breaks_a_test_and_needs_a_separate?= Message-ID: <20170308052033.3050.42539.41209C6C@psf.io> https://hg.python.org/jython/rev/c72ebac874f1 changeset: 8051:c72ebac874f1 user: Darjus Loktevic date: Wed Mar 08 16:19:46 2017 +1100 summary: Backout synchronization part of #2500 as it breaks a test and needs a separate look files: Lib/_socket.py | 4 ++-- Lib/ssl.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/_socket.py b/Lib/_socket.py --- a/Lib/_socket.py +++ b/Lib/_socket.py @@ -803,13 +803,13 @@ selector.notify(self, exception=exception, hangup=hangup) @raises_java_exception - def _handle_channel_future(self, future, reason, wait=False): + def _handle_channel_future(self, future, reason): # All differences between nonblocking vs blocking with optional timeouts # is managed by this method. # # All sockets can be selected on, regardless of blocking/nonblocking state. future.addListener(self._notify_selectors) - if self.timeout is None or wait: + if self.timeout is None: log.debug("Syncing on future %s for %s", future, reason, extra={"sock": self}) return future.sync() elif self.timeout: diff --git a/Lib/ssl.py b/Lib/ssl.py --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -688,7 +688,7 @@ # http://stackoverflow.com/questions/24628271/exception-in-netty-io-netty-util-concurrent-blockingoperationexception # - handshake in the child thread pool else: - self._sock._handle_channel_future(self._handshake_future, "SSL handshake", wait=True) + self._sock._handle_channel_future(self._handshake_future, "SSL handshake") def dup(self): raise NotImplemented("Can't dup() %s instances" % -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 9 12:06:27 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 09 Mar 2017 17:06:27 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Undo_skips_and_work-arounds?= =?utf-8?q?_related_to_=231996_now_fixed=2E?= Message-ID: <20170309170626.30526.61816.C5EF5FF7@psf.io> https://hg.python.org/jython/rev/de7eb61af35a changeset: 8052:de7eb61af35a user: Jeff Allen date: Thu Mar 09 09:15:57 2017 +0000 summary: Undo skips and work-arounds related to #1996 now fixed. files: Lib/test/test_io.py | 21 -------------- Lib/test/test_memoryio.py | 22 --------------- src/org/python/modules/_io/_jyio.java | 4 +-- 3 files changed, 1 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -650,13 +650,6 @@ "len(array.array) returns number of elements rather than bytelength" )(IOTest.test_array_writes) - # When Jython tries to use UnsupportedOperation as _pyio defines it, it runs - # into a problem with multiple inheritance and the slots array: issue 1996. - # Override the affected test version just so we can skip it visibly. - @unittest.skipIf(support.is_jython, "FIXME: Jython issue 1996") - def test_invalid_operations(self): - pass - # Jython does not use integer file descriptors but an object instead. # Unfortunately, _pyio.open checks that it is an int. # Override the affected test versions just so we can skip them visibly. @@ -1480,13 +1473,6 @@ class PyBufferedRWPairTest(BufferedRWPairTest): tp = pyio.BufferedRWPair - # When Jython tries to use UnsupportedOperation as _pyio defines it, it runs - # into a problem with multiple inheritance and the slots array: issue 1996. - # Override the affected test version just so we can skip it visibly. - @unittest.skipIf(support.is_jython, "FIXME: Jython issue 1996") - def test_detach(self): - pass - class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): read_mode = "rb+" @@ -2906,13 +2892,6 @@ class PyMiscIOTest(MiscIOTest): io = pyio - # When Jython tries to use UnsupportedOperation as _pyio defines it, it runs - # into a problem with multiple inheritance and the slots array: issue 1996. - # Override the affected test version just so we can skip it visibly. - @unittest.skipIf(support.is_jython, "FIXME: Jython issue 1996") - def test_io_after_close(self): - pass - # Jython does not use integer file descriptors but an object instead. # Unfortunately, _pyio.open checks that it is an int. # Override the affected test version just so we can skip it visibly. diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -398,13 +398,6 @@ UnsupportedOperation = pyio.UnsupportedOperation - # When Jython tries to use UnsupportedOperation as _pyio defines it, it runs - # into a problem with multiple inheritance and the slots array: issue 1996. - # Override the affected test version just so we can skip it visibly. - @unittest.skipIf(support.is_jython, "FIXME: Jython issue 1996") - def test_detach(self): - pass - @staticmethod def buftype(s): return s.encode("ascii") @@ -595,13 +588,6 @@ UnsupportedOperation = pyio.UnsupportedOperation EOF = "" - # When Jython tries to use UnsupportedOperation as _pyio defines it, it runs - # into a problem with multiple inheritance and the slots array: issue 1996. - # Override the affected test version just so we can skip it visibly. - @unittest.skipIf(support.is_jython, "FIXME: Jython issue 1996") - def test_detach(self): - pass - class PyStringIOPickleTest(TextIOTestMixin, unittest.TestCase): """Test if pickle restores properly the internal state of StringIO. @@ -625,10 +611,6 @@ "array.array() does not have the new buffer API" )(PyBytesIOTest.test_bytes_array) - # Re-instate test_detach skipped by Jython in PyBytesIOTest - if support.is_jython: # FIXME: Jython issue 1996 - test_detach = MemoryTestMixin.test_detach - def test_getstate(self): memio = self.ioclass() state = memio.__getstate__() @@ -673,10 +655,6 @@ # XXX: For the Python version of io.StringIO, this is highly # dependent on the encoding used for the underlying buffer. - # Re-instate test_detach skipped by Jython in PyBytesIOTest - if support.is_jython: # FIXME: Jython issue 1996 - test_detach = MemoryTestMixin.test_detach - # This test checks that tell() results are consistent with the length of # text written, but this is not documented in the API: only that seek() # accept what tell() returns. diff --git a/src/org/python/modules/_io/_jyio.java b/src/org/python/modules/_io/_jyio.java --- a/src/org/python/modules/_io/_jyio.java +++ b/src/org/python/modules/_io/_jyio.java @@ -40,9 +40,7 @@ PyObject ValueError = exceptions.__getattr__("ValueError"); PyObject IOError = exceptions.__getattr__("IOError"); // Equivalent to class UnsupportedOperation(ValueError, IOError) : pass - // UnsupportedOperation = makeException(dict, "UnsupportedOperation", ValueError, IOError); - // XXX Work-around: slots not properly initialised unless IOError comes first - UnsupportedOperation = makeException(dict, "UnsupportedOperation", IOError, ValueError); + UnsupportedOperation = makeException(dict, "UnsupportedOperation", ValueError, IOError); // Hide from Python dict.__setitem__("classDictInit", null); -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sat Mar 11 06:38:29 2017 From: jython-checkins at python.org (jeff.allen) Date: Sat, 11 Mar 2017 11:38:29 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_=232559_Update_test=5Fj?= =?utf-8?q?ython=5Flauncher_with_new_Xss_value?= Message-ID: <20170311113829.28551.45382.308EBEFE@psf.io> https://hg.python.org/jython/rev/17e40de9a541 changeset: 8053:17e40de9a541 user: James Mudd date: Mon Mar 06 22:51:39 2017 +0000 summary: Fix #2559 Update test_jython_launcher with new Xss value Addresses failures in test_marshal on Java 8 caused by Java stack overflow. Stack now -Xss2560k. files: Lib/test/test_jython_launcher.py | 8 ++++---- src/shell/jython | 6 ++---- src/shell/jython.exe | Bin src/shell/jython.py | 2 +- src/shell/python27.dll | Bin 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_jython_launcher.py b/Lib/test/test_jython_launcher.py --- a/Lib/test/test_jython_launcher.py +++ b/Lib/test/test_jython_launcher.py @@ -105,7 +105,7 @@ args = self.get_cmdline([launcher, "--print"], env) self.assertEqual(args[0], os.path.join(my_java, "bin", "java")) self.assertEqual(args[1], "-Xmx512m") - self.assertEqual(args[2], "-Xss1024k") + self.assertEqual(args[2], "-Xss2560k") self.assertEqual(args[-1], "org.python.util.jython") def test_java_opts(self): @@ -115,7 +115,7 @@ props = self.get_properties(args) self.assertEqual(args[0], "java") self.assertEqual(args[1], "-Xmx2g") - self.assertEqual(args[2], "-Xss1024k") + self.assertEqual(args[2], "-Xss2560k") self.assertEqual(args[3], "-classpath", args) self.assertEqual(args[4].split(classpath_delimiter())[-1], some_jar) self.assertEqual(args[-1], "org.python.util.jython") @@ -128,7 +128,7 @@ props = self.get_properties(args) self.assertEqual(args[0], "java") self.assertEqual(args[1], "-Xmx512m") - self.assertEqual(args[2], "-Xss1024k") + self.assertEqual(args[2], "-Xss2560k") self.assertEqual(args[-1], "org.python.util.jython") self.assertIn("python.home", props) self.assertIn("python.executable", props) @@ -159,7 +159,7 @@ args = self.get_cmdline([launcher, "--print"], env) self.assertEqual(args[0], "java") self.assertEqual(args[1], "-Xmx512m") - self.assertEqual(args[2], "-Xss1024k") + self.assertEqual(args[2], "-Xss2560k") self.assertEqual(args[-3], "org.python.util.jython") self.assertEqual(args[-2], "-c") self.assertEqual(args[-1], "print 47") diff --git a/src/shell/jython b/src/shell/jython --- a/src/shell/jython +++ b/src/shell/jython @@ -109,10 +109,8 @@ fi if [ -z "$JAVA_STACK" ]; then - # 32 bit Java 6 needs the stack increased to at least 512k for - # test_cpickle to pass, but we don't want to shrink 64 bit Java's - # default of 1024k - JAVA_STACK=-Xss1024k + # test_marshal.py needs a Xss of 2560k to pass + JAVA_STACK=-Xss2560k fi JAVA_ENCODING="" diff --git a/src/shell/jython.exe b/src/shell/jython.exe index 2a03827dd99c8d2d908074a2e3de15abf56e30d2..7c9cbe9eec239c5768c17f873726220b09966341 GIT binary patch [stripped] diff --git a/src/shell/jython.py b/src/shell/jython.py --- a/src/shell/jython.py +++ b/src/shell/jython.py @@ -208,7 +208,7 @@ if hasattr(self.args, "stack"): return self.args.stack else: - return os.environ.get("JAVA_STACK", "-Xss1024k") + return os.environ.get("JAVA_STACK", "-Xss2560k") @property def java_opts(self): diff --git a/src/shell/python27.dll b/src/shell/python27.dll index 8d99ae24798acc720af4dca97140e0f591f39eb7..f62238f1826f1c1230c0db7938c1133e1ca5ae5d GIT binary patch [stripped] -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 12 20:26:42 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 13 Mar 2017 00:26:42 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_test=5Fdatetime=5Fjy?= =?utf-8?q?=2E?= Message-ID: <20170313002642.29745.57326.1FEEC848@psf.io> https://hg.python.org/jython/rev/3adbec1576f5 changeset: 8054:3adbec1576f5 user: Stefan Richthofer date: Mon Mar 13 01:26:07 2017 +0100 summary: Fixed test_datetime_jy. files: Lib/test/test_datetime_jy.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_datetime_jy.py b/Lib/test/test_datetime_jy.py --- a/Lib/test/test_datetime_jy.py +++ b/Lib/test/test_datetime_jy.py @@ -111,7 +111,10 @@ self.assertIsInstance(y, Date) # Note that java.sql.Date operates regarding to default timezone, so adjust offset off = TimeZone.getDefault().getRawOffset() - self.assertEqual(y.getTime()+off, (x - date(1970, 1, 1)).total_seconds() * 1000) + # It's sufficient for the date to fit; we modulo away the time, so this test + # won't run into TimeZone issues. + self.assertEqual((y.getTime()+off)//(1000*60*60*24), + (x - date(1970, 1, 1)).total_seconds()//(60*60*24)) def test_time(self): self.assertTrue(hasattr(time, "__tojava__")) -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Mar 13 07:43:42 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 13 Mar 2017 11:43:42 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Converted_some_collection_c?= =?utf-8?q?reation_in_PyType_to_org=2Epython=2Eutil=2EGeneric_style=2E?= Message-ID: <20170313114340.50583.91496.23AC0420@psf.io> https://hg.python.org/jython/rev/880c134d14bf changeset: 8055:880c134d14bf user: Stefan Richthofer date: Mon Mar 13 12:43:33 2017 +0100 summary: Converted some collection creation in PyType to org.python.util.Generic style. Added some more methods to org.python.util.Generic for this. files: src/org/python/core/PyType.java | 13 ++---- src/org/python/util/Generic.java | 34 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/org/python/core/PyType.java b/src/org/python/core/PyType.java --- a/src/org/python/core/PyType.java +++ b/src/org/python/core/PyType.java @@ -8,9 +8,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.IdentityHashMap; +import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -233,7 +231,7 @@ * @return position of first ancestor that is not equal to or ancestor of primary base */ private static int findSlottedAncestors(PyType tp, List dest, - IdentityHashMap slotsMap) { + Map slotsMap) { int baseEnd = 0; if (tp.base != null && tp.base.numSlots > 0 && !slotsMap.containsKey(tp.base)) { findSlottedAncestors(tp.base, dest, slotsMap); @@ -284,9 +282,8 @@ * @param mayAddWeak whether a __weakref__ descriptor is allowed on this type */ private void createAllSlots(boolean mayAddDict, boolean mayAddWeak) { - List slottedAncestors = new ArrayList<>(base.mro.length+(bases.length-1)*3+1); - IdentityHashMap slotsMap = - new IdentityHashMap<>(slottedAncestors.size()); + List slottedAncestors = Generic.list(base.mro.length+(bases.length-1)*3+1); + Map slotsMap = Generic.identityHashMap(slottedAncestors.size()); /* Here we would need the mro to search for slots (also in secondary bases) properly, but mro hasn't been set up yet. So we quickly (?) build a pseudo mro sufficient to find all slots. */ @@ -301,7 +298,7 @@ At any time we prevent it from containing __dict__ or __weakref__. */ // we know the required capacity, so the set likely won't be resized - LinkedHashSet allSlots = new LinkedHashSet<>(2*slots_tmp); + Set allSlots = Generic.linkedHashSet(2*slots_tmp); if (baseEnd > 0) { for (int i = 0; i < baseEnd; ++i) { insertSlots(slotsMap.get(slottedAncestors.get(i)), allSlots); diff --git a/src/org/python/util/Generic.java b/src/org/python/util/Generic.java --- a/src/org/python/util/Generic.java +++ b/src/org/python/util/Generic.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -35,6 +36,14 @@ } /** + * Makes a List with its generic type inferred from whatever it's being assigned to. + * Sets initial capacity accordingly. + */ + public static List list(int capacity) { + return new ArrayList(capacity); + } + + /** * Makes a List with its generic type inferred from whatever it's being assigned to filled with * the items in contents. */ @@ -55,6 +64,23 @@ } /** + * Makes an IdentityHashMap using generic types inferred from whatever this is being + * assigned to. + */ + public static Map identityHashMap() { + return new IdentityHashMap(); + } + + /** + * Makes an IdentityHashMap using generic types inferred from whatever this is being + * assigned to. + * Sets initial capacity accordingly. + */ + public static Map identityHashMap(int capacity) { + return new IdentityHashMap(capacity); + } + + /** * Makes a ConcurrentMap using generic types inferred from whatever this is being * assigned to. */ @@ -78,6 +104,14 @@ } /** + * Makes a LinkedHashSet using the generic type inferred from whatever this is being assigned to. + * Sets initial capacity accordingly. + */ + public static Set linkedHashSet(int capacity) { + return new LinkedHashSet(capacity); + } + + /** * Makes a Set using the generic type inferred from whatever this is being assigned to filled * with the items in contents. */ -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Mon Mar 13 07:54:02 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Mon, 13 Mar 2017 11:54:02 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Updated_NEWS=2E?= Message-ID: <20170313115401.106761.78782.9937FCFD@psf.io> https://hg.python.org/jython/rev/3218cf487dc3 changeset: 8056:3218cf487dc3 user: Stefan Richthofer date: Mon Mar 13 12:53:55 2017 +0100 summary: Updated NEWS. files: NEWS | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,9 +4,13 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2559 ] test_marshal fails + - [ 2564 ] test_socket_jy fails on Linux - [ 2524 ] datetime <-> time conversion incorrect in non UTC times - [ 2504 ] datetime.date.__tojava__ returns incorrect dates in non-UTC timezones with negative offset (Jython 2.7.0) + - [ 2500 ] Loading default cacerts on client socket when specifying a default java truststore + unnecessarily searches for more cacerts in the same dir - [ 2561 ] win32_ver raises exception (breaks test_platform on windows) - [ 2521 ] Windows installation (all) fails on Windows 10 - [ 2557 ] ongoing pain with platform detection via os.name and sys.platform -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Mar 15 09:26:41 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Wed, 15 Mar 2017 13:26:41 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232568_=28by_applyin?= =?utf-8?q?g_PR_64_https=3A//github=2Ecom/jythontools/jython/pull/64=29?= Message-ID: <20170315132637.4613.25030.1AD85EA8@psf.io> https://hg.python.org/jython/rev/f06cf81336e0 changeset: 8057:f06cf81336e0 user: James Mudd date: Wed Mar 15 14:26:13 2017 +0100 summary: Fixed #2568 (by applying PR 64 https://github.com/jythontools/jython/pull/64) files: Lib/test/lock_tests.py | 8 +++++++- NEWS | 1 + 2 files changed, 8 insertions(+), 1 deletions(-) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -409,8 +409,14 @@ results.append(t2 - t1) Bunch(f, N).wait_for_finished() self.assertEqual(len(results), 5) + # Fudge factor for running on the JVM - actual waits on + # some OS platforms might be like this example, + # 0.199999809265, slightly less than 0.2 seconds. To avoid + # unnecessary flakiness in testing, make epsilon + # relatively large: + epsilon = 0.01 for dt in results: - self.assertTrue(dt >= 0.2, dt) + self.assertTrue(dt >= 0.2 - epsilon, dt) class BaseSemaphoreTests(BaseTestCase): diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2568 ] test_threading intermittent failure - [ 2559 ] test_marshal fails - [ 2564 ] test_socket_jy fails on Linux - [ 2524 ] datetime <-> time conversion incorrect in non UTC times -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Mar 15 20:01:56 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Thu, 16 Mar 2017 00:01:56 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232318_=28also_see_P?= =?utf-8?q?R_65=2C_https=3A//github=2Ecom/jythontools/jython/pull/65=29?= Message-ID: <20170316000155.59489.15098.DB23924D@psf.io> https://hg.python.org/jython/rev/3898500ffc7e changeset: 8058:3898500ffc7e user: James Mudd date: Thu Mar 16 01:01:34 2017 +0100 summary: Fixed #2318 (also see PR 65, https://github.com/jythontools/jython/pull/65) files: Lib/test/test_zipimport_jy.py | 2 +- NEWS | 1 + 2 files changed, 2 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_zipimport_jy.py b/Lib/test/test_zipimport_jy.py --- a/Lib/test/test_zipimport_jy.py +++ b/Lib/test/test_zipimport_jy.py @@ -47,7 +47,7 @@ class ZipImporterDictTest(unittest.TestCase): def test_subclass_assign_attribute(self): class A(zipimporter): pass - path = os.path.abspath('tests/modjy/lib_python_folder/test_modules.zip') + path = test_support.findfile("zipdir.zip") A(path).somevar = 1 def test_main(): diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2318 ] test_zipimport_jy failure on Windows - [ 2568 ] test_threading intermittent failure - [ 2559 ] test_marshal fails - [ 2564 ] test_socket_jy fails on Linux -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 03:08:52 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 16 Mar 2017 07:08:52 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Re-instate_test=5Ftarfile_a?= =?utf-8?q?s_expected_failure_=28Windows=29?= Message-ID: <20170316070852.92092.36839.1A8B32E7@psf.io> https://hg.python.org/jython/rev/a440ab77aa74 changeset: 8063:a440ab77aa74 user: Jeff Allen date: Thu Mar 16 03:26:48 2017 +0000 summary: Re-instate test_tarfile as expected failure (Windows) Passes on C drive, fails on D drive, but not repeatably the same way. files: Lib/test/regrtest.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1358,6 +1358,7 @@ # test_popen # Passes, but see http://bugs.python.org/issue1559298 test_runpy # OSError: unlink() test_select_new # Hangs (Windows), though ok run singly + test_tarfile # flakey (Windows) test_urllib2 # file not on local host (likely Windows only) """, -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 03:08:52 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 16 Mar 2017 07:08:52 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_test=5Fcodecencodings?= =?utf-8?q?=5Ftw_ahead_of_customising_to_resolve_=232571?= Message-ID: <20170316070852.60286.5801.9E02268B@psf.io> https://hg.python.org/jython/rev/6bdd93c6251c changeset: 8061:6bdd93c6251c user: Jeff Allen date: Tue Mar 14 20:01:39 2017 +0000 summary: Add test_codecencodings_tw ahead of customising to resolve #2571 Adding the file "pure" from lib-python so that the divergence is isolated in a subsequent change set, not lost in the new file. files: Lib/test/test_codecencodings_tw.py | 27 ++++++++++++++++++ 1 files changed, 27 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_codecencodings_tw.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# +# test_codecencodings_tw.py +# Codec encoding tests for ROC encodings. +# + +from test import test_support +from test import test_multibytecodec_support +import unittest + +class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5' + tstring = test_multibytecodec_support.load_teststring('big5') + codectests = ( + # invalid bytes + ("abc\x80\x80\xc1\xc4", "strict", None), + ("abc\xc8", "strict", None), + ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u8b10"), + ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u8b10\ufffd"), + ("abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"), + ) + +def test_main(): + test_support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 03:08:52 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 16 Mar 2017 07:08:52 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Adjust_expected_passes_in_r?= =?utf-8?q?egrtest_=28mostly_Windows=29=2E?= Message-ID: <20170316070852.92426.22621.47BB36B9@psf.io> https://hg.python.org/jython/rev/ca458b567829 changeset: 8060:ca458b567829 user: Jeff Allen date: Mon Mar 13 19:41:32 2017 +0000 summary: Adjust expected passes in regrtest (mostly Windows). test_popen, test_select and test_tarfile now expected to pass. test_logging and test_select_new suppressed because they hang. test_codecencodings_tw suppressed on all platforms, not just Linux. files: Lib/test/regrtest.py | 12 ++++++------ 1 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1294,6 +1294,7 @@ test_codecencodings_iso2022 test_codecencodings_jp test_codecencodings_kr + test_codecencodings_tw # Fails in Java 8 not 7 test_codecmaps_cn test_codecmaps_jp test_codecmaps_kr @@ -1328,7 +1329,6 @@ test_locale test_profile test_pydoc # Hangs with prompt (Windows) - test_select # Unconnected client socket should be selectable test_sundry # ImportError: No module named audiodev test_sys_setprofile # revisit for GC @@ -1337,8 +1337,8 @@ # Unreliable tests test_asynchat # test_gc # Rare failures depending on timing of Java gc - # test_logging - test_tarfile # flakey (Windows) + test_logging # Hangs, though ok run singly + # test_tarfile # warning on irremovable directory (Windows) # test_urllib2net # unexpected output makes this a failure to regrtest.py # Failing tests here are because of lack of STARTTLS; see http://bugs.jython.org/issue2447 @@ -1355,15 +1355,15 @@ 'java.nt': # Expected to fail on Windows """ test_mailbox # fails miserably and ruins other tests - test_os_jy # Locale tests run and fail on Cygwin - test_popen # http://bugs.python.org/issue1559298 + test_os_jy # Locale tests fail on Cygwin (but not Windows) + # test_popen # Passes, but see http://bugs.python.org/issue1559298 test_runpy # OSError: unlink() + test_select_new # Hangs (Windows), though ok run singly test_urllib2 # file not on local host (likely Windows only) """, 'java.posix': # Expected to fail on Linux """ - test_codecencodings_tw # Fails in test_multibytecodec_support.py test_jython_launcher # /usr/bin/env: python2.7 -E: No such file or directory # These leak file handles on a grand scale (observed on Ubuntu 14.04), -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 03:08:52 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 16 Mar 2017 07:08:52 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Skip_parts_of_test=5Fjava?= =?utf-8?q?=5Fintegration_when_jar_command_not_available=2E?= Message-ID: <20170316070851.115180.62949.CFA52DE2@psf.io> https://hg.python.org/jython/rev/ceec31db0cd9 changeset: 8059:ceec31db0cd9 parent: 8053:17e40de9a541 user: Jeff Allen date: Mon Mar 13 07:53:45 2017 +0000 summary: Skip parts of test_java_integration when jar command not available. This test would run in the development environment, because the JDK is on the path, but not for users only installing the JRE. #2554 (not fixed by this) refers. files: Lib/distutils/spawn.py | 6 +++- Lib/test/test_java_integration.py | 23 ++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py --- a/Lib/distutils/spawn.py +++ b/Lib/distutils/spawn.py @@ -235,8 +235,10 @@ paths = path.split(os.pathsep) base, ext = os.path.splitext(executable) - if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): - executable = executable + '.exe' + if (sys.platform == 'win32' or os.name == 'os2' or + sys.platform.startswith('java') and os._name == 'nt'): + if ext != '.exe': + executable = executable + '.exe' if not os.path.isfile(executable): for p in paths: diff --git a/Lib/test/test_java_integration.py b/Lib/test/test_java_integration.py --- a/Lib/test/test_java_integration.py +++ b/Lib/test/test_java_integration.py @@ -13,6 +13,7 @@ from collections import deque from test import test_support +from distutils.spawn import find_executable from java.lang import ( ClassCastException, ExceptionInInitializerError, UnsupportedOperationException, @@ -605,11 +606,13 @@ def find_jython_jars(): # Uses the same classpath resolution as bin/jython - jython_jar_path = os.path.normpath(os.path.join(sys.executable, "../../jython.jar")) - jython_jar_dev_path = os.path.normpath(os.path.join(sys.executable, "../../jython-dev.jar")) + jython_bin = os.path.normpath(os.path.dirname(sys.executable)) + jython_top = os.path.dirname(jython_bin) + jython_jar_path = os.path.join(jython_top, 'jython.jar') + jython_jar_dev_path = os.path.join(jython_top, 'jython-dev.jar') if os.path.exists(jython_jar_dev_path): jars = [jython_jar_dev_path] - jars.extend(glob.glob(os.path.normpath(os.path.join(jython_jar_dev_path, "../javalib/*.jar")))) + jars.extend(glob.glob(os.path.join(jython_top, 'javalib', '*.jar'))) elif os.path.exists(jython_jar_path): jars = [jython_jar_path] else: @@ -684,6 +687,7 @@ names = [x for x in dir(__builtin__)] self.assertEqual(names, roundtrip_serialization(names)) + @unittest.skipUnless(find_executable('jar'), 'Need the jar command to run') def test_proxy_serialization(self): # Proxies can be deserializable in a fresh JVM, including being able # to "findPython" to get a PySystemState. @@ -698,7 +702,8 @@ # Create a jar file containing the Cat proxy; could use Java to do this; do it the easy way for now proxies_jar_path = os.path.join(tempdir, "proxies.jar") - subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, "org/"]) + subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, + "org" + os.path.sep]) # Serialize our cat output = ByteArrayOutputStream() @@ -717,7 +722,7 @@ jars.append(proxies_jar_path) classpath = os.pathsep.join(jars) env = dict(os.environ) - env.update(JYTHONPATH=os.path.normpath(os.path.join(__file__, ".."))) + env.update(JYTHONPATH=os.path.dirname(__file__)) cmd = [os.path.join(System.getProperty("java.home"), "bin", "java"), "-classpath", classpath, "javatests.ProxyDeserialization", @@ -728,6 +733,7 @@ org.python.core.Options.proxyDebugDirectory = old_proxy_debug_dir shutil.rmtree(tempdir) + @unittest.skipUnless(find_executable('jar'), 'Need the jar command to run') def test_custom_proxymaker(self): # Verify custom proxymaker supports direct usage of Python code in Java tempdir = tempfile.mkdtemp() @@ -741,7 +747,8 @@ # Create a jar file containing the org.python.test.Dog proxy proxies_jar_path = os.path.join(tempdir, "proxies.jar") - subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, "org/"]) + subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, + "org" + os.path.sep]) # Build a Java class importing Dog source = """ @@ -775,10 +782,10 @@ # PySystemState (and Jython runtime) is initialized for # the proxy classpath += os.pathsep + tempdir - cmd = [os.path.join(System.getProperty("java.home"), "bin/java"), + cmd = [os.path.join(System.getProperty("java.home"), "bin", "java"), "-classpath", classpath, "BarkTheDog"] env = dict(os.environ) - env.update(JYTHONPATH=os.path.normpath(os.path.join(__file__, ".."))) + env.update(JYTHONPATH=os.path.dirname(__file__)) self.assertRegexpMatches( subprocess.check_output(cmd, env=env, universal_newlines=True, stderr=subprocess.STDOUT), -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 03:08:52 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 16 Mar 2017 07:08:52 +0000 Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_Merge_to_trunk?= Message-ID: <20170316070852.92582.86709.F747BDE1@psf.io> https://hg.python.org/jython/rev/2c0ce625b440 changeset: 8064:2c0ce625b440 parent: 8058:3898500ffc7e parent: 8063:a440ab77aa74 user: Jeff Allen date: Thu Mar 16 07:08:22 2017 +0000 summary: Merge to trunk files: Lib/distutils/spawn.py | 6 +- Lib/test/regrtest.py | 12 ++-- Lib/test/test_codecencodings_tw.py | 47 ++++++++++++++++++ Lib/test/test_java_integration.py | 23 +++++--- NEWS | 1 + 5 files changed, 73 insertions(+), 16 deletions(-) diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py --- a/Lib/distutils/spawn.py +++ b/Lib/distutils/spawn.py @@ -235,8 +235,10 @@ paths = path.split(os.pathsep) base, ext = os.path.splitext(executable) - if (sys.platform == 'win32' or os.name == 'os2') and (ext != '.exe'): - executable = executable + '.exe' + if (sys.platform == 'win32' or os.name == 'os2' or + sys.platform.startswith('java') and os._name == 'nt'): + if ext != '.exe': + executable = executable + '.exe' if not os.path.isfile(executable): for p in paths: diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1328,7 +1328,6 @@ test_locale test_profile test_pydoc # Hangs with prompt (Windows) - test_select # Unconnected client socket should be selectable test_sundry # ImportError: No module named audiodev test_sys_setprofile # revisit for GC @@ -1337,8 +1336,8 @@ # Unreliable tests test_asynchat # test_gc # Rare failures depending on timing of Java gc - # test_logging - test_tarfile # flakey (Windows) + test_logging # Hangs, though ok run singly + # test_tarfile # warning on irremovable directory (Windows) # test_urllib2net # unexpected output makes this a failure to regrtest.py # Failing tests here are because of lack of STARTTLS; see http://bugs.jython.org/issue2447 @@ -1355,15 +1354,16 @@ 'java.nt': # Expected to fail on Windows """ test_mailbox # fails miserably and ruins other tests - test_os_jy # Locale tests run and fail on Cygwin - test_popen # http://bugs.python.org/issue1559298 + test_os_jy # Locale tests fail on Cygwin (but not Windows) + # test_popen # Passes, but see http://bugs.python.org/issue1559298 test_runpy # OSError: unlink() + test_select_new # Hangs (Windows), though ok run singly + test_tarfile # flakey (Windows) test_urllib2 # file not on local host (likely Windows only) """, 'java.posix': # Expected to fail on Linux """ - test_codecencodings_tw # Fails in test_multibytecodec_support.py test_jython_launcher # /usr/bin/env: python2.7 -E: No such file or directory # These leak file handles on a grand scale (observed on Ubuntu 14.04), diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py new file mode 100644 --- /dev/null +++ b/Lib/test/test_codecencodings_tw.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# +# test_codecencodings_tw.py +# Codec encoding tests for ROC encodings. +# + +from test import test_support +from test import test_multibytecodec_support +import unittest +import sys + +# Codecs re-synchronise sooner after illegal byte in Java 8+ than in Java 7 +# (and in CPython 3.3+ than in CPython 2.7-3.2). Either is correct, but we +# need to know which one to expect. +RESYNC_FASTER = sys.platform.startswith('java') and \ + sys.platform[4:7] > "1.7" + +class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'big5' + tstring = test_multibytecodec_support.load_teststring('big5') + if RESYNC_FASTER: + # Version from CPython 3.6 where \0x80\0x80 is two invalid sequences. + # Java 8 agrees with this interpretation. + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"), + ) + else: + # Standard version of test from CPython 2.7 + codectests = ( + # invalid bytes + ("abc\x80\x80\xc1\xc4", "strict", None), + ("abc\xc8", "strict", None), + ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u8b10"), + ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u8b10\ufffd"), + ("abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"), + ) + +def test_main(): + test_support.run_unittest(__name__) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_java_integration.py b/Lib/test/test_java_integration.py --- a/Lib/test/test_java_integration.py +++ b/Lib/test/test_java_integration.py @@ -13,6 +13,7 @@ from collections import deque from test import test_support +from distutils.spawn import find_executable from java.lang import ( ClassCastException, ExceptionInInitializerError, UnsupportedOperationException, @@ -605,11 +606,13 @@ def find_jython_jars(): # Uses the same classpath resolution as bin/jython - jython_jar_path = os.path.normpath(os.path.join(sys.executable, "../../jython.jar")) - jython_jar_dev_path = os.path.normpath(os.path.join(sys.executable, "../../jython-dev.jar")) + jython_bin = os.path.normpath(os.path.dirname(sys.executable)) + jython_top = os.path.dirname(jython_bin) + jython_jar_path = os.path.join(jython_top, 'jython.jar') + jython_jar_dev_path = os.path.join(jython_top, 'jython-dev.jar') if os.path.exists(jython_jar_dev_path): jars = [jython_jar_dev_path] - jars.extend(glob.glob(os.path.normpath(os.path.join(jython_jar_dev_path, "../javalib/*.jar")))) + jars.extend(glob.glob(os.path.join(jython_top, 'javalib', '*.jar'))) elif os.path.exists(jython_jar_path): jars = [jython_jar_path] else: @@ -684,6 +687,7 @@ names = [x for x in dir(__builtin__)] self.assertEqual(names, roundtrip_serialization(names)) + @unittest.skipUnless(find_executable('jar'), 'Need the jar command to run') def test_proxy_serialization(self): # Proxies can be deserializable in a fresh JVM, including being able # to "findPython" to get a PySystemState. @@ -698,7 +702,8 @@ # Create a jar file containing the Cat proxy; could use Java to do this; do it the easy way for now proxies_jar_path = os.path.join(tempdir, "proxies.jar") - subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, "org/"]) + subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, + "org" + os.path.sep]) # Serialize our cat output = ByteArrayOutputStream() @@ -717,7 +722,7 @@ jars.append(proxies_jar_path) classpath = os.pathsep.join(jars) env = dict(os.environ) - env.update(JYTHONPATH=os.path.normpath(os.path.join(__file__, ".."))) + env.update(JYTHONPATH=os.path.dirname(__file__)) cmd = [os.path.join(System.getProperty("java.home"), "bin", "java"), "-classpath", classpath, "javatests.ProxyDeserialization", @@ -728,6 +733,7 @@ org.python.core.Options.proxyDebugDirectory = old_proxy_debug_dir shutil.rmtree(tempdir) + @unittest.skipUnless(find_executable('jar'), 'Need the jar command to run') def test_custom_proxymaker(self): # Verify custom proxymaker supports direct usage of Python code in Java tempdir = tempfile.mkdtemp() @@ -741,7 +747,8 @@ # Create a jar file containing the org.python.test.Dog proxy proxies_jar_path = os.path.join(tempdir, "proxies.jar") - subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, "org/"]) + subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, + "org" + os.path.sep]) # Build a Java class importing Dog source = """ @@ -775,10 +782,10 @@ # PySystemState (and Jython runtime) is initialized for # the proxy classpath += os.pathsep + tempdir - cmd = [os.path.join(System.getProperty("java.home"), "bin/java"), + cmd = [os.path.join(System.getProperty("java.home"), "bin", "java"), "-classpath", classpath, "BarkTheDog"] env = dict(os.environ) - env.update(JYTHONPATH=os.path.normpath(os.path.join(__file__, ".."))) + env.update(JYTHONPATH=os.path.dirname(__file__)) self.assertRegexpMatches( subprocess.check_output(cmd, env=env, universal_newlines=True, stderr=subprocess.STDOUT), diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -5,6 +5,7 @@ Jython 2.7.1rc1 Bugs fixed - [ 2318 ] test_zipimport_jy failure on Windows + - [ 2571 ] Error handling in test_codecencodings_tw fails on Java 8 - [ 2568 ] test_threading intermittent failure - [ 2559 ] test_marshal fails - [ 2564 ] test_socket_jy fails on Linux -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 03:08:53 2017 From: jython-checkins at python.org (jeff.allen) Date: Thu, 16 Mar 2017 07:08:53 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Accept_faster_codec_re-sync?= =?utf-8?q?hronisation_after_error=2E_Fixes_=232571?= Message-ID: <20170316070852.114675.5546.D7509040@psf.io> https://hg.python.org/jython/rev/1f0cc9488f41 changeset: 8062:1f0cc9488f41 user: Jeff Allen date: Tue Mar 14 22:34:45 2017 +0000 summary: Accept faster codec re-synchronisation after error. Fixes #2571 Java 8 codec has different error strategy from Java 7, but acceptable, and matching Python 3.3+. files: Lib/test/regrtest.py | 1 - Lib/test/test_codecencodings_tw.py | 36 ++++++++++++++---- NEWS | 1 + 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1294,7 +1294,6 @@ test_codecencodings_iso2022 test_codecencodings_jp test_codecencodings_kr - test_codecencodings_tw # Fails in Java 8 not 7 test_codecmaps_cn test_codecmaps_jp test_codecmaps_kr diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py --- a/Lib/test/test_codecencodings_tw.py +++ b/Lib/test/test_codecencodings_tw.py @@ -7,18 +7,38 @@ from test import test_support from test import test_multibytecodec_support import unittest +import sys + +# Codecs re-synchronise sooner after illegal byte in Java 8+ than in Java 7 +# (and in CPython 3.3+ than in CPython 2.7-3.2). Either is correct, but we +# need to know which one to expect. +RESYNC_FASTER = sys.platform.startswith('java') and \ + sys.platform[4:7] > "1.7" class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'big5' tstring = test_multibytecodec_support.load_teststring('big5') - codectests = ( - # invalid bytes - ("abc\x80\x80\xc1\xc4", "strict", None), - ("abc\xc8", "strict", None), - ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u8b10"), - ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u8b10\ufffd"), - ("abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"), - ) + if RESYNC_FASTER: + # Version from CPython 3.6 where \0x80\0x80 is two invalid sequences. + # Java 8 agrees with this interpretation. + codectests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\xc8", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"), + ) + else: + # Standard version of test from CPython 2.7 + codectests = ( + # invalid bytes + ("abc\x80\x80\xc1\xc4", "strict", None), + ("abc\xc8", "strict", None), + ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u8b10"), + ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u8b10\ufffd"), + ("abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"), + ) def test_main(): test_support.run_unittest(__name__) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2571 ] Error handling in test_codecencodings_tw fails on Java 8 - [ 2524 ] datetime <-> time conversion incorrect in non UTC times - [ 2504 ] datetime.date.__tojava__ returns incorrect dates in non-UTC timezones with negative offset (Jython 2.7.0) -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Thu Mar 16 17:06:32 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Thu, 16 Mar 2017 21:06:32 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_=232309_=28see_PR_htt?= =?utf-8?q?ps=3A//github=2Ecom/jythontools/jython/pull/66=29?= Message-ID: <20170316210631.115649.92778.443E0ECA@psf.io> https://hg.python.org/jython/rev/e725a0b2d939 changeset: 8065:e725a0b2d939 user: James Mudd date: Thu Mar 16 22:06:21 2017 +0100 summary: Fixed #2309 (see PR https://github.com/jythontools/jython/pull/66) files: Lib/test/test_classpathimporter.py | 6 ++++-- NEWS | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_classpathimporter.py b/Lib/test/test_classpathimporter.py --- a/Lib/test/test_classpathimporter.py +++ b/Lib/test/test_classpathimporter.py @@ -26,13 +26,15 @@ # with sys.path.append where not getting scanned if they start with a top # level package we already have, like the "org" in org.python.* def test_bug1239(self): - with test_support.DirsOnSysPath("Lib/test/bug1239.jar"): + jar = test_support.findfile("bug1239.jar") + with test_support.DirsOnSysPath(jar): import org.test403javapackage.test403 # different from test_bug1239 in that only a Java package is imported, not # a Java class. I'd also like to get rid of this checked in test jar. def test_bug1126(self): - with test_support.DirsOnSysPath("Lib/test/bug1126/bug1126.jar"): + jar = test_support.findfile("bug1126.jar", subdir="bug1126") + with test_support.DirsOnSysPath(jar): import org.subpackage diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2309 ] test_classpathimporter fails on Windows. - [ 2318 ] test_zipimport_jy failure on Windows - [ 2571 ] Error handling in test_codecencodings_tw fails on Java 8 - [ 2568 ] test_threading intermittent failure -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 19 12:24:45 2017 From: jython-checkins at python.org (jeff.allen) Date: Sun, 19 Mar 2017 16:24:45 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Suppress_stack_dumps_from_t?= =?utf-8?q?est=5Fselect_and_test=5Fsocket_=28see_=232517=29=2E?= Message-ID: <20170319162444.115649.62524.77FDCF6C@psf.io> https://hg.python.org/jython/rev/aef41d53f55e changeset: 8066:aef41d53f55e parent: 8064:2c0ce625b440 user: Jeff Allen date: Sun Mar 19 11:02:43 2017 +0000 summary: Suppress stack dumps from test_select and test_socket (see #2517). Tests pass functionally. This change suppresses the stack dumps that appear to result from unsequenced tear-down. files: Lib/test/test_select.py | 14 +++++++++++++- Lib/test/test_socket.py | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -6,13 +6,18 @@ import select import socket import sys -import test_socket +from test import test_socket import unittest from test import test_support HOST = test_socket.HOST PORT = test_socket.PORT + 100 +if test_support.is_jython: + import java.util.logging + from test.test_socket import _set_java_logging + + class SelectWrapper: def __init__(self): @@ -179,6 +184,13 @@ def test_main(): + if test_support.is_jython: + # Netty logs stack dumps when we destroy sockets after their parent + # group, see http://bugs.jython.org/issue2517 . Is this a real bug? + # For now, treat as inconvenient artifact of test. + _set_java_logging("io.netty.channel.ChannelInitializer", + java.util.logging.Level.SEVERE) + tests = [TestSelectInvalidParameters, TestSelectClientSocket, TestPollClientSocket, diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -31,6 +31,10 @@ import _socket _socket._NUM_THREADS = 5 + import java.util.logging + def _set_java_logging(name, level): + java.util.logging.Logger.getLogger(name).setLevel(level) + class SocketTCPTest(unittest.TestCase): @@ -2620,6 +2624,16 @@ def test_main(): + + if test_support.is_jython: + # Netty logs stack dumps when we destroy sockets after their parent + # group, see http://bugs.jython.org/issue2517 . Is this a real bug? + # For now, treat as inconvenient artifact of test. + _set_java_logging("io.netty.channel.ChannelInitializer", + java.util.logging.Level.SEVERE) + _set_java_logging("io.netty.util.concurrent.DefaultPromise", + java.util.logging.Level.OFF) + tests = [ GeneralModuleTests, IPAddressTests, @@ -2672,5 +2686,6 @@ suites = [unittest.makeSuite(klass, 'test') for klass in tests] test_support._run_suite(unittest.TestSuite(suites)) + if __name__ == "__main__": test_main() -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 19 12:24:45 2017 From: jython-checkins at python.org (jeff.allen) Date: Sun, 19 Mar 2017 16:24:45 +0000 Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?= =?utf-8?q?=29=3A_Merge_to_trunk?= Message-ID: <20170319162445.115041.65876.4D8160B3@psf.io> https://hg.python.org/jython/rev/29faab36c068 changeset: 8068:29faab36c068 parent: 8065:e725a0b2d939 parent: 8067:44bf97c4eb3e user: Jeff Allen date: Sun Mar 19 16:24:21 2017 +0000 summary: Merge to trunk files: Lib/test/regrtest.py | 16 ++++++++++++++++ Lib/test/test_select.py | 14 +++++++++++++- Lib/test/test_socket.py | 15 +++++++++++++++ build.xml | 12 ++++++++++++ 4 files changed, 56 insertions(+), 1 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -398,6 +398,22 @@ failures.keep_only(tests) skips.keep_only(tests) + # Output some platform information. Loosely based on CPython 3.6 regrtest. + if (verbose or len(tests)>1) and not (quiet or single): + # Print basic platform information + for t in sys.version.splitlines(): + print "==", t + print "== platform:", sys.platform + print "== encodings: stdin=%s, stdout=%s, FS=%s" % ( + sys.stdin.encoding, sys.stdout.encoding, + sys.getfilesystemencoding()) + try: + import locale + print "== locale: default=%s, actual=%s" % ( + locale.getdefaultlocale(), locale.getlocale()) + except ImportError: + pass + for test in tests: if not quiet: print test diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -6,13 +6,18 @@ import select import socket import sys -import test_socket +from test import test_socket import unittest from test import test_support HOST = test_socket.HOST PORT = test_socket.PORT + 100 +if test_support.is_jython: + import java.util.logging + from test.test_socket import _set_java_logging + + class SelectWrapper: def __init__(self): @@ -179,6 +184,13 @@ def test_main(): + if test_support.is_jython: + # Netty logs stack dumps when we destroy sockets after their parent + # group, see http://bugs.jython.org/issue2517 . Is this a real bug? + # For now, treat as inconvenient artifact of test. + _set_java_logging("io.netty.channel.ChannelInitializer", + java.util.logging.Level.SEVERE) + tests = [TestSelectInvalidParameters, TestSelectClientSocket, TestPollClientSocket, diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -31,6 +31,10 @@ import _socket _socket._NUM_THREADS = 5 + import java.util.logging + def _set_java_logging(name, level): + java.util.logging.Logger.getLogger(name).setLevel(level) + class SocketTCPTest(unittest.TestCase): @@ -2620,6 +2624,16 @@ def test_main(): + + if test_support.is_jython: + # Netty logs stack dumps when we destroy sockets after their parent + # group, see http://bugs.jython.org/issue2517 . Is this a real bug? + # For now, treat as inconvenient artifact of test. + _set_java_logging("io.netty.channel.ChannelInitializer", + java.util.logging.Level.SEVERE) + _set_java_logging("io.netty.util.concurrent.DefaultPromise", + java.util.logging.Level.OFF) + tests = [ GeneralModuleTests, IPAddressTests, @@ -2672,5 +2686,6 @@ suites = [unittest.makeSuite(klass, 'test') for klass in tests] test_support._run_suite(unittest.TestSuite(suites)) + if __name__ == "__main__": test_main() diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -1116,11 +1116,18 @@ + + + + + + + @@ -1128,6 +1135,11 @@ + + + + + -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 19 12:24:46 2017 From: jython-checkins at python.org (jeff.allen) Date: Sun, 19 Mar 2017 16:24:46 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_build_target_regrtes?= =?utf-8?q?t-travis_to_exclude_problematic_tests=2E?= Message-ID: <20170319162445.43128.53222.55ACA6EC@psf.io> https://hg.python.org/jython/rev/44bf97c4eb3e changeset: 8067:44bf97c4eb3e user: Jeff Allen date: Sun Mar 19 14:08:26 2017 +0000 summary: Update build target regrtest-travis to exclude problematic tests. The intention is to get clean-running build bots. As an aid, this change includes confirmation of some platform details from regrtest itself. files: Lib/test/regrtest.py | 16 ++++++++++++++++ build.xml | 12 ++++++++++++ 2 files changed, 28 insertions(+), 0 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -398,6 +398,22 @@ failures.keep_only(tests) skips.keep_only(tests) + # Output some platform information. Loosely based on CPython 3.6 regrtest. + if (verbose or len(tests)>1) and not (quiet or single): + # Print basic platform information + for t in sys.version.splitlines(): + print "==", t + print "== platform:", sys.platform + print "== encodings: stdin=%s, stdout=%s, FS=%s" % ( + sys.stdin.encoding, sys.stdout.encoding, + sys.getfilesystemencoding()) + try: + import locale + print "== locale: default=%s, actual=%s" % ( + locale.getdefaultlocale(), locale.getlocale()) + except ImportError: + pass + for test in tests: if not quiet: print test diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -1116,11 +1116,18 @@ + + + + + + + @@ -1128,6 +1135,11 @@ + + + + + -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 19 20:10:33 2017 From: jython-checkins at python.org (jeff.allen) Date: Mon, 20 Mar 2017 00:10:33 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_More_precisely_specify_vers?= =?utf-8?q?ions_of_Java_in_test=5Fcodecencodings=5Ftw=2E?= Message-ID: <20170320001032.92049.58973.AAB62123@psf.io> https://hg.python.org/jython/rev/cc731a59c5eb changeset: 8069:cc731a59c5eb user: Jeff Allen date: Mon Mar 20 00:09:00 2017 +0000 summary: More precisely specify versions of Java in test_codecencodings_tw. Second attempt at fixing #2571. The change in coded behaviour seems to occur within Java 7, after 1.7.0_60, not between that and Java 8. We make test_support.get_java_version return the precise version in support of this test. files: Lib/test/test_codecencodings_tw.py | 14 +++++++++----- Lib/test/test_support.py | 16 +++++++++++++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py --- a/Lib/test/test_codecencodings_tw.py +++ b/Lib/test/test_codecencodings_tw.py @@ -9,11 +9,15 @@ import unittest import sys -# Codecs re-synchronise sooner after illegal byte in Java 8+ than in Java 7 -# (and in CPython 3.3+ than in CPython 2.7-3.2). Either is correct, but we -# need to know which one to expect. -RESYNC_FASTER = sys.platform.startswith('java') and \ - sys.platform[4:7] > "1.7" +# Codecs re-synchronise faster after illegal byte in Java 8+ than in Java 7 to +# update 60 (and in CPython 3.3+ faster than in CPython 2.7-3.2). Either is +# correct, but we need to know which one to expect. +RESYNC_FASTER = False # True for CPython 3.3 and later + +if sys.platform.startswith('java'): + if test_support.get_java_version() > (1, 7, 0, 60): + RESYNC_FASTER = True + class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'big5' diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -55,9 +55,19 @@ is_jython_posix = is_jython and (os._name == 'posix') if is_jython: - def get_java_version(): - # returns (1, 9) for Java 9, etc - return tuple((int(x) for x in platform.java_ver()[0].split('.')[0:2])) + def get_java_version(version=None): + # returns (1, 8, 0, 121) for version = "1.8.0_121", meaning + # Java 8 update 121, etc.. Conforms to: + # http://www.oracle.com/technetwork/java/javase/versioning-naming-139433.html + # and not yet http://openjdk.java.net/jeps/223 . + if version is None: + version = platform.java_ver()[0] + parse = re.match("(\d+)\.(\d+)\.(\d+)_(\d+)", version) + if parse: + return tuple((int(x) for x in parse.groups())) + else: + return () + class Error(Exception): """Base class for regression test exceptions.""" -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Wed Mar 22 05:19:57 2017 From: jython-checkins at python.org (jeff.allen) Date: Wed, 22 Mar 2017 09:19:57 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixes_=232399_test=5Fsort_f?= =?utf-8?q?ailure_on_Java_8?= Message-ID: <20170322091956.59489.44708.CCCD72FD@psf.io> https://hg.python.org/jython/rev/c677bb7414b1 changeset: 8070:c677bb7414b1 user: James Mudd date: Wed Mar 22 08:04:30 2017 +0000 summary: Fixes #2399 test_sort failure on Java 8 This approach caches the exception raised in the comparator and then allows the sort to continue. At the end it checks if any exceptions were raised and re-throws them. Also adds missing @Override in edited inner classes. files: NEWS | 1 + src/org/python/core/PyList.java | 60 +++++++++++++++++--- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/NEWS b/NEWS --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Jython 2.7.1rc1 Bugs fixed + - [ 2399 ] test_sort failure on Java 8 - [ 2309 ] test_classpathimporter fails on Windows. - [ 2318 ] test_zipimport_jy failure on Windows - [ 2571 ] Error handling in test_codecencodings_tw fails on Java 8 diff --git a/src/org/python/core/PyList.java b/src/org/python/core/PyList.java --- a/src/org/python/core/PyList.java +++ b/src/org/python/core/PyList.java @@ -802,7 +802,11 @@ if (reverse) { Collections.reverse(list); // maintain stability of sort by reversing first } - Collections.sort(list, new PyObjectDefaultComparator(this)); + final PyObjectDefaultComparator comparator = new PyObjectDefaultComparator(this); + Collections.sort(list, comparator); + if (comparator.raisedException()) { + throw comparator.getRaisedException(); + } if (reverse) { Collections.reverse(list); // maintain stability of sort by reversing first } @@ -812,20 +816,33 @@ private static class PyObjectDefaultComparator implements Comparator { private final PyList list; + private PyException comparatorException; PyObjectDefaultComparator(PyList list) { this.list = list; } + public PyException getRaisedException() { + return comparatorException; + } + + public boolean raisedException() { + return comparatorException != null; + } + + @Override public int compare(PyObject o1, PyObject o2) { // PEP 207 specifies that sort should only depend on "less-than" (Issue #1767) - int result; - if (o1._lt(o2).__nonzero__()) { - result = -1; - } else if (o2._lt(o1).__nonzero__()) { - result = 1; - } else { - result = 0; + int result = 0; // If exception is raised return objects are equal + try { + if (o1._lt(o2).__nonzero__()) { + result = -1; + } else if (o2._lt(o1).__nonzero__()) { + result = 1; + } + } catch (PyException pye) { + // #2399 Stash the exception so we can rethrow it later, and allow the sort to continue + comparatorException = pye; } if (this.list.gListAllocatedStatus >= 0) { throw Py.ValueError("list modified during sort"); @@ -833,6 +850,7 @@ return result; } + @Override public boolean equals(Object o) { if (o == this) { return true; @@ -853,8 +871,11 @@ if (reverse) { Collections.reverse(list); // maintain stability of sort by reversing first } - PyObjectComparator c = new PyObjectComparator(this, compare); - Collections.sort(list, c); + final PyObjectComparator comparator = new PyObjectComparator(this, compare); + Collections.sort(list, comparator); + if (comparator.raisedException()) { + throw comparator.getRaisedException(); + } if (reverse) { Collections.reverse(list); } @@ -865,20 +886,37 @@ private final PyList list; private final PyObject cmp; + private PyException comparatorException; PyObjectComparator(PyList list, PyObject cmp) { this.list = list; this.cmp = cmp; } + public PyException getRaisedException() { + return comparatorException; + } + + public boolean raisedException() { + return comparatorException != null; + } + + @Override public int compare(PyObject o1, PyObject o2) { - int result = cmp.__call__(o1, o2).asInt(); + int result = 0; // If exception is raised return objects are equal + try { + result = cmp.__call__(o1, o2).asInt(); + } catch (PyException pye) { + // #2399 Stash the exception so we can rethrow it later, and allow the sort to continue + comparatorException = pye; + } if (this.list.gListAllocatedStatus >= 0) { throw Py.ValueError("list modified during sort"); } return result; } + @Override public boolean equals(Object o) { if (o == this) { return true; -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Sun Mar 26 09:01:41 2017 From: jython-checkins at python.org (jeff.allen) Date: Sun, 26 Mar 2017 13:01:41 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Exclude_test=5Ftarfile_from?= =?utf-8?q?_regrtest_=28all_platforms=29=2E_=232574_refers=2E?= Message-ID: <20170326130141.124086.98653.54EAB9E9@psf.io> https://hg.python.org/jython/rev/f586a4ebf4ff changeset: 8071:f586a4ebf4ff user: Jeff Allen date: Sun Mar 26 09:03:37 2017 +0100 summary: Exclude test_tarfile from regrtest (all platforms). #2574 refers. files: Lib/test/regrtest.py | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1352,8 +1352,8 @@ # Unreliable tests test_asynchat # test_gc # Rare failures depending on timing of Java gc - test_logging # Hangs, though ok run singly - # test_tarfile # warning on irremovable directory (Windows) + test_logging # Hangs, though ok run singly. Issue #2536 + test_tarfile # flakey everywhere. Issue #2574 # test_urllib2net # unexpected output makes this a failure to regrtest.py # Failing tests here are because of lack of STARTTLS; see http://bugs.jython.org/issue2447 @@ -1374,7 +1374,6 @@ # test_popen # Passes, but see http://bugs.python.org/issue1559298 test_runpy # OSError: unlink() test_select_new # Hangs (Windows), though ok run singly - test_tarfile # flakey (Windows) test_urllib2 # file not on local host (likely Windows only) """, -- Repository URL: https://hg.python.org/jython From jython-checkins at python.org Fri Mar 31 09:47:29 2017 From: jython-checkins at python.org (stefan.richthofer) Date: Fri, 31 Mar 2017 13:47:29 +0000 Subject: [Jython-checkins] =?utf-8?q?jython=3A_Implemented_sys=2E=5Fcurren?= =?utf-8?b?dF9mcmFtZXMu?= Message-ID: <20170331134728.29276.13920.82588597@psf.io> https://hg.python.org/jython/rev/b051f30c4cd4 changeset: 8072:b051f30c4cd4 user: Stefan Richthofer date: Fri Mar 31 15:45:30 2017 +0200 summary: Implemented sys._current_frames. files: src/org/python/core/PySystemState.java | 4 + src/org/python/core/ThreadStateMapping.java | 21 ++++++++++ 2 files changed, 25 insertions(+), 0 deletions(-) diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java --- a/src/org/python/core/PySystemState.java +++ b/src/org/python/core/PySystemState.java @@ -1524,6 +1524,10 @@ return f; } + public static PyDictionary _current_frames() { + return ThreadStateMapping._current_frames(); + } + public void registerCloser(Callable resourceCloser) { closer.registerCloser(resourceCloser); } diff --git a/src/org/python/core/ThreadStateMapping.java b/src/org/python/core/ThreadStateMapping.java --- a/src/org/python/core/ThreadStateMapping.java +++ b/src/org/python/core/ThreadStateMapping.java @@ -88,4 +88,25 @@ scoped[1] = null; // allow corresponding PySystemState to be GCed } } + + @SuppressWarnings("unchecked") + private static Map.Entry[] entriesPrototype = new Map.Entry[0]; + public static PyDictionary _current_frames() { + Map.Entry[] entries = globalThreadStates.entrySet().toArray(entriesPrototype); + int i = 0; + for (Map.Entry entry: entries) { + if (entry.getValue().frame != null) { + ++i; + } + } + PyObject elements[] = new PyObject[i*2]; + i = 0; + for (Map.Entry entry: entries) { + if (entry.getValue().frame != null) { + elements[i++] = Py.newInteger(entry.getKey().getId()); + elements[i++] = entry.getValue().frame; + } + } + return new PyDictionary(elements); + } } -- Repository URL: https://hg.python.org/jython