From jython-checkins at python.org  Sun Nov  1 15:37:30 2015
From: jython-checkins at python.org (jim.baker)
Date: Sun, 01 Nov 2015 20:37:30 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Use_=5Fio_to_name_backing_m?=
 =?utf-8?q?odule_for_io=2C_vs_=5Fjyio=2E_Fixes_=232368=2E?=
Message-ID: <20151101203730.10172.99830@psf.io>

https://hg.python.org/jython/rev/8e135fff613a
changeset:   7789:8e135fff613a
user:        Jim Baker <jim.baker at rackspace.com>
date:        Sun Nov 01 13:37:19 2015 -0700
summary:
  Use _io to name backing module for io, vs _jyio. Fixes #2368.

Existing Python code out there expects that names that were defined in
_jyio to be available in _io. The easiest way to fix this issue is to
rename org.python.modules._io._io to org.python.modules._io._jyio; and
to rename Lib/_jyio.py to Lib/_io.py; and fix up various imports as
necessary.

files:
  Lib/_io.py                                     |  38 +++++----
  Lib/io.py                                      |  14 +-
  Lib/test/test_io_jy.py                         |  23 ++++++
  src/org/python/modules/Setup.java              |   2 +-
  src/org/python/modules/_io/PyIOBase.java       |   4 +-
  src/org/python/modules/_io/PyRawIOBase.java    |   6 +-
  src/org/python/modules/_io/_jyio.java          |   4 +-
  tests/java/org/python/modules/_io/_ioTest.java |  12 +--
  8 files changed, 63 insertions(+), 40 deletions(-)


diff --git a/Lib/_jyio.py b/Lib/_io.py
rename from Lib/_jyio.py
rename to Lib/_io.py
--- a/Lib/_jyio.py
+++ b/Lib/_io.py
@@ -1,14 +1,21 @@
-"""
-This is based on  _pyio.py from CPython 2.7 which is Python implementation of
+"""This is based on  _pyio.py from CPython 2.7 which is Python implementation of
 the io module. The upgrade from a 2.6-ish version accounts for the large
 number of changes made all at once.
 
-It is here to stand in for classes that should be provided by the _io module.
-In CPython 2.7, when client code imports io, that module imports a set of
-classes from _io and re-exports them as its own. In Jython, io.py imports
-those things from _jyio, which in turn imports from _io those so far
-implemented in Java. _jyio implements the rest here using nearly the same
-code as _pyio.
+It is here to stand in for classes that should be provided by the Java
+implementation of the _io module.  In CPython 2.7, when client code
+imports io, that module imports a set of classes from _io and
+re-exports them as its own. In Jython, io.py imports those things from
+_io, which in turn imports from _jyio those so far implemented in
+Java. Consequently _io implements the rest here using nearly the same
+code as _pyio. (Previous to Jython 2.7.1, the import was reversed:
+this specific Python-based module was named _jyio, and it imported
+from org.python.modules.io._io; although reasonable enough for Jython
+itself, we found that extant Python code expected that the _io module
+was the one defining various classes and constants. See
+http://bugs.jython.org/issue2368 for more background. If we ever get
+around to rewriting this module completely to Java, which is doubtful,
+this problem will go away.)
 
 Some classes have gained an underscore to match their _io module names:
 _IOBase, _RawIOBase, _BufferedIOBase, _TextIOBase.
@@ -16,12 +23,11 @@
 As Jython implements more and more of _io in Java, the Python implementations here
 will progressively be replaced with imports from _io. Eventually we should implement
 all this in Java, remove this module and revert io.py to its CPython original.
+
 """
 
 from __future__ import (print_function, unicode_literals)
 
-import _io  # Java implementations to replace this module
-
 import os
 import abc
 import codecs
@@ -41,7 +47,7 @@
 __metaclass__ = type
 
 # open() uses st_blksize whenever we can
-from _io import DEFAULT_BUFFER_SIZE
+from _jyio import DEFAULT_BUFFER_SIZE
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
 # defined in io.py.
@@ -58,7 +64,7 @@
         self.characters_written = characters_written
 
 
-from _io import (open, UnsupportedOperation, _IOBase, _RawIOBase, FileIO)
+from _jyio import (open, UnsupportedOperation, _IOBase, _RawIOBase, FileIO)
 
 
 class _BufferedIOBase(_IOBase):
@@ -253,9 +259,9 @@
         try:
             name = self.name
         except AttributeError:
-            return "<_jyio.{0}>".format(clsname)
+            return "<_io.{0}>".format(clsname)
         else:
-            return "<_jyio.{0} name={1!r}>".format(clsname, name)
+            return "<_io.{0} name={1!r}>".format(clsname, name)
 
     ### Lower-level APIs ###
 
@@ -1076,9 +1082,9 @@
         try:
             name = self.name
         except AttributeError:
-            return "<_jyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
+            return "<_io.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
-            return "<_jyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
+            return "<_io.TextIOWrapper name={0!r} encoding='{1}'>".format(
                 name, self.encoding)
 
     @property
diff --git a/Lib/io.py b/Lib/io.py
--- a/Lib/io.py
+++ b/Lib/io.py
@@ -66,15 +66,15 @@
 import abc
 
 # For the time being, import everything via _jyio instead of from _io directly
-import _jyio
-from _jyio import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
+import _io
+from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
                  open,
                  FileIO,
                  BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
                  IncrementalNewlineDecoder, TextIOWrapper)
 
-OpenWrapper = _jyio.open # for compatibility with _pyio
+OpenWrapper = _io.open # for compatibility with _pyio
 
 # for seek()
 SEEK_SET = 0
@@ -84,16 +84,16 @@
 # Declaring ABCs in C is tricky so we do it here.
 # Method descriptions and default implementations are inherited from the C
 # version however.
-class IOBase(_jyio._IOBase):
+class IOBase(_io._IOBase):
     __metaclass__ = abc.ABCMeta
 
-class RawIOBase(_jyio._RawIOBase, IOBase):
+class RawIOBase(_io._RawIOBase, IOBase):
     pass
 
-class BufferedIOBase(_jyio._BufferedIOBase, IOBase):
+class BufferedIOBase(_io._BufferedIOBase, IOBase):
     pass
 
-class TextIOBase(_jyio._TextIOBase, IOBase):
+class TextIOBase(_io._TextIOBase, IOBase):
     pass
 
 RawIOBase.register(FileIO)
diff --git a/Lib/test/test_io_jy.py b/Lib/test/test_io_jy.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_io_jy.py
@@ -0,0 +1,23 @@
+import unittest
+from test import test_support
+
+import _io
+
+class NameTest(unittest.TestCase):
+
+    def test_names_available_in__io_module(self):
+        # verifies fix for http://bugs.jython.org/issue2368
+        self.assertGreaterEqual(
+            set(dir(_io)),
+            { 'BlockingIOError', 'BufferedRWPair', 'BufferedRandom',
+              'BufferedReader', 'BufferedWriter', 'BytesIO',
+              'DEFAULT_BUFFER_SIZE', 'FileIO', 'IncrementalNewlineDecoder',
+              'TextIOWrapper', 'UnsupportedOperation',
+              '_BufferedIOBase', '_IOBase', '_RawIOBase', '_TextIOBase'
+            })
+
+def test_main():
+    test_support.run_unittest(NameTest)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/src/org/python/modules/Setup.java b/src/org/python/modules/Setup.java
--- a/src/org/python/modules/Setup.java
+++ b/src/org/python/modules/Setup.java
@@ -34,7 +34,7 @@
         "_csv:org.python.modules._csv._csv",
         "_functools:org.python.modules._functools._functools",
         "_hashlib",
-        "_io:org.python.modules._io._io",
+        "_jyio:org.python.modules._io._jyio",
         "_json:org.python.modules._json._json",
         "_jythonlib:org.python.modules._jythonlib._jythonlib",
         "_marshal",
diff --git a/src/org/python/modules/_io/PyIOBase.java b/src/org/python/modules/_io/PyIOBase.java
--- a/src/org/python/modules/_io/PyIOBase.java
+++ b/src/org/python/modules/_io/PyIOBase.java
@@ -95,7 +95,7 @@
     protected PyException unsupported(String op) {
         String fmt = "%s.%s() not supported";
         String msg = String.format(fmt, getType().fastGetName(), op);
-        return _io.UnsupportedOperation(msg);
+        return _jyio.UnsupportedOperation(msg);
     }
 
     @ExposedMethod(doc = "Internal: raise an exception for unsupported operations.")
@@ -437,7 +437,7 @@
      * Return a file descriptor for the stream. A CPython file descriptor is an int, but this is not
      * the natural choice in Jython, since Java has no such convention of using integers. File
      * descriptors should be passed around opaquely, so their actual type is irrelevant, as long as
-     * (say) {@link _io#open(PyObject[], String[])} accepts the type that {@link FileIO#fileno()}
+     * (say) {@link _jyio#open(PyObject[], String[])} accepts the type that {@link FileIO#fileno()}
      * returns.
      *
      * @return a file descriptor (as opaque object)
diff --git a/src/org/python/modules/_io/PyRawIOBase.java b/src/org/python/modules/_io/PyRawIOBase.java
--- a/src/org/python/modules/_io/PyRawIOBase.java
+++ b/src/org/python/modules/_io/PyRawIOBase.java
@@ -134,7 +134,7 @@
         PyObject readMethod = __getattr__("read");
 
         // Quite often, a single read operation will do the trick
-        PyObject prev = readMethod.__call__(_io.DEFAULT_BUFFER_SIZE);
+        PyObject prev = readMethod.__call__(_jyio.DEFAULT_BUFFER_SIZE);
 
         if (!prev.__nonzero__()) {
             // Nothing on the first read: that means we're done
@@ -142,7 +142,7 @@
 
         } else {
             // Try a second read
-            PyObject curr = readMethod.__call__(_io.DEFAULT_BUFFER_SIZE);
+            PyObject curr = readMethod.__call__(_jyio.DEFAULT_BUFFER_SIZE);
             if (!curr.__nonzero__()) {
                 // Nothing on the second read: the result is just the first one
                 return prev;
@@ -155,7 +155,7 @@
                 // Accumulate the current read result and get another, until we run out of bytes.
                 do {
                     list.add(curr);
-                    curr = readMethod.__call__(_io.DEFAULT_BUFFER_SIZE);
+                    curr = readMethod.__call__(_jyio.DEFAULT_BUFFER_SIZE);
                 } while (curr.__nonzero__());
 
                 // Stitch it all together
diff --git a/src/org/python/modules/_io/_io.java b/src/org/python/modules/_io/_jyio.java
rename from src/org/python/modules/_io/_io.java
rename to src/org/python/modules/_io/_jyio.java
--- a/src/org/python/modules/_io/_io.java
+++ b/src/org/python/modules/_io/_jyio.java
@@ -16,7 +16,7 @@
 /**
  * The Python _io module implemented in Java.
  */
-public class _io implements ClassDictInit {
+public class _jyio implements ClassDictInit {
 
     /**
      * This method is called when the module is loaded, to populate the namespace (dictionary) of
@@ -26,7 +26,7 @@
      * @param dict namespace of the module
      */
     public static void classDictInit(PyObject dict) {
-        dict.__setitem__("__name__", new PyString("_io"));
+        dict.__setitem__("__name__", new PyString("_jyio"));
         dict.__setitem__("__doc__", new PyString(__doc__));
         dict.__setitem__("DEFAULT_BUFFER_SIZE", DEFAULT_BUFFER_SIZE);
 
diff --git a/tests/java/org/python/modules/_io/_ioTest.java b/tests/java/org/python/modules/_io/_ioTest.java
--- a/tests/java/org/python/modules/_io/_ioTest.java
+++ b/tests/java/org/python/modules/_io/_ioTest.java
@@ -5,15 +5,10 @@
 import static org.junit.matchers.JUnitMatchers.*;
 
 import java.io.File;
-import java.io.FileDescriptor;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.FileChannel;
-import java.nio.channels.WritableByteChannel;
 import java.util.Arrays;
 
 import org.junit.Before;
@@ -24,7 +19,6 @@
 import org.python.core.PyObject;
 import org.python.core.PyStringMap;
 import org.python.core.PySystemState;
-import org.python.core.imp;
 import org.python.core.io.RawIOBase;
 import org.python.util.PythonInterpreter;
 
@@ -78,7 +72,7 @@
         interp.exec("import io");
 
         // There should be a helper function
-        PyException pye = _io.UnsupportedOperation("Message from _ioTest");
+        PyException pye = _jyio.UnsupportedOperation("Message from _ioTest");
         PyObject type = pye.type;
         String repr = type.toString();
         assertEquals("Class name", "<class '_io.UnsupportedOperation'>", repr);
@@ -102,7 +96,7 @@
             interp.exec("raise _io.UnsupportedOperation()");
             fail("_io.UnsupportedOperation not raised when expected");
         } catch (PyException e) {
-            assertEquals(_io.UnsupportedOperation, e.type);
+            assertEquals(_jyio.UnsupportedOperation, e.type);
         }
     }
 
@@ -157,7 +151,7 @@
         RawIOBase fd = (RawIOBase)pyfd.__tojava__(RawIOBase.class);
         PyObject[] args = new PyObject[] {pyfd, Py.newString(mode), Py.False};
         String[] kwds = {"closefd"};
-        PyObject file2 = _io.open(args, kwds);
+        PyObject file2 = _jyio.open(args, kwds);
         file2.invoke("close");
     }
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov  1 19:05:55 2015
From: jython-checkins at python.org (jeff.allen)
Date: Mon, 02 Nov 2015 00:05:55 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_expected_failures_to_re?=
 =?utf-8?q?grtest=2Epy_for_Posix_platforms=2E?=
Message-ID: <20151102000550.8611.31347@psf.io>

https://hg.python.org/jython/rev/0c7f3b5e2d06
changeset:   7790:0c7f3b5e2d06
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sun Nov 01 23:19:24 2015 +0000
summary:
  Add expected failures to regrtest.py for Posix platforms.

Complementary change set to 7b1ad371113e suppressing tests found to fail on
Ubuntu 14.04, and a group of tests that (without themselves failing) cause file
handle exhaustion (issue #2420).

files:
  Lib/test/regrtest.py |  19 +++++++++++++++++++
  1 files changed, 19 insertions(+), 0 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1364,6 +1364,25 @@
         test_runpy             # OSError: unlink()
         test_urllib2           # file not on local host (likely Windows only)
         """,
+
+    'java.posix':   # Expected to fail on Linux
+        """
+        test_classpathimporter  # Fails in test_loader_get_code
+        test_codecencodings_tw  # Fails in test_multibytecodec_support.py
+        test_jython_launcher    # /usr/bin/env: python2.7 -E: No such file or directory
+
+        # These leak file handles on a grand scale (observed on Ubuntu 14.04),
+        # causing failures elsewhere, but they don't actually fail.
+        test_docxmlrpc          #  206 leaked handles issue 2320
+        test_httpservers        #  721 leaked handles issue 2320
+        ## test_imaplib            #   92 leaked handles issue 2320 (tolerable)
+        test_socketserver       # 1344 leaked handles issue 2320
+        test_telnetlib          # 1588 leaked handles issue 2320
+        ## test_timeout            #  123 leaked handles issue 2320 (tolerable)
+        test_urllib2_localnet   #  763 leaked handles issue 2320
+        test_xmlrpc             #  453 leaked handles issue 2320
+
+        """
 }
 
 _platform = sys.platform

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Mon Nov  2 10:45:50 2015
From: jython-checkins at python.org (jim.baker)
Date: Mon, 02 Nov 2015 15:45:50 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_FIXME_updates_for_Jython_te?=
 =?utf-8?q?sts=2E_Minor_change_for_=231861=2E?=
Message-ID: <20151102154550.12549.40196@psf.io>

https://hg.python.org/jython/rev/159a0fc6d562
changeset:   7791:159a0fc6d562
user:        Jim Baker <jim.baker at rackspace.com>
date:        Mon Nov 02 08:45:44 2015 -0700
summary:
  FIXME updates for Jython tests. Minor change for #1861.

files:
  Lib/test/test_codecs.py |  5 ++---
  1 files changed, 2 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -602,7 +602,7 @@
     def test_bad_args(self):
         self.assertRaises(TypeError, codecs.utf_16_ex_decode)
 
- at unittest.skipIf(test_support.is_jython, "Jython has no _codecs.readbuffer_encode method")
+ at unittest.skipIf(test_support.is_jython, "FIXME Jython has no _codecs.readbuffer_encode method")
 class ReadBufferTest(unittest.TestCase):
 
     def test_array(self):
@@ -619,7 +619,7 @@
         self.assertRaises(TypeError, codecs.readbuffer_encode)
         self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
 
- at unittest.skipIf(test_support.is_jython, "Jython has no _codecs.charbuffer_encode method")
+ at unittest.skipIf(test_support.is_jython, "FIXME Jython has no _codecs.charbuffer_encode method")
 class CharBufferTest(unittest.TestCase):
 
     def test_string(self):
@@ -1084,7 +1084,6 @@
                 except Exception,e:
                     raise test_support.TestFailed("Test 3.%d: %s" % (pos+1, str(e)))
 
-# @unittest.skipIf(test_support.is_jython, "FIXME: Jython issue 1153 missing support for IDNA")
 class IDNACodecTest(unittest.TestCase):
     def test_builtin_decode(self):
         self.assertEqual(unicode("python.org", "idna"), u"python.org")

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov  8 17:19:03 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sun, 08 Nov 2015 22:19:03 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Declare_ssl=2EHAS=5F*_accor?=
 =?utf-8?q?dingly?=
Message-ID: <20151108221902.16157.29979@psf.io>

https://hg.python.org/jython/rev/c4e71b10bb23
changeset:   7792:c4e71b10bb23
user:        Darjus Loktevic <darjus at gmail.com>
date:        Mon Nov 09 09:18:54 2015 +1100
summary:
  Declare ssl.HAS_* accordingly

files:
  Lib/ssl.py |  5 +++++
  1 files changed, 5 insertions(+), 0 deletions(-)


diff --git a/Lib/ssl.py b/Lib/ssl.py
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -65,6 +65,11 @@
 
 OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1 = range(4)
 
+
+# https://docs.python.org/2/library/ssl.html#ssl.HAS_ALPN etc...
+HAS_ALPN, HAS_NPN, HAS_ECDH, HAS_SNI = False, False, True, True
+
+
 _rfc2822_date_format = SimpleDateFormat("MMM dd HH:mm:ss yyyy z", Locale.US)
 _rfc2822_date_format.setTimeZone(TimeZone.getTimeZone("GMT"))
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov  8 17:58:19 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sun, 08 Nov 2015 22:58:19 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improve_ssl=2ESSLContext=2E?=
 =?utf-8?q?=5Fparse=5Fdn_to_support_quoted_names?=
Message-ID: <20151108225818.58395.99807@psf.io>

https://hg.python.org/jython/rev/c1ff0eb2cec6
changeset:   7793:c1ff0eb2cec6
user:        Darjus Loktevic <darjus at gmail.com>
date:        Mon Nov 09 09:58:09 2015 +1100
summary:
  Improve ssl.SSLContext._parse_dn to support quoted names

files:
  Lib/ssl.py |  21 ++++++++++++++-------
  1 files changed, 14 insertions(+), 7 deletions(-)


diff --git a/Lib/ssl.py b/Lib/ssl.py
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -7,6 +7,7 @@
 from java.security import KeyStore
 from java.security.cert import CertificateParsingException
 from javax.net.ssl import TrustManagerFactory
+from javax.naming.ldap import LdapName
 import logging
 import os.path
 import textwrap
@@ -38,7 +39,7 @@
 from _sslcerts import SSLContext as _JavaSSLContext
 
 from java.text import SimpleDateFormat
-from java.util import ArrayList, Locale, TimeZone
+from java.util import ArrayList, Locale, TimeZone, NoSuchElementException
 from java.util.concurrent import CountDownLatch
 from javax.naming.ldap import LdapName
 from javax.security.auth.x500 import X500Principal
@@ -588,11 +589,17 @@
 
     @classmethod
     def _parse_dn(cls, dn):
+        dn_lst = []
+
+        ln = LdapName(unicode(dn))
+        ln_iter = ln.getAll()
         try:
-            dn_dct = dict([iss.split('=', 1) for iss in unicode(dn).split(',')])
-        except ValueError:
-            # FIXME CN=Starfield Root Certificate Authority - G2, O="Starfield Technologies, Inc.",
-            log.error("Failed to parse {}".format(dn), exc_info=True)
-            return tuple()
+            ln_value = ln_iter.nextElement()
+            while ln_value:
+                dn_lst.append(tuple(ln_value.split('=', 1)))
 
-        return tuple((cls._DN_TO_CPY.get(key.strip(), 'unk'), val) for key, val in dn_dct.iteritems())
+                ln_value = ln_iter.nextElement()
+        except NoSuchElementException:
+            pass
+
+        return tuple(dn_lst)

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov  8 18:04:44 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sun, 08 Nov 2015 23:04:44 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Cleanup_SSLContext?=
Message-ID: <20151108230444.10006.38582@psf.io>

https://hg.python.org/jython/rev/fa0239372f0e
changeset:   7794:fa0239372f0e
user:        Darjus Loktevic <darjus at gmail.com>
date:        Mon Nov 09 10:04:37 2015 +1100
summary:
  Cleanup SSLContext

files:
  Lib/ssl.py |  3 ---
  1 files changed, 0 insertions(+), 3 deletions(-)


diff --git a/Lib/ssl.py b/Lib/ssl.py
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -441,9 +441,6 @@
 
 
 class SSLContext(object):
-    _DN_TO_CPY = {'CN': 'commonName', 'O': 'commonOrganization', 'C': 'countryName', 'DC': 'domainComponent',
-                  'SN': 'surname', 'GN': 'givenName', 'OU': 'organizationalUnitName', 'ST': 'stateOrProvinceName',
-                  'L': 'localityName', 'SERIALNUMBER': 'serialNumber', 'EMAILADDRESS': 'emailAddress'}
 
     def __init__(self, protocol):
         protocol_name = _PROTOCOL_NAMES[protocol]

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Tue Nov 10 14:07:25 2015
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 10 Nov 2015 19:07:25 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Ensure_os=2Efstat=28=29_tre?=
 =?utf-8?q?ats_fileno=28=29_object_as_valid_in_Windows=2E?=
Message-ID: <20151110190724.33546.94839@psf.io>

https://hg.python.org/jython/rev/a01cd923262f
changeset:   7797:a01cd923262f
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Mon Nov 09 21:00:09 2015 +0000
summary:
  Ensure os.fstat() treats fileno() object as valid in Windows.

The JNR fstat() function is now called correctly, but as it is still
buggy, we have to exclude test_fileinput for now. Another small step
towards fixing #2320.

files:
  Lib/test/regrtest.py                          |  1 +
  Lib/test/test_tempfile.py                     |  5 ++++-
  src/org/python/modules/posix/PosixModule.java |  2 ++
  3 files changed, 7 insertions(+), 1 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1358,6 +1358,7 @@
 
     'java.nt':     # Expected to fail on Windows
         """
+        test_fileinput         # issue 2320 (because fstat().st_mode is zero)
         test_mailbox           # fails miserably and ruins other tests
         test_os_jy             # Locale tests run and fail on Cygwin
         test_popen             # http://bugs.python.org/issue1559298
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -848,7 +848,10 @@
         # A SpooledTemporaryFile should roll over to a real file on fileno()
         f = self.do_create(max_size=30)
         self.assertFalse(f._rolled)
-        self.assertTrue(f.fileno() > 0)
+        if support.is_jython:
+            self.assertIsNotNone(f.fileno())
+        else:
+            self.assertGreater(f.fileno(), 0)
         self.assertTrue(f._rolled)
 
     def test_multiple_close_before_rollover(self):
diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -222,6 +222,8 @@
         if (fdObj.isInteger()) {
             int intFd = fdObj.asInt();
             switch (intFd) {
+                case -1:
+                    break;
                 case 0:
                     return new FDUnion(FileDescriptor.in);
                 case 1:

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Tue Nov 10 14:07:25 2015
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 10 Nov 2015 19:07:25 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_jnr-posix_to_help_wi?=
 =?utf-8?b?dGggIzIzMjAu?=
Message-ID: <20151110190724.58409.46991@psf.io>

https://hg.python.org/jython/rev/60e8cd0fc6c6
changeset:   7796:60e8cd0fc6c6
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Nov 07 16:01:03 2015 +0000
summary:
  Update jnr-posix to help with #2320.

files:
  build.xml                       |    8 ++++----
  extlibs/jnr-constants-0.8.8.jar |  Bin 
  extlibs/jnr-constants-0.9.0.jar |  Bin 
  extlibs/jnr-posix-3.0.17.jar    |  Bin 
  extlibs/jnr-posix-3.0.20.jar    |  Bin 
  5 files changed, 4 insertions(+), 4 deletions(-)


diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -174,8 +174,8 @@
             <pathelement path="${extlibs.dir}/jffi-1.2.7.jar"/>
             <pathelement path="${extlibs.dir}/jnr-ffi-2.0.4.jar"/>
             <pathelement path="${extlibs.dir}/jnr-netdb-1.1.4.jar"/>
-            <pathelement path="${extlibs.dir}/jnr-posix-3.0.17.jar"/>
-            <pathelement path="${extlibs.dir}/jnr-constants-0.8.8.jar"/>
+            <pathelement path="${extlibs.dir}/jnr-posix-3.0.20.jar"/>
+            <pathelement path="${extlibs.dir}/jnr-constants-0.9.0.jar"/>
             <pathelement path="${extlibs.dir}/jline-2.12.1.jar"/>
             <pathelement path="${extlibs.dir}/netty-buffer-4.0.31.Final.jar"/>
             <pathelement path="${extlibs.dir}/netty-codec-4.0.31.Final.jar"/>
@@ -618,8 +618,8 @@
             <zipfileset src="extlibs/jffi-1.2.7.jar"/>
             <zipfileset src="${extlibs.dir}/jnr-ffi-2.0.4.jar"/>
             <zipfileset src="${extlibs.dir}/jnr-netdb-1.1.4.jar"/>
-            <zipfileset src="${extlibs.dir}/jnr-posix-3.0.17.jar"/>
-            <zipfileset src="${extlibs.dir}/jnr-constants-0.8.8.jar"/>
+            <zipfileset src="${extlibs.dir}/jnr-posix-3.0.20.jar"/>
+            <zipfileset src="${extlibs.dir}/jnr-constants-0.9.0.jar"/>
             <zipfileset src="extlibs/xml-apis-2.11.0.jar" excludes="META-INF/services/*"/>
             <zipfileset src="extlibs/xercesImpl-2.11.0.jar" excludes="META-INF/services/*"/>
             <rule pattern="org.apache.xml.**" result="org.python.apache.xml. at 1"/>
diff --git a/extlibs/jnr-constants-0.8.8.jar b/extlibs/jnr-constants-0.8.8.jar
deleted file mode 100644
index eee5ab879c62b3a5354ca8d2e9b9564b55122be0..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/jnr-constants-0.9.0.jar b/extlibs/jnr-constants-0.9.0.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..52040d3c89fb843129c7e566a1e8dab0a7f05d81
GIT binary patch
[stripped]
diff --git a/extlibs/jnr-posix-3.0.17.jar b/extlibs/jnr-posix-3.0.17.jar
deleted file mode 100644
index 47694b9c819436d6f39633d13d1dc6c071ce4a33..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/jnr-posix-3.0.20.jar b/extlibs/jnr-posix-3.0.20.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f0c5961c380097cb98754976e37f33da0c50303f
GIT binary patch
[stripped]

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Tue Nov 10 14:07:25 2015
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 10 Nov 2015 19:07:25 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_a_skip_for_isolated_fai?=
 =?utf-8?q?lure_in_test=5Fclasspathimporter_=28=232422=29?=
Message-ID: <20151110190724.33528.49788@psf.io>

https://hg.python.org/jython/rev/fa60627c599f
changeset:   7795:fa60627c599f
parent:      7791:159a0fc6d562
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Fri Nov 06 20:41:38 2015 +0000
summary:
  Add a skip for isolated failure in test_classpathimporter (#2422)

Also updates expected failures in regrtest and corrects a typo.

files:
  Lib/test/regrtest.py               |  17 ++++++++---------
  Lib/test/test_classpathimporter.py |   1 +
  Lib/test/test_support.py           |   1 +
  3 files changed, 10 insertions(+), 9 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1367,20 +1367,19 @@
 
     'java.posix':   # Expected to fail on Linux
         """
-        test_classpathimporter  # Fails in test_loader_get_code
         test_codecencodings_tw  # Fails in test_multibytecodec_support.py
         test_jython_launcher    # /usr/bin/env: python2.7 -E: No such file or directory
 
         # These leak file handles on a grand scale (observed on Ubuntu 14.04),
         # causing failures elsewhere, but they don't actually fail.
-        test_docxmlrpc          #  206 leaked handles issue 2320
-        test_httpservers        #  721 leaked handles issue 2320
-        ## test_imaplib            #   92 leaked handles issue 2320 (tolerable)
-        test_socketserver       # 1344 leaked handles issue 2320
-        test_telnetlib          # 1588 leaked handles issue 2320
-        ## test_timeout            #  123 leaked handles issue 2320 (tolerable)
-        test_urllib2_localnet   #  763 leaked handles issue 2320
-        test_xmlrpc             #  453 leaked handles issue 2320
+        test_docxmlrpc          #  206 leaked handles issue 2420
+        test_httpservers        #  721 leaked handles issue 2420
+        ## test_imaplib            #   92 leaked handles issue 2420 (tolerable)
+        test_socketserver       # 1344 leaked handles issue 2420
+        test_telnetlib          # 1588 leaked handles issue 2420
+        ## test_timeout            #  123 leaked handles issue 2420 (tolerable)
+        test_urllib2_localnet   #  763 leaked handles issue 2420
+        test_xmlrpc             #  453 leaked handles issue 2420
 
         """
 }
diff --git a/Lib/test/test_classpathimporter.py b/Lib/test/test_classpathimporter.py
--- a/Lib/test/test_classpathimporter.py
+++ b/Lib/test/test_classpathimporter.py
@@ -118,6 +118,7 @@
         self.assertTrue(loader.is_package('jar_pkg'))
         self.assertFalse(loader.is_package('jar_pkg.prefer_compiled'))
 
+    @unittest.skipIf(test_support.is_jython_posix, "FIXME: failing on Linux issue #2422")
     def test_loader_get_code(self):
         # Execute Python code out of the JAR
         jar = self.prepareJar('classimport.jar')
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -52,6 +52,7 @@
 # We use these extensively in adapting the regression tests for Jython
 is_jython = sys.platform.startswith('java')
 is_jython_nt = is_jython and (os._name == 'nt')
+is_jython_posix = is_jython and (os._name == 'posix')
 
 class Error(Exception):
     """Base class for regression test exceptions."""

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Tue Nov 10 14:07:25 2015
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 10 Nov 2015 19:07:25 +0000
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge_to_trunk?=
Message-ID: <20151110190725.17063.55238@psf.io>

https://hg.python.org/jython/rev/f7db3ab5ca7b
changeset:   7799:f7db3ab5ca7b
parent:      7794:fa0239372f0e
parent:      7798:0381d60ca611
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Tue Nov 10 19:06:30 2015 +0000
summary:
  Merge to trunk

files:
  Lib/test/regrtest.py                          |   17 +-
  Lib/test/test_classpathimporter.py            |    1 +
  Lib/test/test_support.py                      |    1 +
  Lib/test/test_tempfile.py                     |    5 +-
  build.xml                                     |    8 +-
  extlibs/jnr-constants-0.8.8.jar               |  Bin 
  extlibs/jnr-constants-0.9.0.jar               |  Bin 
  extlibs/jnr-posix-3.0.17.jar                  |  Bin 
  extlibs/jnr-posix-3.0.20.jar                  |  Bin 
  src/org/python/modules/posix/PosixModule.java |   68 ++++++++-
  10 files changed, 79 insertions(+), 21 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1367,20 +1367,19 @@
 
     'java.posix':   # Expected to fail on Linux
         """
-        test_classpathimporter  # Fails in test_loader_get_code
         test_codecencodings_tw  # Fails in test_multibytecodec_support.py
         test_jython_launcher    # /usr/bin/env: python2.7 -E: No such file or directory
 
         # These leak file handles on a grand scale (observed on Ubuntu 14.04),
         # causing failures elsewhere, but they don't actually fail.
-        test_docxmlrpc          #  206 leaked handles issue 2320
-        test_httpservers        #  721 leaked handles issue 2320
-        ## test_imaplib            #   92 leaked handles issue 2320 (tolerable)
-        test_socketserver       # 1344 leaked handles issue 2320
-        test_telnetlib          # 1588 leaked handles issue 2320
-        ## test_timeout            #  123 leaked handles issue 2320 (tolerable)
-        test_urllib2_localnet   #  763 leaked handles issue 2320
-        test_xmlrpc             #  453 leaked handles issue 2320
+        test_docxmlrpc          #  206 leaked handles issue 2420
+        test_httpservers        #  721 leaked handles issue 2420
+        ## test_imaplib            #   92 leaked handles issue 2420 (tolerable)
+        test_socketserver       # 1344 leaked handles issue 2420
+        test_telnetlib          # 1588 leaked handles issue 2420
+        ## test_timeout            #  123 leaked handles issue 2420 (tolerable)
+        test_urllib2_localnet   #  763 leaked handles issue 2420
+        test_xmlrpc             #  453 leaked handles issue 2420
 
         """
 }
diff --git a/Lib/test/test_classpathimporter.py b/Lib/test/test_classpathimporter.py
--- a/Lib/test/test_classpathimporter.py
+++ b/Lib/test/test_classpathimporter.py
@@ -118,6 +118,7 @@
         self.assertTrue(loader.is_package('jar_pkg'))
         self.assertFalse(loader.is_package('jar_pkg.prefer_compiled'))
 
+    @unittest.skipIf(test_support.is_jython_posix, "FIXME: failing on Linux issue #2422")
     def test_loader_get_code(self):
         # Execute Python code out of the JAR
         jar = self.prepareJar('classimport.jar')
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -52,6 +52,7 @@
 # We use these extensively in adapting the regression tests for Jython
 is_jython = sys.platform.startswith('java')
 is_jython_nt = is_jython and (os._name == 'nt')
+is_jython_posix = is_jython and (os._name == 'posix')
 
 class Error(Exception):
     """Base class for regression test exceptions."""
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -848,7 +848,10 @@
         # A SpooledTemporaryFile should roll over to a real file on fileno()
         f = self.do_create(max_size=30)
         self.assertFalse(f._rolled)
-        self.assertTrue(f.fileno() > 0)
+        if support.is_jython:
+            self.assertIsNotNone(f.fileno())
+        else:
+            self.assertGreater(f.fileno(), 0)
         self.assertTrue(f._rolled)
 
     def test_multiple_close_before_rollover(self):
diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -174,8 +174,8 @@
             <pathelement path="${extlibs.dir}/jffi-1.2.7.jar"/>
             <pathelement path="${extlibs.dir}/jnr-ffi-2.0.4.jar"/>
             <pathelement path="${extlibs.dir}/jnr-netdb-1.1.4.jar"/>
-            <pathelement path="${extlibs.dir}/jnr-posix-3.0.17.jar"/>
-            <pathelement path="${extlibs.dir}/jnr-constants-0.8.8.jar"/>
+            <pathelement path="${extlibs.dir}/jnr-posix-3.0.20.jar"/>
+            <pathelement path="${extlibs.dir}/jnr-constants-0.9.0.jar"/>
             <pathelement path="${extlibs.dir}/jline-2.12.1.jar"/>
             <pathelement path="${extlibs.dir}/netty-buffer-4.0.31.Final.jar"/>
             <pathelement path="${extlibs.dir}/netty-codec-4.0.31.Final.jar"/>
@@ -618,8 +618,8 @@
             <zipfileset src="extlibs/jffi-1.2.7.jar"/>
             <zipfileset src="${extlibs.dir}/jnr-ffi-2.0.4.jar"/>
             <zipfileset src="${extlibs.dir}/jnr-netdb-1.1.4.jar"/>
-            <zipfileset src="${extlibs.dir}/jnr-posix-3.0.17.jar"/>
-            <zipfileset src="${extlibs.dir}/jnr-constants-0.8.8.jar"/>
+            <zipfileset src="${extlibs.dir}/jnr-posix-3.0.20.jar"/>
+            <zipfileset src="${extlibs.dir}/jnr-constants-0.9.0.jar"/>
             <zipfileset src="extlibs/xml-apis-2.11.0.jar" excludes="META-INF/services/*"/>
             <zipfileset src="extlibs/xercesImpl-2.11.0.jar" excludes="META-INF/services/*"/>
             <rule pattern="org.apache.xml.**" result="org.python.apache.xml. at 1"/>
diff --git a/extlibs/jnr-constants-0.8.8.jar b/extlibs/jnr-constants-0.8.8.jar
deleted file mode 100644
index eee5ab879c62b3a5354ca8d2e9b9564b55122be0..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/jnr-constants-0.9.0.jar b/extlibs/jnr-constants-0.9.0.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..52040d3c89fb843129c7e566a1e8dab0a7f05d81
GIT binary patch
[stripped]
diff --git a/extlibs/jnr-posix-3.0.17.jar b/extlibs/jnr-posix-3.0.17.jar
deleted file mode 100644
index 47694b9c819436d6f39633d13d1dc6c071ce4a33..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/jnr-posix-3.0.20.jar b/extlibs/jnr-posix-3.0.20.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..f0c5961c380097cb98754976e37f33da0c50303f
GIT binary patch
[stripped]
diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -15,8 +15,8 @@
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.NotLinkException;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributeView;
@@ -34,9 +34,11 @@
 import jnr.posix.FileStat;
 import jnr.posix.POSIX;
 import jnr.posix.POSIXFactory;
+import jnr.posix.POSIXHandler;
 import jnr.posix.Times;
+import jnr.posix.WindowsRawFileStat;
 import jnr.posix.util.FieldAccess;
-import jnr.posix.util.Platform;
+import jnr.posix.windows.CommonFileInformation;
 
 import org.python.core.BufferProtocol;
 import org.python.core.ClassDictInit;
@@ -53,8 +55,8 @@
 import org.python.core.PyString;
 import org.python.core.PySystemState;
 import org.python.core.PyTuple;
+import org.python.core.Untraversable;
 import org.python.core.imp;
-import org.python.core.Untraversable;
 import org.python.core.io.FileIO;
 import org.python.core.io.IOBase;
 import org.python.core.io.RawIOBase;
@@ -75,7 +77,8 @@
     private static final OS os = OS.getOS();
 
     /** Platform specific POSIX services. */
-    private static final POSIX posix = POSIXFactory.getPOSIX(new PythonPOSIXHandler(), true);
+    private static final POSIXHandler posixHandler = new PythonPOSIXHandler();
+    private static final POSIX posix = POSIXFactory.getPOSIX(posixHandler, true);
 
     /** os.open flags. */
     private static final int O_RDONLY = 0x0;
@@ -222,6 +225,8 @@
         if (fdObj.isInteger()) {
             int intFd = fdObj.asInt();
             switch (intFd) {
+                case -1:
+                    break;
                 case 0:
                     return new FDUnion(FileDescriptor.in);
                 case 1:
@@ -352,6 +357,7 @@
         }
     }
 
+    @Hide(OS.NT)
     public static void closerange(PyObject fd_lowObj, PyObject fd_highObj) {
         int fd_low = getFD(fd_lowObj).getIntFD(false);
         int fd_high = getFD(fd_highObj).getIntFD(false);
@@ -1363,10 +1369,18 @@
             try {
                 FDUnion fd = getFD(fdObj);
                 FileStat stat;
-                if (fd.isIntFD()) {
-                    stat = posix.fstat(fd.intFD);
+                if (os != OS.NT) {
+                    if (fd.isIntFD()) {
+                        stat = posix.fstat(fd.intFD);
+                    } else {
+                        stat = posix.fstat(fd.javaFD);
+                    }
                 } else {
-                    stat = posix.fstat(fd.javaFD);
+                    // FIXME: jnr-posix fstat work-around. See issue #2320.
+                    stat = new WindowsRawFileStat2(posix, posixHandler);
+                    if (posix.fstat(fd.javaFD, stat) < 0) {
+                        throw Py.OSError(Errno.EBADF);
+                    }
                 }
                 return PyStatResult.fromFileStat(stat);
             } catch (PyException ex) {
@@ -1374,4 +1388,44 @@
             }
         }
     }
+
+    /*
+     * Extend the Windows stat object defined by jnr.posix, which in jnr-posix 2.0.4 is buggy to the
+     * extent that st_mode is always zero. Remarkably, it is possible to fix this by defining a
+     * cunning replacement.
+     */
+    private static class WindowsRawFileStat2 extends WindowsRawFileStat {
+
+        public WindowsRawFileStat2(POSIX posix, POSIXHandler handler) {
+            super(posix, handler);
+        }
+
+        private int mode; // Replaces st_mode
+
+        @Override
+        public void setup(CommonFileInformation fileInfo) {
+            super.setup(fileInfo);
+            // CommonFileInformation gives us (DOS-style) file attributes, not access rights.
+            int attr = fileInfo.getFileAttributes();
+            int mode = ALL_READ;
+            if ((attr & CommonFileInformation.FILE_ATTRIBUTE_READONLY) == 0) {
+                // Writable: assume by all
+                mode |= ALL_WRITE;
+            }
+            if ((attr & CommonFileInformation.FILE_ATTRIBUTE_DIRECTORY) != 0) {
+                // Directory: assume by all can look things up in it.
+                mode |= S_IFDIR | S_IXUGO;
+            } else {
+                // Regular file
+                mode |= S_IFREG;
+            }
+            this.mode = mode;
+        }
+
+        @Override
+        public int mode() {
+            return mode;
+        }
+    }
+
 }

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Tue Nov 10 14:07:25 2015
From: jython-checkins at python.org (jeff.allen)
Date: Tue, 10 Nov 2015 19:07:25 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Ensure_os=2Efstat=28=29_ret?=
 =?utf-8?q?urns_a_valid_st=5Fmode_on_Windows=2E_Addresses_=232320=2E?=
Message-ID: <20151110190724.103686.16445@psf.io>

https://hg.python.org/jython/rev/0381d60ca611
changeset:   7798:0381d60ca611
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Tue Nov 10 07:27:49 2015 +0000
summary:
  Ensure os.fstat() returns a valid st_mode on Windows. Addresses #2320.

Compensate for (probable) bug in jnr-posix whereby fstat st_mode is
always zero. We provide a custom stat-object able to synthesize a mode
from DOS-ish file attributes. Also we suppress os.closerange on Windows
until real integer file descriptors are available.

files:
  Lib/test/regrtest.py                          |   1 -
  src/org/python/modules/posix/PosixModule.java |  66 ++++++++-
  2 files changed, 59 insertions(+), 8 deletions(-)


diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1358,7 +1358,6 @@
 
     'java.nt':     # Expected to fail on Windows
         """
-        test_fileinput         # issue 2320 (because fstat().st_mode is zero)
         test_mailbox           # fails miserably and ruins other tests
         test_os_jy             # Locale tests run and fail on Cygwin
         test_popen             # http://bugs.python.org/issue1559298
diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -15,8 +15,8 @@
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.NotLinkException;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributeView;
@@ -34,9 +34,11 @@
 import jnr.posix.FileStat;
 import jnr.posix.POSIX;
 import jnr.posix.POSIXFactory;
+import jnr.posix.POSIXHandler;
 import jnr.posix.Times;
+import jnr.posix.WindowsRawFileStat;
 import jnr.posix.util.FieldAccess;
-import jnr.posix.util.Platform;
+import jnr.posix.windows.CommonFileInformation;
 
 import org.python.core.BufferProtocol;
 import org.python.core.ClassDictInit;
@@ -53,8 +55,8 @@
 import org.python.core.PyString;
 import org.python.core.PySystemState;
 import org.python.core.PyTuple;
+import org.python.core.Untraversable;
 import org.python.core.imp;
-import org.python.core.Untraversable;
 import org.python.core.io.FileIO;
 import org.python.core.io.IOBase;
 import org.python.core.io.RawIOBase;
@@ -75,7 +77,8 @@
     private static final OS os = OS.getOS();
 
     /** Platform specific POSIX services. */
-    private static final POSIX posix = POSIXFactory.getPOSIX(new PythonPOSIXHandler(), true);
+    private static final POSIXHandler posixHandler = new PythonPOSIXHandler();
+    private static final POSIX posix = POSIXFactory.getPOSIX(posixHandler, true);
 
     /** os.open flags. */
     private static final int O_RDONLY = 0x0;
@@ -354,6 +357,7 @@
         }
     }
 
+    @Hide(OS.NT)
     public static void closerange(PyObject fd_lowObj, PyObject fd_highObj) {
         int fd_low = getFD(fd_lowObj).getIntFD(false);
         int fd_high = getFD(fd_highObj).getIntFD(false);
@@ -1365,10 +1369,18 @@
             try {
                 FDUnion fd = getFD(fdObj);
                 FileStat stat;
-                if (fd.isIntFD()) {
-                    stat = posix.fstat(fd.intFD);
+                if (os != OS.NT) {
+                    if (fd.isIntFD()) {
+                        stat = posix.fstat(fd.intFD);
+                    } else {
+                        stat = posix.fstat(fd.javaFD);
+                    }
                 } else {
-                    stat = posix.fstat(fd.javaFD);
+                    // FIXME: jnr-posix fstat work-around. See issue #2320.
+                    stat = new WindowsRawFileStat2(posix, posixHandler);
+                    if (posix.fstat(fd.javaFD, stat) < 0) {
+                        throw Py.OSError(Errno.EBADF);
+                    }
                 }
                 return PyStatResult.fromFileStat(stat);
             } catch (PyException ex) {
@@ -1376,4 +1388,44 @@
             }
         }
     }
+
+    /*
+     * Extend the Windows stat object defined by jnr.posix, which in jnr-posix 2.0.4 is buggy to the
+     * extent that st_mode is always zero. Remarkably, it is possible to fix this by defining a
+     * cunning replacement.
+     */
+    private static class WindowsRawFileStat2 extends WindowsRawFileStat {
+
+        public WindowsRawFileStat2(POSIX posix, POSIXHandler handler) {
+            super(posix, handler);
+        }
+
+        private int mode; // Replaces st_mode
+
+        @Override
+        public void setup(CommonFileInformation fileInfo) {
+            super.setup(fileInfo);
+            // CommonFileInformation gives us (DOS-style) file attributes, not access rights.
+            int attr = fileInfo.getFileAttributes();
+            int mode = ALL_READ;
+            if ((attr & CommonFileInformation.FILE_ATTRIBUTE_READONLY) == 0) {
+                // Writable: assume by all
+                mode |= ALL_WRITE;
+            }
+            if ((attr & CommonFileInformation.FILE_ATTRIBUTE_DIRECTORY) != 0) {
+                // Directory: assume by all can look things up in it.
+                mode |= S_IFDIR | S_IXUGO;
+            } else {
+                // Regular file
+                mode |= S_IFREG;
+            }
+            this.mode = mode;
+        }
+
+        @Override
+        public int mode() {
+            return mode;
+        }
+    }
+
 }

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Wed Nov 11 20:58:53 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Thu, 12 Nov 2015 01:58:53 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Make_sure_java_named_tuples?=
 =?utf-8?q?_are_PyTupleDerived=2E_Issue2329?=
Message-ID: <20151112015853.19475.71333@psf.io>

https://hg.python.org/jython/rev/bb887a886c4b
changeset:   7800:bb887a886c4b
user:        Darjus Loktevic <darjus at gmail.com>
date:        Thu Nov 12 12:58:46 2015 +1100
summary:
  Make sure java named tuples are PyTupleDerived. Issue2329

files:
  Lib/test/test_sys_jy.py                        |   4 +++
  src/org/python/core/PySystemState.java         |   4 +-
  src/org/python/core/PyVersionInfo.java         |   2 +-
  src/org/python/modules/posix/PyStatResult.java |  12 +--------
  src/org/python/modules/time/PyTimeTuple.java   |  11 +-------
  5 files changed, 11 insertions(+), 22 deletions(-)


diff --git a/Lib/test/test_sys_jy.py b/Lib/test/test_sys_jy.py
--- a/Lib/test/test_sys_jy.py
+++ b/Lib/test/test_sys_jy.py
@@ -68,6 +68,10 @@
         reload(sys)
         self.assert_(type(sys.getdefaultencoding) == type(gde))
 
+    def test_get_tuple_from_version_info(self):
+        # sys.version_info is a tuple subclass
+        self.assertEqual(type(tuple(sys.version_info)), tuple)
+
 
 def exec_code_separately(function, sharing=False):
     """Runs code in a separate context: (thread, PySystemState, PythonInterpreter)
diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java
--- a/src/org/python/core/PySystemState.java
+++ b/src/org/python/core/PySystemState.java
@@ -1831,7 +1831,7 @@
 
 
 @ExposedType(name = "sys.float_info", isBaseType = false)
-class FloatInfo extends PyTuple {
+class FloatInfo extends PyTupleDerived {
 
     @ExposedGet
     public PyObject max, max_exp, max_10_exp, min, min_exp, min_10_exp, dig, mant_dig, epsilon,
@@ -1951,7 +1951,7 @@
 
 
 @ExposedType(name = "sys.long_info", isBaseType = false)
-class LongInfo extends PyTuple {
+class LongInfo extends PyTupleDerived {
 
     @ExposedGet
     public PyObject bits_per_digit, sizeof_digit;
diff --git a/src/org/python/core/PyVersionInfo.java b/src/org/python/core/PyVersionInfo.java
--- a/src/org/python/core/PyVersionInfo.java
+++ b/src/org/python/core/PyVersionInfo.java
@@ -6,7 +6,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "sys.version_info", isBaseType = false)
-public class PyVersionInfo extends PyTuple {
+public class PyVersionInfo extends PyTupleDerived {
     public static final PyType TYPE = PyType.fromClass(PyVersionInfo.class);
 
     @ExposedGet
diff --git a/src/org/python/modules/posix/PyStatResult.java b/src/org/python/modules/posix/PyStatResult.java
--- a/src/org/python/modules/posix/PyStatResult.java
+++ b/src/org/python/modules/posix/PyStatResult.java
@@ -3,15 +3,7 @@
 
 import jnr.posix.FileStat;
 
-import org.python.core.ArgParser;
-import org.python.core.Py;
-import org.python.core.PyList;
-import org.python.core.PyNewWrapper;
-import org.python.core.PyObject;
-import org.python.core.PyString;
-import org.python.core.PyTuple;
-import org.python.core.PyType;
-import org.python.core.Visitproc;
+import org.python.core.*;
 
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
@@ -25,7 +17,7 @@
 import java.util.concurrent.TimeUnit;
 
 @ExposedType(name = "stat_result", isBaseType = false)
-public class PyStatResult extends PyTuple {
+public class PyStatResult extends PyTupleDerived {
 
     public static final PyType TYPE = PyType.fromClass(PyStatResult.class);
 
diff --git a/src/org/python/modules/time/PyTimeTuple.java b/src/org/python/modules/time/PyTimeTuple.java
--- a/src/org/python/modules/time/PyTimeTuple.java
+++ b/src/org/python/modules/time/PyTimeTuple.java
@@ -1,14 +1,7 @@
 /* Copyright (c) 2005-2008 Jython Developers */
 package org.python.modules.time;
 
-import org.python.core.ArgParser;
-import org.python.core.Py;
-import org.python.core.PyList;
-import org.python.core.PyNewWrapper;
-import org.python.core.PyObject;
-import org.python.core.PyTuple;
-import org.python.core.PyType;
-import org.python.core.Visitproc;
+import org.python.core.*;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
@@ -20,7 +13,7 @@
  *
  */
 @ExposedType(name = "time.struct_time", isBaseType = false)
-public class PyTimeTuple extends PyTuple {
+public class PyTimeTuple extends PyTupleDerived {
 
     @ExposedGet
     public PyObject tm_year, tm_mon, tm_mday, tm_hour, tm_min, tm_sec, tm_wday, tm_yday, tm_isdst;

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Wed Nov 11 21:47:49 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Thu, 12 Nov 2015 02:47:49 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Back_out_7800_and_make_sure?=
 =?utf-8?q?_anything_that_isn=27t_PyTuple_itself_is_actually?=
Message-ID: <20151112024749.19479.38634@psf.io>

https://hg.python.org/jython/rev/03f797ecd7ea
changeset:   7801:03f797ecd7ea
user:        Darjus Loktevic <darjus at gmail.com>
date:        Thu Nov 12 13:47:41 2015 +1100
summary:
  Back out 7800 and make sure anything that isn't PyTuple itself is actually copied issue2329

files:
  Lib/test/test_sys_jy.py                        |   6 ++++-
  src/org/python/core/PySystemState.java         |   4 +-
  src/org/python/core/PyTuple.java               |   6 ++--
  src/org/python/core/PyVersionInfo.java         |   2 +-
  src/org/python/modules/posix/PyStatResult.java |  12 ++++++++-
  src/org/python/modules/time/PyTimeTuple.java   |  11 +++++++-
  6 files changed, 30 insertions(+), 11 deletions(-)


diff --git a/Lib/test/test_sys_jy.py b/Lib/test/test_sys_jy.py
--- a/Lib/test/test_sys_jy.py
+++ b/Lib/test/test_sys_jy.py
@@ -70,7 +70,11 @@
 
     def test_get_tuple_from_version_info(self):
         # sys.version_info is a tuple subclass
-        self.assertEqual(type(tuple(sys.version_info)), tuple)
+        pass #self.assertEqual(type(tuple(sys.version_info)), tuple)
+
+    def test_version_info_gt_lt(self):
+        self.assertTrue(sys.version_info > (0, 0))
+        self.assertTrue(sys.version_info < (99, 99))
 
 
 def exec_code_separately(function, sharing=False):
diff --git a/src/org/python/core/PySystemState.java b/src/org/python/core/PySystemState.java
--- a/src/org/python/core/PySystemState.java
+++ b/src/org/python/core/PySystemState.java
@@ -1831,7 +1831,7 @@
 
 
 @ExposedType(name = "sys.float_info", isBaseType = false)
-class FloatInfo extends PyTupleDerived {
+class FloatInfo extends PyTuple {
 
     @ExposedGet
     public PyObject max, max_exp, max_10_exp, min, min_exp, min_10_exp, dig, mant_dig, epsilon,
@@ -1951,7 +1951,7 @@
 
 
 @ExposedType(name = "sys.long_info", isBaseType = false)
-class LongInfo extends PyTupleDerived {
+class LongInfo extends PyTuple {
 
     @ExposedGet
     public PyObject bits_per_digit, sizeof_digit;
diff --git a/src/org/python/core/PyTuple.java b/src/org/python/core/PyTuple.java
--- a/src/org/python/core/PyTuple.java
+++ b/src/org/python/core/PyTuple.java
@@ -79,12 +79,12 @@
             if (S == null) {
                 return Py.EmptyTuple;
             }
+            if (S.getType() == PyTuple.TYPE) {
+                return S;
+            }
             if (S instanceof PyTupleDerived) {
                 return new PyTuple(((PyTuple) S).getArray());
             }
-            if (S instanceof PyTuple) {
-                return S;
-            }
             return fromArrayNoCopy(Py.make_array(S));
         } else {
             if (S == null) {
diff --git a/src/org/python/core/PyVersionInfo.java b/src/org/python/core/PyVersionInfo.java
--- a/src/org/python/core/PyVersionInfo.java
+++ b/src/org/python/core/PyVersionInfo.java
@@ -6,7 +6,7 @@
 import org.python.expose.ExposedType;
 
 @ExposedType(name = "sys.version_info", isBaseType = false)
-public class PyVersionInfo extends PyTupleDerived {
+public class PyVersionInfo extends PyTuple {
     public static final PyType TYPE = PyType.fromClass(PyVersionInfo.class);
 
     @ExposedGet
diff --git a/src/org/python/modules/posix/PyStatResult.java b/src/org/python/modules/posix/PyStatResult.java
--- a/src/org/python/modules/posix/PyStatResult.java
+++ b/src/org/python/modules/posix/PyStatResult.java
@@ -3,7 +3,15 @@
 
 import jnr.posix.FileStat;
 
-import org.python.core.*;
+import org.python.core.ArgParser;
+import org.python.core.Py;
+import org.python.core.PyList;
+import org.python.core.PyNewWrapper;
+import org.python.core.PyObject;
+import org.python.core.PyString;
+import org.python.core.PyTuple;
+import org.python.core.PyType;
+import org.python.core.Visitproc;
 
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
@@ -17,7 +25,7 @@
 import java.util.concurrent.TimeUnit;
 
 @ExposedType(name = "stat_result", isBaseType = false)
-public class PyStatResult extends PyTupleDerived {
+public class PyStatResult extends PyTuple {
 
     public static final PyType TYPE = PyType.fromClass(PyStatResult.class);
 
diff --git a/src/org/python/modules/time/PyTimeTuple.java b/src/org/python/modules/time/PyTimeTuple.java
--- a/src/org/python/modules/time/PyTimeTuple.java
+++ b/src/org/python/modules/time/PyTimeTuple.java
@@ -1,7 +1,14 @@
 /* Copyright (c) 2005-2008 Jython Developers */
 package org.python.modules.time;
 
-import org.python.core.*;
+import org.python.core.ArgParser;
+import org.python.core.Py;
+import org.python.core.PyList;
+import org.python.core.PyNewWrapper;
+import org.python.core.PyObject;
+import org.python.core.PyTuple;
+import org.python.core.PyType;
+import org.python.core.Visitproc;
 import org.python.expose.ExposedGet;
 import org.python.expose.ExposedMethod;
 import org.python.expose.ExposedNew;
@@ -13,7 +20,7 @@
  *
  */
 @ExposedType(name = "time.struct_time", isBaseType = false)
-public class PyTimeTuple extends PyTupleDerived {
+public class PyTimeTuple extends PyTuple {
 
     @ExposedGet
     public PyObject tm_year, tm_mon, tm_mday, tm_hour, tm_min, tm_sec, tm_wday, tm_yday, tm_isdst;

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Nov 12 21:13:09 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Fri, 13 Nov 2015 02:13:09 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_named_tuples_tests_for_?=
 =?utf-8?q?previous_commit?=
Message-ID: <20151113021309.117144.52839@psf.io>

https://hg.python.org/jython/rev/e2ecaaa9ac97
changeset:   7802:e2ecaaa9ac97
user:        Darjus Loktevic <darjus at gmail.com>
date:        Fri Nov 13 13:13:02 2015 +1100
summary:
  Add named tuples tests for previous commit

files:
  Lib/test/test_posix.py  |   3 +++
  Lib/test/test_sys_jy.py |  10 ++++++++--
  Lib/test/test_time.py   |   4 ++++
  3 files changed, 15 insertions(+), 2 deletions(-)


diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py
--- a/Lib/test/test_posix.py
+++ b/Lib/test/test_posix.py
@@ -217,6 +217,9 @@
         if hasattr(posix, 'stat'):
             self.assertTrue(posix.stat(test_support.TESTFN))
 
+    def test_stat_tuple(self):
+        self.assertEqual(tuple(posix.stat(".")), posix.stat("."))
+
     def _test_all_chown_common(self, chown_func, first_param):
         """Common code for chown, fchown and lchown tests."""
         if os.getuid() == 0:
diff --git a/Lib/test/test_sys_jy.py b/Lib/test/test_sys_jy.py
--- a/Lib/test/test_sys_jy.py
+++ b/Lib/test/test_sys_jy.py
@@ -69,14 +69,20 @@
         self.assert_(type(sys.getdefaultencoding) == type(gde))
 
     def test_get_tuple_from_version_info(self):
-        # sys.version_info is a tuple subclass
-        pass #self.assertEqual(type(tuple(sys.version_info)), tuple)
+        self.assertEqual(type(tuple(sys.version_info)), tuple)
+
+    def test_float_info_tuple(self):
+        self.assertEqual(tuple(sys.float_info), sys.float_info)
+
+    def test_long_info_tuple(self):
+        self.assertEqual(tuple(sys.long_info), sys.long_info)
 
     def test_version_info_gt_lt(self):
         self.assertTrue(sys.version_info > (0, 0))
         self.assertTrue(sys.version_info < (99, 99))
 
 
+
 def exec_code_separately(function, sharing=False):
     """Runs code in a separate context: (thread, PySystemState, PythonInterpreter)
 
diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py
--- a/Lib/test/test_time.py
+++ b/Lib/test/test_time.py
@@ -234,6 +234,10 @@
         t1 = time.mktime(gt1)
         self.assertTrue(0 <= (t1-t0) < 0.2)
 
+    def test_gmtime_tmetuple(self):
+        t = time.gmtime()
+        self.assertEqual(tuple(t), t)
+
     def test_localtime_without_arg(self):
         lt0 = time.localtime()
         lt1 = time.localtime(None)

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Nov 12 22:45:46 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Fri, 13 Nov 2015 03:45:46 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_PyListDerived_should_be_ite?=
 =?utf-8?q?rated_over_and_not_plain_copied_for_cases_where?=
Message-ID: <20151113034546.18190.26076@psf.io>

https://hg.python.org/jython/rev/522c0382a022
changeset:   7803:522c0382a022
user:        Darjus Loktevic <darjus at gmail.com>
date:        Fri Nov 13 14:45:38 2015 +1100
summary:
  PyListDerived should be iterated over and not plain copied for cases where someone subclasses list and overrides __iter__ + test

files:
  Lib/test/test_list_jy.py        |  27 ++++++++++++++++++--
  src/org/python/core/PyList.java |  10 +++++++-
  2 files changed, 33 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_list_jy.py b/Lib/test/test_list_jy.py
--- a/Lib/test/test_list_jy.py
+++ b/Lib/test/test_list_jy.py
@@ -52,7 +52,7 @@
 
     def test_tuple_equality(self):
         self.assertEqual([(1,), [1]].count([1]), 1) # http://bugs.jython.org/issue1317
- 
+
     def test_big_list(self):
         """Verify that fairly large collection literals of primitives can be constructed."""
         # use \n to separate to avoid parser problems
@@ -236,11 +236,32 @@
         self.assertEqual(jl, [1,2,3,4])
 
 
+class ListSubclassTestCase(unittest.TestCase):
+
+    def test_subclass_iter_copy(self):
+
+        class MyList(list):
+
+            def __iter__(self):
+                i = 0
+                results = super(MyList, self).__iter__()
+                for result in results:
+                    yield result
+                    i += 1
+
+                # add extra result for validation
+                yield i
+
+        lst = MyList(['a', 'b', 'c'])
+        self.assertEqual(list(lst), ['a', 'b', 'c', 3])
+
+
 def test_main():
-    test_support.run_unittest(ListTestCase,
+    test_support.run_unittest(ListSubclassTestCase,
+                              ListTestCase,
                               ThreadSafetyTestCase,
                               ExtendedSliceTestCase,
                               JavaListTestCase)
-
+    
 if __name__ == "__main__":
     test_main()
diff --git a/src/org/python/core/PyList.java b/src/org/python/core/PyList.java
--- a/src/org/python/core/PyList.java
+++ b/src/org/python/core/PyList.java
@@ -127,7 +127,15 @@
         if (seq == null) {
             return;
         }
-        if (seq instanceof PyList) {
+
+        /* PyListDerived should be iterated over and not plain copied for cases where someone subclasses list
+        and overrides __iter__
+         */
+        if (seq instanceof PyListDerived) {
+            for (PyObject item : seq.asIterable()) {
+                append(item);
+            }
+        } else if (seq instanceof PyList) {
             list.addAll(((PyList) seq).list); // don't convert
         } else if (seq instanceof PyTuple) {
             list.addAll(((PyTuple) seq).getList());

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 13 23:22:26 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 14 Nov 2015 04:22:26 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_programming_error_in_?=
 =?utf-8?q?=5Fsslcerts=2Epy?=
Message-ID: <20151114042226.34155.1290@psf.io>

https://hg.python.org/jython/rev/17443ce4824e
changeset:   7804:17443ce4824e
user:        Tom Alexander <tom.alexander at datastax.com>
date:        Sat Nov 14 15:20:17 2015 +1100
summary:
  Fix programming error in _sslcerts.py

files:
  Lib/_sslcerts.py |  8 ++++----
  1 files changed, 4 insertions(+), 4 deletions(-)


diff --git a/Lib/_sslcerts.py b/Lib/_sslcerts.py
--- a/Lib/_sslcerts.py
+++ b/Lib/_sslcerts.py
@@ -97,18 +97,18 @@
 
 
     if _key_store is None:
-        key_store = KeyStore.getInstance(KeyStore.getDefaultType())
-        key_store.load(None, None)
+        _key_store = KeyStore.getInstance(KeyStore.getDefaultType())
+        _key_store.load(None, None)
 
     if cert_file is not None:
         if not private_key:
             from _socket import SSLError, SSL_ERROR_SSL
             raise SSLError(SSL_ERROR_SSL, "No private key loaded")
 
-        key_store.setKeyEntry(str(uuid.uuid4()), private_key, [], certs)
+        _key_store.setKeyEntry(str(uuid.uuid4()), private_key, [], certs)
 
     kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm())
-    kmf.init(key_store, [])
+    kmf.init(_key_store, [])
     return kmf
 
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 13 23:22:26 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 14 Nov 2015 04:22:26 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Handle_shutdown_with_SHUT?=
 =?utf-8?q?=5FRDWR?=
Message-ID: <20151114042226.37486.91108@psf.io>

https://hg.python.org/jython/rev/e974921ae9bb
changeset:   7805:e974921ae9bb
user:        Tom Alexander <tom.alexander at datastax.com>
date:        Sat Nov 14 15:21:06 2015 +1100
summary:
  Handle shutdown with SHUT_RDWR

files:
  Lib/_socket.py |  4 ++--
  1 files changed, 2 insertions(+), 2 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -1051,12 +1051,12 @@
     def shutdown(self, how):
         log.debug("Got request to shutdown socket how=%s", how, extra={"sock": self})
         self._verify_channel()
-        if how & SHUT_RD:
+        if how & SHUT_RD or how & SHUT_RDWR:
             try:
                 self.channel.pipeline().remove(self.python_inbound_handler)
             except NoSuchElementException:
                 pass  # already removed, can safely ignore (presumably)
-        if how & SHUT_WR:
+        if how & SHUT_WR or how & SHUT_RDWR:
             self._can_write = False
 
     def _readable(self):

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 13 23:23:47 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 14 Nov 2015 04:23:47 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_ACKNOWLEDGMENTS?=
Message-ID: <20151114042347.120362.20649@psf.io>

https://hg.python.org/jython/rev/4642ea7ca23e
changeset:   7806:4642ea7ca23e
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sat Nov 14 15:23:39 2015 +1100
summary:
  Update ACKNOWLEDGMENTS

files:
  ACKNOWLEDGMENTS |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -164,6 +164,7 @@
     Richard Fearn
     Adam Burke
     Eric L Frederich
+    Tom Alexander
 
 Local Variables:
 mode: indented-text

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 13 23:33:43 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 14 Nov 2015 04:33:43 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_ACKNOWLEDGMENTS_http?=
 =?utf-8?q?s=3A//github=2Ecom/jythontools/jython/pull/13?=
Message-ID: <20151114043343.4679.27259@psf.io>

https://hg.python.org/jython/rev/ca32babcd1ee
changeset:   7808:ca32babcd1ee
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sat Nov 14 15:33:31 2015 +1100
summary:
  Update ACKNOWLEDGMENTS https://github.com/jythontools/jython/pull/13

files:
  ACKNOWLEDGMENTS |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -165,6 +165,7 @@
     Adam Burke
     Eric L Frederich
     Tom Alexander
+    Caleb P. Burns
 
 Local Variables:
 mode: indented-text

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 13 23:33:43 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 14 Nov 2015 04:33:43 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Accept_annotations_in_org?=
 =?utf-8?q?=2Epython=2Ecompiler=2EClassFile=23addField?=
Message-ID: <20151114043343.14371.92111@psf.io>

https://hg.python.org/jython/rev/81adb2def410
changeset:   7807:81adb2def410
user:        cpburnz <cpburnz at gmail.com>
date:        Sat Nov 14 15:30:55 2015 +1100
summary:
  Accept annotations in org.python.compiler.ClassFile#addField

files:
  src/org/python/compiler/ClassFile.java |  17 ++++++++++++++
  1 files changed, 17 insertions(+), 0 deletions(-)


diff --git a/src/org/python/compiler/ClassFile.java b/src/org/python/compiler/ClassFile.java
--- a/src/org/python/compiler/ClassFile.java
+++ b/src/org/python/compiler/ClassFile.java
@@ -162,7 +162,24 @@
     public void addField(String name, String type, int access)
         throws IOException
     {
+        addField(name, type, access, null);
+    }
+
+    public void addField(String name, String type, int access, AnnotationDescr[] annotationDescrs)
+        throws IOException
+    {
         FieldVisitor fv = cw.visitField(access, name, type, null, null);
+
+        if (annotationDescrs != null) {
+            for (AnnotationDescr ad: annotationDescrs) {
+                AnnotationVisitor av = fv.visitAnnotation(ad.getName(), true);
+                if (ad.hasFields()) {
+                    visitAnnotations(av, ad.getFields());
+                }
+                av.visitEnd();
+            }
+        }
+
         fieldVisitors.add(fv);
     }
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 14 09:17:55 2015
From: jython-checkins at python.org (jim.baker)
Date: Sat, 14 Nov 2015 14:17:55 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Change_socket=2Econnect=5Fe?=
 =?utf-8?q?x_to_report_intermediate_connection_states=2E_Fixes_=232428=2E?=
Message-ID: <20151114141755.14379.32564@psf.io>

https://hg.python.org/jython/rev/962002f7d96b
changeset:   7809:962002f7d96b
user:        Jim Baker <jim.baker at rackspace.com>
date:        Sat Nov 14 07:17:46 2015 -0700
summary:
  Change socket.connect_ex to report intermediate connection states. Fixes #2428.

A major usage of socket.connect_ex is to poll the state of a
connection in progress. The proper reported sequence for a successful
connection is zero or one EINPROGRESS, zero or more EALREADY, followed
by EISCONN. In addition, extant code assumes that the poll will take a
nonnegible amount of time (~ 1-2 ms) or will fail in exhausting a busy
loop. With this fix, Jython gets CPython-compliant semantics.

files:
  Lib/_socket.py             |  58 +++++++++++------
  Lib/test/test_socket_jy.py |  85 ++++++++++++++++++++++++++
  2 files changed, 121 insertions(+), 22 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -28,7 +28,8 @@
 from java.util import NoSuchElementException
 from java.util.concurrent import (
     ArrayBlockingQueue, CopyOnWriteArrayList, CountDownLatch, LinkedBlockingQueue,
-    RejectedExecutionException, ThreadFactory, TimeUnit)
+    ExecutionException, RejectedExecutionException, ThreadFactory,
+    TimeoutException, TimeUnit)
 from java.util.concurrent.atomic import AtomicBoolean, AtomicLong
 from javax.net.ssl import SSLPeerUnverifiedException, SSLException
 
@@ -852,7 +853,7 @@
 
         if self.bind_addr:
             log.debug("Connect %s to %s", self.bind_addr, addr, extra={"sock": self})
-            bind_future = bootstrap.bind(self.bind_addr)
+            bind_future = bootstrap.bind(self.bind_addr).sync()
             self._handle_channel_future(bind_future, "local bind")
             self.channel = bind_future.channel()
         else:
@@ -888,16 +889,39 @@
             log.debug("Completed connection to %s", addr, extra={"sock": self})
 
     def connect_ex(self, addr):
+        was_connecting = self.connected  # actually means self.connecting if
+                                         # not blocking
         if not self.connected:
             try:
                 self.connect(addr)
             except error as e:
                 return e.errno
         if not self.connect_future.isDone():
-            return errno.EINPROGRESS
+            if was_connecting:
+                try:
+                    # Timing is based on CPython and was empirically
+                    # guestimated. Of course this means user code is
+                    # polling, so the the best we can do is wait like
+                    # this in supposedly nonblocking mode without
+                    # completely busy waiting!
+                    self.connect_future.get(1500, TimeUnit.MICROSECONDS)
+                except ExecutionException:
+                    # generally raised if closed; pick up the state
+                    # when testing for success
+                    pass
+                except TimeoutException:
+                    # more than 1.5ms, will report EALREADY below
+                    pass
+
+        if not self.connect_future.isDone():
+            if was_connecting:
+                return errno.EALREADY
+            else:
+                return errno.EINPROGRESS
         elif self.connect_future.isSuccess():
             return errno.EISCONN
         else:
+            print self.connect_future.cause()
             return errno.ENOTCONN
 
     # SERVER METHODS
@@ -1241,27 +1265,17 @@
                 raise error(errno.ENOTCONN, "Socket is not connected")
             else:
                 return _socktuple(self.bind_addr)
-        # Netty 4 currently races between bind to ephemeral port and the availability
-        # of the local address for the channel. Poll to work around this issue.
-        while True:
-            local_addr = self.channel.localAddress()
-            if local_addr:
-                if hasattr(self, "bind_future"):
-                    if self.bind_future.isDone():
-                        break
-                else:
-                    break
-            if time.time() - self.bind_timestamp > 1:
-                # Presumably after a second something is completely wrong,
-                # so punt
-                raise error(errno.ENOTCONN, "Socket is not connected")
-            log.debug("Poll for local address", extra={"sock": self})
-            time.sleep(0.01)  # completely arbitrary
+        if hasattr(self, "bind_future"):
+            self.bind_future.sync()
+        local_addr = self.channel.localAddress()
         if local_addr.getAddress().isAnyLocalAddress():
-            # Netty 4 will default to an IPv6 "any" address from a channel even if it was originally bound to an IPv4 "any" address
-            # so, as a workaround, let's construct a new "any" address using the port information gathered above
+            # Netty 4 will default to an IPv6 "any" address from a
+            # channel even if it was originally bound to an IPv4 "any"
+            # address so, as a workaround, let's construct a new "any"
+            # address using the port information gathered above
             if type(self.bind_addr.getAddress()) != type(local_addr.getAddress()):
-                return _socktuple(java.net.InetSocketAddress(self.bind_addr.getAddress(), local_addr.getPort()))
+                return _socktuple(java.net.InetSocketAddress(
+                    self.bind_addr.getAddress(), local_addr.getPort()))
         return _socktuple(local_addr)
 
     def getpeername(self):
diff --git a/Lib/test/test_socket_jy.py b/Lib/test/test_socket_jy.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_socket_jy.py
@@ -0,0 +1,85 @@
+import errno
+import socket
+import threading
+import unittest
+from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+from SocketServer import ThreadingMixIn
+from test import test_support
+
+
+def start_server():
+    server_address = ('127.0.0.1', 0)
+
+    class DaemonThreadingMixIn(ThreadingMixIn):
+        daemon_threads = True
+
+    class ThreadedHTTPServer(DaemonThreadingMixIn, HTTPServer):
+        """Handle requests in a separate thread."""
+
+    # not actually going to do anything with this server, so a
+    # do-nothing handler is reasonable
+    httpd = ThreadedHTTPServer(server_address, BaseHTTPRequestHandler)
+    server_thread = threading.Thread(target=httpd.serve_forever)
+    server_thread.daemon = True
+    server_thread.start()
+    return httpd, server_thread
+
+
+class SocketConnectTest(unittest.TestCase):
+
+    def setUp(self):
+        self.httpd, self.server_thread = start_server()
+        self.address = self.httpd.server_name, self.httpd.server_port
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.server_thread.join()
+
+    def do_nonblocking_connection(self, results, index):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.setblocking(0)
+        connect_errno = 0
+        connect_attempt = 0
+
+        while connect_errno != errno.EISCONN and connect_attempt < 100:
+            connect_attempt += 1
+            connect_errno = sock.connect_ex(self.address)
+            results[index].append(connect_errno)
+        sock.close()
+
+    def do_workout(self, num_threads=10):
+        connect_results = []
+        connect_threads = []
+        for i in xrange(num_threads):
+            connect_results.append([])
+            connect_threads.append(threading.Thread(
+                target=self.do_nonblocking_connection,
+                name="socket-workout-%s" % i,
+                args=(connect_results, i)))
+                           
+        for thread in connect_threads:
+            thread.start()
+        for thread in connect_threads:
+            thread.join()
+        return connect_results
+
+    def test_connect_ex_workout(self):
+        """Verify connect_ex states go through EINPROGRESS?, EALREADY*, EISCONN"""
+        # Tests fix for http://bugs.jython.org/issue2428; based in part on the
+        # code showing failure that was submitted with that bug
+
+        #self.httpd, self.server_thread = start_server()
+        #self.address = self.httpd.server_name, self.httpd.server_port
+        for result in self.do_workout():
+            self.assertIn(result[0], {errno.EINPROGRESS, errno.EISCONN})
+            self.assertEqual(result[-1], errno.EISCONN)
+            for code in result[1:-1]:
+                self.assertEqual(code, errno.EALREADY)
+
+
+def test_main():
+    test_support.run_unittest(SocketConnectTest)
+
+
+if __name__ == "__main__":
+    test_main()

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov 15 19:56:16 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Mon, 16 Nov 2015 00:56:16 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Set_seekable=3Dfalse_when_s?=
 =?utf-8?q?eek=28=29_raises_IOError?=
Message-ID: <20151116005616.21853.67055@psf.io>

https://hg.python.org/jython/rev/fc33f0456a64
changeset:   7810:fc33f0456a64
parent:      7808:ca32babcd1ee
user:        Stephen Drake <steve at synergyconsultingnz.com>
date:        Sat Nov 14 21:15:37 2015 +1100
summary:
  Set seekable=false when seek() raises IOError
If a file is not seekable, seekable() should return False rather than raise
IOError.  See https://docs.python.org/2/library/io.html#i-o-base-classes

files:
  src/org/python/modules/_io/PyFileIO.java |  10 +++++++++-
  1 files changed, 9 insertions(+), 1 deletions(-)


diff --git a/src/org/python/modules/_io/PyFileIO.java b/src/org/python/modules/_io/PyFileIO.java
--- a/src/org/python/modules/_io/PyFileIO.java
+++ b/src/org/python/modules/_io/PyFileIO.java
@@ -392,7 +392,15 @@
             throw closedValueError();
         }
         if (!seekableKnown) {
-            seekable = ioDelegate.seek(0, 1) >= 0;  // Trial seek
+            try {
+                ioDelegate.seek(0, 1);  // Trial seek
+                seekable = true;
+            } catch (PyException exc) {
+                if (!exc.match(Py.IOError)) {
+                    throw exc;
+                }
+                seekable = false;
+            }
             seekableKnown = true;
         }
         return seekable;

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov 15 19:56:17 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Mon, 16 Nov 2015 00:56:17 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Implement_os=2Epipe=28=29_u?=
 =?utf-8?q?sing_java=2Enio=2Echannels=2EPipe=2E__The_pipe=27s_source?=
Message-ID: <20151116005616.109181.27281@psf.io>

https://hg.python.org/jython/rev/9ac36add65aa
changeset:   7811:9ac36add65aa
user:        Stephen Drake <steve at synergyconsultingnz.com>
date:        Sat Nov 14 21:20:35 2015 +1100
summary:
  Implement os.pipe() using java.nio.channels.Pipe.  The pipe's source
and sink channels are wrapped with StreamIO to be compatible with
fdopen() and io.open().

Fixes http://bugs.jython.org/issue1984

files:
  src/org/python/modules/posix/PosixModule.java |  21 ++++++++++
  1 files changed, 21 insertions(+), 0 deletions(-)


diff --git a/src/org/python/modules/posix/PosixModule.java b/src/org/python/modules/posix/PosixModule.java
--- a/src/org/python/modules/posix/PosixModule.java
+++ b/src/org/python/modules/posix/PosixModule.java
@@ -12,6 +12,7 @@
 import java.nio.channels.Channel;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
+import java.nio.channels.Pipe;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
@@ -50,6 +51,7 @@
 import org.python.core.PyException;
 import org.python.core.PyFile;
 import org.python.core.PyFloat;
+import org.python.core.PyJavaType;
 import org.python.core.PyList;
 import org.python.core.PyObject;
 import org.python.core.PyString;
@@ -60,6 +62,7 @@
 import org.python.core.io.FileIO;
 import org.python.core.io.IOBase;
 import org.python.core.io.RawIOBase;
+import org.python.core.io.StreamIO;
 import org.python.core.util.StringUtil;
 
 /**
@@ -779,6 +782,24 @@
         return new FileIO((PyString) path, fileIOMode);
     }
 
+    public static PyString __doc__pipe = new PyString(
+        "pipe() -> (read_end, write_end)\n\n" +
+        "Create a pipe.");
+    public static PyTuple pipe() {
+        try {
+            Pipe pipe = Pipe.open();
+
+            StreamIO pipe_read = new StreamIO(pipe.source());
+            StreamIO pipe_write = new StreamIO(pipe.sink());
+
+            return new PyTuple(
+                PyJavaType.wrapJavaObject(pipe_read.fileno()),
+                PyJavaType.wrapJavaObject(pipe_write.fileno()));
+        } catch (IOException ioe) {
+            throw Py.OSError(ioe);
+        }
+    }
+
     public static PyString __doc__popen = new PyString(
         "popen(command [, mode='r' [, bufsize]]) -> pipe\n\n" +
         "Open a pipe to/from a command returning a file object.");

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov 15 19:56:17 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Mon, 16 Nov 2015 00:56:17 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_a_test_for_seekable_pip?=
 =?utf-8?q?e_and_update_ACKNOWLEDGMENTS_=28thanks_Stephen_Drake!=29?=
Message-ID: <20151116005617.88506.34230@psf.io>

https://hg.python.org/jython/rev/b76b7b840349
changeset:   7812:b76b7b840349
user:        Darjus Loktevic <darjus at gmail.com>
date:        Mon Nov 16 11:54:08 2015 +1100
summary:
  Add a test for seekable pipe and update ACKNOWLEDGMENTS (thanks Stephen Drake!)

files:
  ACKNOWLEDGMENTS        |   1 +
  Lib/test/test_io_jy.py |  16 +++++++++++++++-
  2 files changed, 16 insertions(+), 1 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -166,6 +166,7 @@
     Eric L Frederich
     Tom Alexander
     Caleb P. Burns
+    Stephen Drake
 
 Local Variables:
 mode: indented-text
diff --git a/Lib/test/test_io_jy.py b/Lib/test/test_io_jy.py
--- a/Lib/test/test_io_jy.py
+++ b/Lib/test/test_io_jy.py
@@ -1,8 +1,12 @@
 import unittest
 from test import test_support
 
+import io
 import _io
 
+from os import pipe
+
+
 class NameTest(unittest.TestCase):
 
     def test_names_available_in__io_module(self):
@@ -16,8 +20,18 @@
               '_BufferedIOBase', '_IOBase', '_RawIOBase', '_TextIOBase'
             })
 
+
+class PipeTestCase(unittest.TestCase):
+
+    def test_pipe_seekable_bool(self):
+        r, _ = pipe()
+
+        self.assertFalse(io.open(r).seekable())
+
+
 def test_main():
-    test_support.run_unittest(NameTest)
+    test_support.run_unittest(NameTest, PipeTestCase)
+
 
 if __name__ == "__main__":
     test_main()

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sun Nov 15 19:56:17 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Mon, 16 Nov 2015 00:56:17 +0000
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge?=
Message-ID: <20151116005617.37460.83442@psf.io>

https://hg.python.org/jython/rev/b3f43050645c
changeset:   7813:b3f43050645c
parent:      7812:b76b7b840349
parent:      7809:962002f7d96b
user:        Darjus Loktevic <darjus at gmail.com>
date:        Mon Nov 16 11:56:08 2015 +1100
summary:
  Merge

files:
  Lib/_socket.py             |  58 +++++++++++------
  Lib/test/test_socket_jy.py |  85 ++++++++++++++++++++++++++
  2 files changed, 121 insertions(+), 22 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -28,7 +28,8 @@
 from java.util import NoSuchElementException
 from java.util.concurrent import (
     ArrayBlockingQueue, CopyOnWriteArrayList, CountDownLatch, LinkedBlockingQueue,
-    RejectedExecutionException, ThreadFactory, TimeUnit)
+    ExecutionException, RejectedExecutionException, ThreadFactory,
+    TimeoutException, TimeUnit)
 from java.util.concurrent.atomic import AtomicBoolean, AtomicLong
 from javax.net.ssl import SSLPeerUnverifiedException, SSLException
 
@@ -852,7 +853,7 @@
 
         if self.bind_addr:
             log.debug("Connect %s to %s", self.bind_addr, addr, extra={"sock": self})
-            bind_future = bootstrap.bind(self.bind_addr)
+            bind_future = bootstrap.bind(self.bind_addr).sync()
             self._handle_channel_future(bind_future, "local bind")
             self.channel = bind_future.channel()
         else:
@@ -888,16 +889,39 @@
             log.debug("Completed connection to %s", addr, extra={"sock": self})
 
     def connect_ex(self, addr):
+        was_connecting = self.connected  # actually means self.connecting if
+                                         # not blocking
         if not self.connected:
             try:
                 self.connect(addr)
             except error as e:
                 return e.errno
         if not self.connect_future.isDone():
-            return errno.EINPROGRESS
+            if was_connecting:
+                try:
+                    # Timing is based on CPython and was empirically
+                    # guestimated. Of course this means user code is
+                    # polling, so the the best we can do is wait like
+                    # this in supposedly nonblocking mode without
+                    # completely busy waiting!
+                    self.connect_future.get(1500, TimeUnit.MICROSECONDS)
+                except ExecutionException:
+                    # generally raised if closed; pick up the state
+                    # when testing for success
+                    pass
+                except TimeoutException:
+                    # more than 1.5ms, will report EALREADY below
+                    pass
+
+        if not self.connect_future.isDone():
+            if was_connecting:
+                return errno.EALREADY
+            else:
+                return errno.EINPROGRESS
         elif self.connect_future.isSuccess():
             return errno.EISCONN
         else:
+            print self.connect_future.cause()
             return errno.ENOTCONN
 
     # SERVER METHODS
@@ -1241,27 +1265,17 @@
                 raise error(errno.ENOTCONN, "Socket is not connected")
             else:
                 return _socktuple(self.bind_addr)
-        # Netty 4 currently races between bind to ephemeral port and the availability
-        # of the local address for the channel. Poll to work around this issue.
-        while True:
-            local_addr = self.channel.localAddress()
-            if local_addr:
-                if hasattr(self, "bind_future"):
-                    if self.bind_future.isDone():
-                        break
-                else:
-                    break
-            if time.time() - self.bind_timestamp > 1:
-                # Presumably after a second something is completely wrong,
-                # so punt
-                raise error(errno.ENOTCONN, "Socket is not connected")
-            log.debug("Poll for local address", extra={"sock": self})
-            time.sleep(0.01)  # completely arbitrary
+        if hasattr(self, "bind_future"):
+            self.bind_future.sync()
+        local_addr = self.channel.localAddress()
         if local_addr.getAddress().isAnyLocalAddress():
-            # Netty 4 will default to an IPv6 "any" address from a channel even if it was originally bound to an IPv4 "any" address
-            # so, as a workaround, let's construct a new "any" address using the port information gathered above
+            # Netty 4 will default to an IPv6 "any" address from a
+            # channel even if it was originally bound to an IPv4 "any"
+            # address so, as a workaround, let's construct a new "any"
+            # address using the port information gathered above
             if type(self.bind_addr.getAddress()) != type(local_addr.getAddress()):
-                return _socktuple(java.net.InetSocketAddress(self.bind_addr.getAddress(), local_addr.getPort()))
+                return _socktuple(java.net.InetSocketAddress(
+                    self.bind_addr.getAddress(), local_addr.getPort()))
         return _socktuple(local_addr)
 
     def getpeername(self):
diff --git a/Lib/test/test_socket_jy.py b/Lib/test/test_socket_jy.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_socket_jy.py
@@ -0,0 +1,85 @@
+import errno
+import socket
+import threading
+import unittest
+from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+from SocketServer import ThreadingMixIn
+from test import test_support
+
+
+def start_server():
+    server_address = ('127.0.0.1', 0)
+
+    class DaemonThreadingMixIn(ThreadingMixIn):
+        daemon_threads = True
+
+    class ThreadedHTTPServer(DaemonThreadingMixIn, HTTPServer):
+        """Handle requests in a separate thread."""
+
+    # not actually going to do anything with this server, so a
+    # do-nothing handler is reasonable
+    httpd = ThreadedHTTPServer(server_address, BaseHTTPRequestHandler)
+    server_thread = threading.Thread(target=httpd.serve_forever)
+    server_thread.daemon = True
+    server_thread.start()
+    return httpd, server_thread
+
+
+class SocketConnectTest(unittest.TestCase):
+
+    def setUp(self):
+        self.httpd, self.server_thread = start_server()
+        self.address = self.httpd.server_name, self.httpd.server_port
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.server_thread.join()
+
+    def do_nonblocking_connection(self, results, index):
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.setblocking(0)
+        connect_errno = 0
+        connect_attempt = 0
+
+        while connect_errno != errno.EISCONN and connect_attempt < 100:
+            connect_attempt += 1
+            connect_errno = sock.connect_ex(self.address)
+            results[index].append(connect_errno)
+        sock.close()
+
+    def do_workout(self, num_threads=10):
+        connect_results = []
+        connect_threads = []
+        for i in xrange(num_threads):
+            connect_results.append([])
+            connect_threads.append(threading.Thread(
+                target=self.do_nonblocking_connection,
+                name="socket-workout-%s" % i,
+                args=(connect_results, i)))
+                           
+        for thread in connect_threads:
+            thread.start()
+        for thread in connect_threads:
+            thread.join()
+        return connect_results
+
+    def test_connect_ex_workout(self):
+        """Verify connect_ex states go through EINPROGRESS?, EALREADY*, EISCONN"""
+        # Tests fix for http://bugs.jython.org/issue2428; based in part on the
+        # code showing failure that was submitted with that bug
+
+        #self.httpd, self.server_thread = start_server()
+        #self.address = self.httpd.server_name, self.httpd.server_port
+        for result in self.do_workout():
+            self.assertIn(result[0], {errno.EINPROGRESS, errno.EISCONN})
+            self.assertEqual(result[-1], errno.EISCONN)
+            for code in result[1:-1]:
+                self.assertEqual(code, errno.EALREADY)
+
+
+def test_main():
+    test_support.run_unittest(SocketConnectTest)
+
+
+if __name__ == "__main__":
+    test_main()

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 20 16:07:01 2015
From: jython-checkins at python.org (jim.baker)
Date: Fri, 20 Nov 2015 21:07:01 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_commented_out_code?=
Message-ID: <20151120210659.21869.88411@psf.io>

https://hg.python.org/jython/rev/fed939712ce7
changeset:   7814:fed939712ce7
user:        Jim Baker <jim.baker at rackspace.com>
date:        Fri Nov 20 14:06:23 2015 -0700
summary:
  Remove commented out code

files:
  Lib/test/test_socket_jy.py |  3 ---
  1 files changed, 0 insertions(+), 3 deletions(-)


diff --git a/Lib/test/test_socket_jy.py b/Lib/test/test_socket_jy.py
--- a/Lib/test/test_socket_jy.py
+++ b/Lib/test/test_socket_jy.py
@@ -67,9 +67,6 @@
         """Verify connect_ex states go through EINPROGRESS?, EALREADY*, EISCONN"""
         # Tests fix for http://bugs.jython.org/issue2428; based in part on the
         # code showing failure that was submitted with that bug
-
-        #self.httpd, self.server_thread = start_server()
-        #self.address = self.httpd.server_name, self.httpd.server_port
         for result in self.do_workout():
             self.assertIn(result[0], {errno.EINPROGRESS, errno.EISCONN})
             self.assertEqual(result[-1], errno.EISCONN)

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 20 16:08:22 2015
From: jython-checkins at python.org (jim.baker)
Date: Fri, 20 Nov 2015 21:08:22 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Upgrade_Netty_bundled_jars_?=
 =?utf-8?b?dG8gNC4wLjMz?=
Message-ID: <20151120210820.63500.45988@psf.io>

https://hg.python.org/jython/rev/37578797caba
changeset:   7816:37578797caba
user:        Jim Baker <jim.baker at rackspace.com>
date:        Fri Nov 20 14:08:11 2015 -0700
summary:
  Upgrade Netty bundled jars to 4.0.33

files:
  build.xml                                |   20 +++++-----
  extlibs/netty-buffer-4.0.31.Final.jar    |  Bin 
  extlibs/netty-buffer-4.0.33.Final.jar    |  Bin 
  extlibs/netty-codec-4.0.31.Final.jar     |  Bin 
  extlibs/netty-codec-4.0.33.Final.jar     |  Bin 
  extlibs/netty-common-4.0.31.Final.jar    |  Bin 
  extlibs/netty-common-4.0.33.Final.jar    |  Bin 
  extlibs/netty-handler-4.0.31.Final.jar   |  Bin 
  extlibs/netty-handler-4.0.33.Final.jar   |  Bin 
  extlibs/netty-transport-4.0.31.Final.jar |  Bin 
  extlibs/netty-transport-4.0.33.Final.jar |  Bin 
  11 files changed, 10 insertions(+), 10 deletions(-)


diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -177,11 +177,11 @@
             <pathelement path="${extlibs.dir}/jnr-posix-3.0.20.jar"/>
             <pathelement path="${extlibs.dir}/jnr-constants-0.9.0.jar"/>
             <pathelement path="${extlibs.dir}/jline-2.12.1.jar"/>
-            <pathelement path="${extlibs.dir}/netty-buffer-4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-codec-4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-common-4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-handler--4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-transport-4.0.31.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-buffer-4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-codec-4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-common-4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-handler--4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-transport-4.0.33.Final.jar"/>
         </path>
 
         <available property="informix.present" classname="com.informix.jdbc.IfxDriver" classpath="${informix.jar}" />
@@ -587,15 +587,15 @@
             <rule pattern="com.google.**" result="org.python.google. at 1"/>
             <zipfileset src="extlibs/icu4j-54_1_1.jar"/>
             <rule pattern="com.ibm.icu.**" result="org.python.icu. at 1"/>
-            <zipfileset src="extlibs/netty-buffer-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-buffer-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-codec-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-codec-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-common-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-common-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-handler-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-handler-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-transport-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-transport-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
             <zipfileset src="extlibs/jffi-arm-Linux.jar"/>
             <zipfileset src="extlibs/jffi-Darwin.jar"/>
diff --git a/extlibs/netty-buffer-4.0.31.Final.jar b/extlibs/netty-buffer-4.0.31.Final.jar
deleted file mode 100644
index 759696378f5a57652b4db83462043bc5bca8b542..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-buffer-4.0.33.Final.jar b/extlibs/netty-buffer-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5e644b4aa8aa4cb736415d71e6a53929fae23005
GIT binary patch
[stripped]
diff --git a/extlibs/netty-codec-4.0.31.Final.jar b/extlibs/netty-codec-4.0.31.Final.jar
deleted file mode 100644
index a943e6bf7361835342c8637c8db3588e87d70e19..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-codec-4.0.33.Final.jar b/extlibs/netty-codec-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b58013e04bcbcdf8eca12e986b1cd4704cadd1c5
GIT binary patch
[stripped]
diff --git a/extlibs/netty-common-4.0.31.Final.jar b/extlibs/netty-common-4.0.31.Final.jar
deleted file mode 100644
index ed507f2d323abfbd593442e32d9ecbd2ace149b4..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-common-4.0.33.Final.jar b/extlibs/netty-common-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b4990abb33b774c96ee0922b2b5505285fade9d5
GIT binary patch
[stripped]
diff --git a/extlibs/netty-handler-4.0.31.Final.jar b/extlibs/netty-handler-4.0.31.Final.jar
deleted file mode 100644
index 1a002fb26d8b46e7e132eabf952ba127d9ac2bb3..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-handler-4.0.33.Final.jar b/extlibs/netty-handler-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ea26491f786d9d4463e1f09a8f7ddebacd9e8763
GIT binary patch
[stripped]
diff --git a/extlibs/netty-transport-4.0.31.Final.jar b/extlibs/netty-transport-4.0.31.Final.jar
deleted file mode 100644
index 87881ca0beac4e5c5aecb099acffd4d673922d55..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-transport-4.0.33.Final.jar b/extlibs/netty-transport-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7be560332f0c25040e9def2f66ee5f7d7b0cec6c
GIT binary patch
[stripped]

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 20 16:08:22 2015
From: jython-checkins at python.org (jim.baker)
Date: Fri, 20 Nov 2015 21:08:22 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Support_socket_shutdown_of_?=
 =?utf-8?q?incompletely_set_up_sockets?=
Message-ID: <20151120210819.79049.57038@psf.io>

https://hg.python.org/jython/rev/e7c755355c88
changeset:   7815:e7c755355c88
user:        Jim Baker <jim.baker at rackspace.com>
date:        Fri Nov 20 14:07:36 2015 -0700
summary:
  Support socket shutdown of incompletely set up sockets

files:
  Lib/_socket.py |  2 ++
  1 files changed, 2 insertions(+), 0 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -1080,6 +1080,8 @@
                 self.channel.pipeline().remove(self.python_inbound_handler)
             except NoSuchElementException:
                 pass  # already removed, can safely ignore (presumably)
+            except AttributeError:
+                pass  # inbound handler never set up, also ignore
         if how & SHUT_WR or how & SHUT_RDWR:
             self._can_write = False
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 20 19:54:33 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 21 Nov 2015 00:54:33 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fixed_for_zero-width_match_?=
 =?utf-8?q?protection_in_re_+_tests=2E_Fixed_a_few_smaller_issues?=
Message-ID: <20151121005432.21521.26853@psf.io>

https://hg.python.org/jython/rev/636b124a7587
changeset:   7818:636b124a7587
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sat Nov 21 11:49:04 2015 +1100
summary:
  Fixed for zero-width match protection in re + tests. Fixed a few smaller issues around re with refreshed test_re from CPython.

files:
  Lib/test/test_re.py                           |  28 +++++++--
  Lib/test/test_re_jy.py                        |  10 +++
  Lib/test/test_support.py                      |   5 +
  src/org/python/modules/sre/MatchObject.java   |  21 +++---
  src/org/python/modules/sre/PatternObject.java |  10 +-
  src/org/python/modules/sre/SRE_REPEAT.java    |   1 +
  src/org/python/modules/sre/SRE_STATE.java     |  11 +--
  7 files changed, 59 insertions(+), 27 deletions(-)


diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -3,7 +3,7 @@
     verbose, run_unittest, import_module,
     precisionbigmemtest, _2G, cpython_only,
     captured_stdout, have_unicode, requires_unicode, u,
-    check_warnings)
+    check_warnings, is_jython)
 import locale
 import re
 from re import Scanner
@@ -22,6 +22,10 @@
 
 import unittest
 
+
+todo_on_jython = unittest.skipIf(is_jython, 'no jython support yet')
+
+
 class ReTests(unittest.TestCase):
 
     def test_weakref(self):
@@ -431,6 +435,7 @@
         self.assertEqual(len(re.findall(r"\B", " ")), 2)
 
     @requires_unicode
+    @todo_on_jython
     def test_bigcharset(self):
         self.assertEqual(re.match(u(r"([\u2222\u2223])"),
                                   unichr(0x2222)).group(1), unichr(0x2222))
@@ -487,6 +492,9 @@
         self.assertIsNone(re.match(r'ab(?<=c)c', 'abc'))
         self.assertIsNone(re.match(r'ab(?<!b)c', 'abc'))
         self.assertTrue(re.match(r'ab(?<!c)c', 'abc'))
+
+        # TODO Jython warnings support
+        return
         # Group reference.
         with check_warnings(('', RuntimeWarning)):
             re.compile(r'(a)a(?<=\1)c')
@@ -512,7 +520,7 @@
         self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a")
         self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
 
-        if have_unicode:
+        if have_unicode and not is_jython:  # TODO Jython Unicode :)
             assert u(r'\u212a').lower() == u'k' # '?'
             self.assertTrue(re.match(ur'K', u(r'\u212a'), re.U | re.I))
             self.assertTrue(re.match(ur'k', u(r'\u212a'), re.U | re.I))
@@ -529,7 +537,7 @@
         self.assertTrue(re.match(r'[19a]', 'a', re.I))
         self.assertTrue(re.match(r'[19a]', 'A', re.I))
         self.assertTrue(re.match(r'[19A]', 'a', re.I))
-        if have_unicode:
+        if have_unicode and not is_jython:  # TODO Jython Unicode :)
             self.assertTrue(re.match(ur'[19A]', u'A', re.U | re.I))
             self.assertTrue(re.match(ur'[19a]', u'a', re.U | re.I))
             self.assertTrue(re.match(ur'[19a]', u'A', re.U | re.I))
@@ -545,6 +553,7 @@
             self.assertTrue(re.match(u(r'[19\u017f]'), u'S', re.U | re.I))
             self.assertTrue(re.match(u(r'[19\u017f]'), u's', re.U | re.I))
 
+    @todo_on_jython  # implement 17381
     def test_ignore_case_range(self):
         # Issues #3511, #17381.
         self.assertTrue(re.match(r'[9-a]', '_', re.I))
@@ -553,7 +562,7 @@
         self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I))
         self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7',re.I))
         self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I))
-        if have_unicode:
+        if have_unicode and not is_jython:  # TODO Jython Unicode :)
             self.assertTrue(re.match(u(r'[9-a]'), u(r'_'), re.U | re.I))
             self.assertIsNone(re.match(u(r'[9-A]'), u(r'_'), re.U | re.I))
             self.assertTrue(re.match(u(r'[\xc0-\xde]'),
@@ -739,6 +748,7 @@
         # should, instead provoking a TypeError.
         self.assertRaises(re.error, re.compile, 'foo[a-')
 
+    @todo_on_jython  # RuntimeError: maximum recursion depth exceeded (Java StackOverflowError)
     def test_bug_418626(self):
         # bugs 418626 at al. -- Testing Greg Chapman's addition of op code
         # SRE_OP_MIN_REPEAT_ONE for eliminating recursion on simple uses of
@@ -756,6 +766,7 @@
         pat=u"["+re.escape(unichr(0x2039))+u"]"
         self.assertEqual(re.compile(pat) and 1, 1)
 
+    @todo_on_jython  # RuntimeError: maximum recursion depth exceeded (Java StackOverflowError)
     def test_stack_overflow(self):
         # nasty cases that used to overflow the straightforward recursive
         # implementation of repeated groups.
@@ -763,6 +774,7 @@
         self.assertEqual(re.match('(x)*y', 50000*'x'+'y').group(1), 'x')
         self.assertEqual(re.match('(x)*?y', 50000*'x'+'y').group(1), 'x')
 
+    @todo_on_jython  # RuntimeError: maximum recursion depth exceeded (Java StackOverflowError)
     def test_unlimited_zero_width_repeat(self):
         # Issue #9669
         self.assertIsNone(re.match(r'(?:a?)*y', 'z'))
@@ -940,6 +952,7 @@
         self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a#\nb#\nc#')
         self.assertEqual(pattern.sub('#', '\n'), '#\n#')
 
+    @unittest.skipIf(is_jython, "CPython specific")
     def test_dealloc(self):
         # issue 3299: check for segfault in debug build
         import _sre
@@ -987,6 +1000,7 @@
         self.assertEqual(n, size + 1)
 
 
+    @todo_on_jython  # OverflowError: the repetition number is too large
     def test_repeat_minmax_overflow(self):
         # Issue #13169
         string = "x" * 100000
@@ -1019,12 +1033,12 @@
 
     def test_backref_group_name_in_exception(self):
         # Issue 17341: Poor error message when compiling invalid regex
-        with self.assertRaisesRegexp(sre_constants.error, '<foo>'):
+        with self.assertRaisesRegexp(sre_constants.error, 'bad character in group name'):
             re.compile('(?P=<foo>)')
 
     def test_group_name_in_exception(self):
         # Issue 17341: Poor error message when compiling invalid regex
-        with self.assertRaisesRegexp(sre_constants.error, '\?foo'):
+        with self.assertRaisesRegexp(sre_constants.error, 'bad character in group name'):
             re.compile('(?P<?foo>)')
 
     def test_issue17998(self):
@@ -1039,6 +1053,7 @@
                                      [u'xyz'], msg=pattern)
 
 
+    @todo_on_jython
     def test_bug_2537(self):
         # issue 2537: empty submatches
         for outer_op in ('{0,}', '*', '+', '{1,187}'):
@@ -1049,6 +1064,7 @@
                 self.assertEqual(m.group(1), "")
                 self.assertEqual(m.group(2), "y")
 
+    @todo_on_jython
     def test_debug_flag(self):
         pat = r'(\.)(?:[ch]|py)(?(1)$|: )'
         with captured_stdout() as out:
diff --git a/Lib/test/test_re_jy.py b/Lib/test/test_re_jy.py
--- a/Lib/test/test_re_jy.py
+++ b/Lib/test/test_re_jy.py
@@ -71,6 +71,16 @@
                 self.assertNotRegexpMatches(c, ws_re)
                 self.assertRegexpMatches(c, not_ws_re)
 
+    def test_start_is_end(self):
+        COMMENT_RE = re.compile(r'(\A)+')
+
+        requirements = ''
+        self.assertEqual(COMMENT_RE.search(requirements).groups(), (requirements, ))
+
+    def test_pip_comment(self):
+        COMMENT_RE = re.compile(r'(^|\s)+#.*$')
+        self.assertEqual(COMMENT_RE.sub('', '#'), '')
+
 
 def test_main():
     test.test_support.run_unittest(ReTest)
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -433,6 +433,11 @@
 except NameError:
     have_unicode = False
 
+requires_unicode = unittest.skipUnless(have_unicode, 'no unicode support')
+
+def u(s):
+    return unicode(s, 'unicode-escape')
+
 if is_jython:
     def make_jar_classloader(jar):
         import os
diff --git a/src/org/python/modules/sre/MatchObject.java b/src/org/python/modules/sre/MatchObject.java
--- a/src/org/python/modules/sre/MatchObject.java
+++ b/src/org/python/modules/sre/MatchObject.java
@@ -15,16 +15,9 @@
 
 package org.python.modules.sre;
 
-import org.python.core.ArgParser;
-import org.python.core.Py;
-import org.python.core.PyDictionary;
-import org.python.core.PyInteger;
-import org.python.core.PyObject;
-import org.python.core.PyString;
-import org.python.core.PyTuple;
-import org.python.core.Traverseproc;
-import org.python.core.Visitproc;
-import org.python.core.imp;
+import org.python.core.*;
+
+import java.math.BigInteger;
 
 
 public class MatchObject extends PyObject implements Traverseproc {
@@ -155,6 +148,14 @@
     private int getindex(PyObject index) {
         if (index instanceof PyInteger)
             return ((PyInteger) index).getValue();
+        if (index instanceof PyLong) {
+            BigInteger idx = ((PyLong) index).getValue();
+            if (idx.compareTo(PyInteger.MAX_INT) == 1) {
+                throw Py.IndexError("no such group");
+            } else {
+                return idx.intValue();
+            }
+        }
 
         int i = -1;
 
diff --git a/src/org/python/modules/sre/PatternObject.java b/src/org/python/modules/sre/PatternObject.java
--- a/src/org/python/modules/sre/PatternObject.java
+++ b/src/org/python/modules/sre/PatternObject.java
@@ -44,8 +44,8 @@
     }
 
     public MatchObject match(PyObject[] args, String[] kws) {
-        ArgParser ap = new ArgParser("search", args, kws,
-                                     "pattern", "pos", "endpos");
+        ArgParser ap = new ArgParser("match", args, kws,
+                                     "string", "pos", "endpos");
         PyString string = extractPyString(ap, 0);
         int start = ap.getInt(1, 0);
         int end = ap.getInt(2, string.__len__());
@@ -59,7 +59,7 @@
     
     public MatchObject search(PyObject[] args, String[] kws) {
         ArgParser ap = new ArgParser("search", args, kws,
-                                     "pattern", "pos", "endpos");
+                                     "string", "pos", "endpos");
         PyString string = extractPyString(ap, 0);
         int start = ap.getInt(1, 0);
         int end = ap.getInt(2, string.__len__());
@@ -184,7 +184,7 @@
 
     public PyObject split(PyObject[] args, String[] kws) {
         ArgParser ap = new ArgParser("split", args, kws,
-                                     "source", "maxsplit");
+                                     "string", "maxsplit");
         PyString string = extractPyString(ap, 0);
         int maxsplit = ap.getInt(1, 0);
 
@@ -240,7 +240,7 @@
 
     public PyObject findall(PyObject[] args, String[] kws) {
         ArgParser ap = new ArgParser("findall", args, kws,
-                                     "source", "pos", "endpos");
+                                     "string", "pos", "endpos");
         PyString string = extractPyString(ap, 0);
         int start = ap.getInt(1, 0);
         int end = ap.getInt(2, Integer.MAX_VALUE);
diff --git a/src/org/python/modules/sre/SRE_REPEAT.java b/src/org/python/modules/sre/SRE_REPEAT.java
--- a/src/org/python/modules/sre/SRE_REPEAT.java
+++ b/src/org/python/modules/sre/SRE_REPEAT.java
@@ -20,6 +20,7 @@
 public class SRE_REPEAT {
     int count;
     int pidx;
+    int last_ptr = -1;
 
     SRE_REPEAT prev;
 
diff --git a/src/org/python/modules/sre/SRE_STATE.java b/src/org/python/modules/sre/SRE_STATE.java
--- a/src/org/python/modules/sre/SRE_STATE.java
+++ b/src/org/python/modules/sre/SRE_STATE.java
@@ -883,9 +883,6 @@
                 /* maximizing repeat */
                 /* <REPEAT> <skip> <1=min> <2=max> item <MAX_UNTIL> tail */
 
-                /* FIXME: we probably need to deal with zero-width
-                   matches in here... */
-
                 SRE_REPEAT rp = this.repeat;
                 if (rp == null)
                     return SRE_ERROR_STATE;
@@ -908,11 +905,14 @@
                     return 0;
                 }
 
-                if (count < pattern[rp.pidx+2] ||
-                                            pattern[rp.pidx+2] == 65535) {
+                if ((count < pattern[rp.pidx+2] ||
+                        pattern[rp.pidx+2] == 65535) &&
+                        // see: http://git.io/v4Q0I for zero-width match protection
+                        ptr != rp.last_ptr) {
                     /* we may have enough matches, but if we can
                        match another item, do so */
                     rp.count = count;
+                    rp.last_ptr = ptr;
                     lastmark = this.lastmark;
                     lastindex = this.lastindex;
                     mark_stack_base = mark_save(0, lastmark);
@@ -1216,7 +1216,6 @@
         return status;
     }
 
-
     /* string pointers */
     int ptr; /* current position (also end of current slice) */
     int beginning; /* start of original string */

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 20 19:54:34 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 21 Nov 2015 00:54:34 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Copy_Lib/test/test=5Fre=2Ep?=
 =?utf-8?q?y_from_CPython_2=2E7=2E10?=
Message-ID: <20151121005432.20256.36818@psf.io>

https://hg.python.org/jython/rev/e6726cdde3c1
changeset:   7817:e6726cdde3c1
parent:      7813:b3f43050645c
user:        Darjus Loktevic <darjus at gmail.com>
date:        Fri Nov 20 16:17:56 2015 +1100
summary:
  Copy Lib/test/test_re.py from CPython 2.7.10

files:
  Lib/test/test_re.py |  696 ++++++++++++++++++++++++++-----
  1 files changed, 579 insertions(+), 117 deletions(-)


diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -1,16 +1,23 @@
-import sys
-sys.path = ['.'] + sys.path
-
-from test.test_support import verbose, run_unittest
+# -*- coding: utf-8 -*-
+from test.test_support import (
+    verbose, run_unittest, import_module,
+    precisionbigmemtest, _2G, cpython_only,
+    captured_stdout, have_unicode, requires_unicode, u,
+    check_warnings)
+import locale
 import re
 from re import Scanner
-import sys, os, traceback
+import sre_constants
+import sys
+import string
+import traceback
 from weakref import proxy
 
+
 # Misc tests from Tim Peters' re.doc
 
 # WARNING: Don't change details in these tests if you don't know
-# what you're doing. Some of these tests were carefuly modeled to
+# what you're doing. Some of these tests were carefully modeled to
 # cover most of the code.
 
 import unittest
@@ -28,12 +35,12 @@
         self.assertEqual(re.search('x*', 'axx').span(), (0, 0))
         self.assertEqual(re.search('x+', 'axx').span(0), (1, 3))
         self.assertEqual(re.search('x+', 'axx').span(), (1, 3))
-        self.assertEqual(re.search('x', 'aaa'), None)
+        self.assertIsNone(re.search('x', 'aaa'))
         self.assertEqual(re.match('a*', 'xxx').span(0), (0, 0))
         self.assertEqual(re.match('a*', 'xxx').span(), (0, 0))
         self.assertEqual(re.match('x*', 'xxxa').span(0), (0, 3))
         self.assertEqual(re.match('x*', 'xxxa').span(), (0, 3))
-        self.assertEqual(re.match('a+', 'xxx'), None)
+        self.assertIsNone(re.match('a+', 'xxx'))
 
     def bump_num(self, matchobj):
         int_value = int(matchobj.group(0))
@@ -83,6 +90,7 @@
         self.assertEqual(re.sub('\r\n', '\n', 'abc\r\ndef\r\n'),
                          'abc\ndef\n')
 
+    @requires_unicode
     def test_bug_1140(self):
         # re.sub(x, y, u'') should return u'', not '', and
         # re.sub(x, y, '') should return '', not u''.
@@ -108,6 +116,18 @@
                 self.assertEqual(z, y)
                 self.assertEqual(type(z), type(y))
 
+    def test_bug_1661(self):
+        # Verify that flags do not get silently ignored with compiled patterns
+        pattern = re.compile('.')
+        self.assertRaises(ValueError, re.match, pattern, 'A', re.I)
+        self.assertRaises(ValueError, re.search, pattern, 'A', re.I)
+        self.assertRaises(ValueError, re.findall, pattern, 'A', re.I)
+        self.assertRaises(ValueError, re.compile, pattern, re.I)
+
+    def test_bug_3629(self):
+        # A regex that triggered a bug in the sre-code validator
+        re.compile("(?P<quote>)(?(quote))")
+
     def test_sub_template_numeric_escape(self):
         # bug 776311 and friends
         self.assertEqual(re.sub('x', r'\0', 'x'), '\0')
@@ -163,11 +183,31 @@
         self.assertEqual(re.sub('x*', '-', 'abxd'), '-a-b-d-')
         self.assertEqual(re.sub('x+', '-', 'abxd'), 'ab-d')
 
+    def test_symbolic_groups(self):
+        re.compile('(?P<a>x)(?P=a)(?(a)y)')
+        re.compile('(?P<a1>x)(?P=a1)(?(a1)y)')
+        self.assertRaises(re.error, re.compile, '(?P<a>)(?P<a>)')
+        self.assertRaises(re.error, re.compile, '(?Px)')
+        self.assertRaises(re.error, re.compile, '(?P=)')
+        self.assertRaises(re.error, re.compile, '(?P=1)')
+        self.assertRaises(re.error, re.compile, '(?P=a)')
+        self.assertRaises(re.error, re.compile, '(?P=a1)')
+        self.assertRaises(re.error, re.compile, '(?P=a.)')
+        self.assertRaises(re.error, re.compile, '(?P<)')
+        self.assertRaises(re.error, re.compile, '(?P<>)')
+        self.assertRaises(re.error, re.compile, '(?P<1>)')
+        self.assertRaises(re.error, re.compile, '(?P<a.>)')
+        self.assertRaises(re.error, re.compile, '(?())')
+        self.assertRaises(re.error, re.compile, '(?(a))')
+        self.assertRaises(re.error, re.compile, '(?(1a))')
+        self.assertRaises(re.error, re.compile, '(?(a.))')
+
     def test_symbolic_refs(self):
         self.assertRaises(re.error, re.sub, '(?P<a>x)', '\g<a', 'xx')
         self.assertRaises(re.error, re.sub, '(?P<a>x)', '\g<', 'xx')
         self.assertRaises(re.error, re.sub, '(?P<a>x)', '\g', 'xx')
         self.assertRaises(re.error, re.sub, '(?P<a>x)', '\g<a a>', 'xx')
+        self.assertRaises(re.error, re.sub, '(?P<a>x)', '\g<>', 'xx')
         self.assertRaises(re.error, re.sub, '(?P<a>x)', '\g<1a1>', 'xx')
         self.assertRaises(IndexError, re.sub, '(?P<a>x)', '\g<ab>', 'xx')
         self.assertRaises(re.error, re.sub, '(?P<a>x)|(?P<b>y)', '\g<b>', 'xx')
@@ -249,8 +289,8 @@
                          ('(', 'a'))
         self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', 'a').groups(),
                          (None, 'a'))
-        self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', 'a)'), None)
-        self.assertEqual(re.match('^(\()?([^()]+)(?(1)\))$', '(a'), None)
+        self.assertIsNone(re.match('^(\()?([^()]+)(?(1)\))$', 'a)'))
+        self.assertIsNone(re.match('^(\()?([^()]+)(?(1)\))$', '(a'))
         self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'ab').groups(),
                          ('a', 'b'))
         self.assertEqual(re.match('^(?:(a)|c)((?(1)b|d))$', 'cd').groups(),
@@ -266,8 +306,8 @@
                          ('a', 'b', 'c'))
         self.assertEqual(p.match('ad').groups(),
                          ('a', None, 'd'))
-        self.assertEqual(p.match('abd'), None)
-        self.assertEqual(p.match('ac'), None)
+        self.assertIsNone(p.match('abd'))
+        self.assertIsNone(p.match('ac'))
 
 
     def test_re_groupref(self):
@@ -275,8 +315,8 @@
                          ('|', 'a'))
         self.assertEqual(re.match(r'^(\|)?([^()]+)\1?$', 'a').groups(),
                          (None, 'a'))
-        self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', 'a|'), None)
-        self.assertEqual(re.match(r'^(\|)?([^()]+)\1$', '|a'), None)
+        self.assertIsNone(re.match(r'^(\|)?([^()]+)\1$', 'a|'))
+        self.assertIsNone(re.match(r'^(\|)?([^()]+)\1$', '|a'))
         self.assertEqual(re.match(r'^(?:(a)|c)(\1)$', 'aa').groups(),
                          ('a', 'a'))
         self.assertEqual(re.match(r'^(?:(a)|c)(\1)?$', 'c').groups(),
@@ -294,10 +334,10 @@
                          "second first second first")
 
     def test_repeat_minmax(self):
-        self.assertEqual(re.match("^(\w){1}$", "abc"), None)
-        self.assertEqual(re.match("^(\w){1}?$", "abc"), None)
-        self.assertEqual(re.match("^(\w){1,2}$", "abc"), None)
-        self.assertEqual(re.match("^(\w){1,2}?$", "abc"), None)
+        self.assertIsNone(re.match("^(\w){1}$", "abc"))
+        self.assertIsNone(re.match("^(\w){1}?$", "abc"))
+        self.assertIsNone(re.match("^(\w){1,2}$", "abc"))
+        self.assertIsNone(re.match("^(\w){1,2}?$", "abc"))
 
         self.assertEqual(re.match("^(\w){3}$", "abc").group(1), "c")
         self.assertEqual(re.match("^(\w){1,3}$", "abc").group(1), "c")
@@ -308,29 +348,29 @@
         self.assertEqual(re.match("^(\w){1,4}?$", "abc").group(1), "c")
         self.assertEqual(re.match("^(\w){3,4}?$", "abc").group(1), "c")
 
-        self.assertEqual(re.match("^x{1}$", "xxx"), None)
-        self.assertEqual(re.match("^x{1}?$", "xxx"), None)
-        self.assertEqual(re.match("^x{1,2}$", "xxx"), None)
-        self.assertEqual(re.match("^x{1,2}?$", "xxx"), None)
+        self.assertIsNone(re.match("^x{1}$", "xxx"))
+        self.assertIsNone(re.match("^x{1}?$", "xxx"))
+        self.assertIsNone(re.match("^x{1,2}$", "xxx"))
+        self.assertIsNone(re.match("^x{1,2}?$", "xxx"))
 
-        self.assertNotEqual(re.match("^x{3}$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{1,3}$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{1,4}$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{3,4}?$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{3}?$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{1,3}?$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{1,4}?$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{3,4}?$", "xxx"), None)
+        self.assertTrue(re.match("^x{3}$", "xxx"))
+        self.assertTrue(re.match("^x{1,3}$", "xxx"))
+        self.assertTrue(re.match("^x{1,4}$", "xxx"))
+        self.assertTrue(re.match("^x{3,4}?$", "xxx"))
+        self.assertTrue(re.match("^x{3}?$", "xxx"))
+        self.assertTrue(re.match("^x{1,3}?$", "xxx"))
+        self.assertTrue(re.match("^x{1,4}?$", "xxx"))
+        self.assertTrue(re.match("^x{3,4}?$", "xxx"))
 
-        self.assertEqual(re.match("^x{}$", "xxx"), None)
-        self.assertNotEqual(re.match("^x{}$", "x{}"), None)
+        self.assertIsNone(re.match("^x{}$", "xxx"))
+        self.assertTrue(re.match("^x{}$", "x{}"))
 
     def test_getattr(self):
         self.assertEqual(re.match("(a)", "a").pos, 0)
         self.assertEqual(re.match("(a)", "a").endpos, 1)
         self.assertEqual(re.match("(a)", "a").string, "a")
         self.assertEqual(re.match("(a)", "a").regs, ((0, 1), (0, 1)))
-        self.assertNotEqual(re.match("(a)", "a").re, None)
+        self.assertTrue(re.match("(a)", "a").re)
 
     def test_special_escapes(self):
         self.assertEqual(re.search(r"\b(b.)\b",
@@ -341,36 +381,69 @@
                                    "abcd abc bcd bx", re.LOCALE).group(1), "bx")
         self.assertEqual(re.search(r"\B(b.)\B",
                                    "abc bcd bc abxd", re.LOCALE).group(1), "bx")
-        self.assertEqual(re.search(r"\b(b.)\b",
-                                   "abcd abc bcd bx", re.UNICODE).group(1), "bx")
-        self.assertEqual(re.search(r"\B(b.)\B",
-                                   "abc bcd bc abxd", re.UNICODE).group(1), "bx")
+        if have_unicode:
+            self.assertEqual(re.search(r"\b(b.)\b",
+                                       "abcd abc bcd bx", re.UNICODE).group(1), "bx")
+            self.assertEqual(re.search(r"\B(b.)\B",
+                                       "abc bcd bc abxd", re.UNICODE).group(1), "bx")
         self.assertEqual(re.search(r"^abc$", "\nabc\n", re.M).group(0), "abc")
         self.assertEqual(re.search(r"^\Aabc\Z$", "abc", re.M).group(0), "abc")
-        self.assertEqual(re.search(r"^\Aabc\Z$", "\nabc\n", re.M), None)
+        self.assertIsNone(re.search(r"^\Aabc\Z$", "\nabc\n", re.M))
         self.assertEqual(re.search(r"\b(b.)\b",
                                    u"abcd abc bcd bx").group(1), "bx")
         self.assertEqual(re.search(r"\B(b.)\B",
                                    u"abc bcd bc abxd").group(1), "bx")
         self.assertEqual(re.search(r"^abc$", u"\nabc\n", re.M).group(0), "abc")
         self.assertEqual(re.search(r"^\Aabc\Z$", u"abc", re.M).group(0), "abc")
-        self.assertEqual(re.search(r"^\Aabc\Z$", u"\nabc\n", re.M), None)
+        self.assertIsNone(re.search(r"^\Aabc\Z$", u"\nabc\n", re.M))
         self.assertEqual(re.search(r"\d\D\w\W\s\S",
                                    "1aa! a").group(0), "1aa! a")
         self.assertEqual(re.search(r"\d\D\w\W\s\S",
                                    "1aa! a", re.LOCALE).group(0), "1aa! a")
-        self.assertEqual(re.search(r"\d\D\w\W\s\S",
-                                   "1aa! a", re.UNICODE).group(0), "1aa! a")
+        if have_unicode:
+            self.assertEqual(re.search(r"\d\D\w\W\s\S",
+                                       "1aa! a", re.UNICODE).group(0), "1aa! a")
 
-    def test_ignore_case(self):
-        self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC")
-        self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC")
+    def test_string_boundaries(self):
+        # See http://bugs.python.org/issue10713
+        self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1),
+                         "abc")
+        # There's a word boundary at the start of a string.
+        self.assertTrue(re.match(r"\b", "abc"))
+        # A non-empty string includes a non-boundary zero-length match.
+        self.assertTrue(re.search(r"\B", "abc"))
+        # There is no non-boundary match at the start of a string.
+        self.assertFalse(re.match(r"\B", "abc"))
+        # However, an empty string contains no word boundaries, and also no
+        # non-boundaries.
+        self.assertIsNone(re.search(r"\B", ""))
+        # This one is questionable and different from the perlre behaviour,
+        # but describes current behavior.
+        self.assertIsNone(re.search(r"\b", ""))
+        # A single word-character string has two boundaries, but no
+        # non-boundary gaps.
+        self.assertEqual(len(re.findall(r"\b", "a")), 2)
+        self.assertEqual(len(re.findall(r"\B", "a")), 0)
+        # If there are no words, there are no boundaries
+        self.assertEqual(len(re.findall(r"\b", " ")), 0)
+        self.assertEqual(len(re.findall(r"\b", "   ")), 0)
+        # Can match around the whitespace.
+        self.assertEqual(len(re.findall(r"\B", " ")), 2)
 
+    @requires_unicode
     def test_bigcharset(self):
-        self.assertEqual(re.match(u"([\u2222\u2223])",
-                                  u"\u2222").group(1), u"\u2222")
-        self.assertEqual(re.match(u"([\u2222\u2223])",
-                                  u"\u2222", re.UNICODE).group(1), u"\u2222")
+        self.assertEqual(re.match(u(r"([\u2222\u2223])"),
+                                  unichr(0x2222)).group(1), unichr(0x2222))
+        self.assertEqual(re.match(u(r"([\u2222\u2223])"),
+                                  unichr(0x2222), re.UNICODE).group(1), unichr(0x2222))
+        r = u'[%s]' % u''.join(map(unichr, range(256, 2**16, 255)))
+        self.assertEqual(re.match(r, unichr(0xff01), re.UNICODE).group(), unichr(0xff01))
+
+    def test_big_codesize(self):
+        # Issue #1160
+        r = re.compile('|'.join(('%d'%x for x in range(10000))))
+        self.assertTrue(r.match('1000'))
+        self.assertTrue(r.match('9999'))
 
     def test_anyall(self):
         self.assertEqual(re.match("a.b", "a\nb", re.DOTALL).group(0),
@@ -378,7 +451,7 @@
         self.assertEqual(re.match("a.*b", "a\n\nb", re.DOTALL).group(0),
                          "a\n\nb")
 
-    def test_non_consuming(self):
+    def test_lookahead(self):
         self.assertEqual(re.match("(a(?=\s[^a]))", "a b").group(1), "a")
         self.assertEqual(re.match("(a(?=\s[^a]*))", "a b").group(1), "a")
         self.assertEqual(re.match("(a(?=\s[abc]))", "a b").group(1), "a")
@@ -392,7 +465,44 @@
         self.assertEqual(re.match(r"(a)(?!\s\1)", "a b").group(1), "a")
         self.assertEqual(re.match(r"(a)(?!\s(abc|a))", "a b").group(1), "a")
 
+        # Group reference.
+        self.assertTrue(re.match(r'(a)b(?=\1)a', 'aba'))
+        self.assertIsNone(re.match(r'(a)b(?=\1)c', 'abac'))
+        # Named group reference.
+        self.assertTrue(re.match(r'(?P<g>a)b(?=(?P=g))a', 'aba'))
+        self.assertIsNone(re.match(r'(?P<g>a)b(?=(?P=g))c', 'abac'))
+        # Conditional group reference.
+        self.assertTrue(re.match(r'(?:(a)|(x))b(?=(?(2)x|c))c', 'abc'))
+        self.assertIsNone(re.match(r'(?:(a)|(x))b(?=(?(2)c|x))c', 'abc'))
+        self.assertTrue(re.match(r'(?:(a)|(x))b(?=(?(2)x|c))c', 'abc'))
+        self.assertIsNone(re.match(r'(?:(a)|(x))b(?=(?(1)b|x))c', 'abc'))
+        self.assertTrue(re.match(r'(?:(a)|(x))b(?=(?(1)c|x))c', 'abc'))
+        # Group used before defined.
+        self.assertTrue(re.match(r'(a)b(?=(?(2)x|c))(c)', 'abc'))
+        self.assertIsNone(re.match(r'(a)b(?=(?(2)b|x))(c)', 'abc'))
+        self.assertTrue(re.match(r'(a)b(?=(?(1)c|x))(c)', 'abc'))
+
+    def test_lookbehind(self):
+        self.assertTrue(re.match(r'ab(?<=b)c', 'abc'))
+        self.assertIsNone(re.match(r'ab(?<=c)c', 'abc'))
+        self.assertIsNone(re.match(r'ab(?<!b)c', 'abc'))
+        self.assertTrue(re.match(r'ab(?<!c)c', 'abc'))
+        # Group reference.
+        with check_warnings(('', RuntimeWarning)):
+            re.compile(r'(a)a(?<=\1)c')
+        # Named group reference.
+        with check_warnings(('', RuntimeWarning)):
+            re.compile(r'(?P<g>a)a(?<=(?P=g))c')
+        # Conditional group reference.
+        with check_warnings(('', RuntimeWarning)):
+            re.compile(r'(a)b(?<=(?(1)b|x))c')
+        # Group used before defined.
+        with check_warnings(('', RuntimeWarning)):
+            re.compile(r'(a)b(?<=(?(2)b|x))(c)')
+
     def test_ignore_case(self):
+        self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC")
+        self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC")
         self.assertEqual(re.match(r"(a\s[^a])", "a b", re.I).group(1), "a b")
         self.assertEqual(re.match(r"(a\s[^a]*)", "a bb", re.I).group(1), "a bb")
         self.assertEqual(re.match(r"(a\s[abc])", "a b", re.I).group(1), "a b")
@@ -402,6 +512,87 @@
         self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a")
         self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
 
+        if have_unicode:
+            assert u(r'\u212a').lower() == u'k' # '?'
+            self.assertTrue(re.match(ur'K', u(r'\u212a'), re.U | re.I))
+            self.assertTrue(re.match(ur'k', u(r'\u212a'), re.U | re.I))
+            self.assertTrue(re.match(u(r'\u212a'), u'K', re.U | re.I))
+            self.assertTrue(re.match(u(r'\u212a'), u'k', re.U | re.I))
+            assert u(r'\u017f').upper() == u'S' # '?'
+            self.assertTrue(re.match(ur'S', u(r'\u017f'), re.U | re.I))
+            self.assertTrue(re.match(ur's', u(r'\u017f'), re.U | re.I))
+            self.assertTrue(re.match(u(r'\u017f'), u'S', re.U | re.I))
+            self.assertTrue(re.match(u(r'\u017f'), u's', re.U | re.I))
+
+    def test_ignore_case_set(self):
+        self.assertTrue(re.match(r'[19A]', 'A', re.I))
+        self.assertTrue(re.match(r'[19a]', 'a', re.I))
+        self.assertTrue(re.match(r'[19a]', 'A', re.I))
+        self.assertTrue(re.match(r'[19A]', 'a', re.I))
+        if have_unicode:
+            self.assertTrue(re.match(ur'[19A]', u'A', re.U | re.I))
+            self.assertTrue(re.match(ur'[19a]', u'a', re.U | re.I))
+            self.assertTrue(re.match(ur'[19a]', u'A', re.U | re.I))
+            self.assertTrue(re.match(ur'[19A]', u'a', re.U | re.I))
+            assert u(r'\u212a').lower() == u'k' # '?'
+            self.assertTrue(re.match(u(r'[19K]'), u(r'\u212a'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[19k]'), u(r'\u212a'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[19\u212a]'), u'K', re.U | re.I))
+            self.assertTrue(re.match(u(r'[19\u212a]'), u'k', re.U | re.I))
+            assert u(r'\u017f').upper() == u'S' # '?'
+            self.assertTrue(re.match(ur'[19S]', u(r'\u017f'), re.U | re.I))
+            self.assertTrue(re.match(ur'[19s]', u(r'\u017f'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[19\u017f]'), u'S', re.U | re.I))
+            self.assertTrue(re.match(u(r'[19\u017f]'), u's', re.U | re.I))
+
+    def test_ignore_case_range(self):
+        # Issues #3511, #17381.
+        self.assertTrue(re.match(r'[9-a]', '_', re.I))
+        self.assertIsNone(re.match(r'[9-A]', '_', re.I))
+        self.assertTrue(re.match(r'[\xc0-\xde]', '\xd7', re.I))
+        self.assertIsNone(re.match(r'[\xc0-\xde]', '\xf7', re.I))
+        self.assertTrue(re.match(r'[\xe0-\xfe]', '\xf7',re.I))
+        self.assertIsNone(re.match(r'[\xe0-\xfe]', '\xd7', re.I))
+        if have_unicode:
+            self.assertTrue(re.match(u(r'[9-a]'), u(r'_'), re.U | re.I))
+            self.assertIsNone(re.match(u(r'[9-A]'), u(r'_'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\xc0-\xde]'),
+                                     u(r'\xd7'), re.U | re.I))
+            self.assertIsNone(re.match(u(r'[\xc0-\xde]'),
+                                       u(r'\xf7'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\xe0-\xfe]'),
+                                     u(r'\xf7'), re.U | re.I))
+            self.assertIsNone(re.match(u(r'[\xe0-\xfe]'),
+                                       u(r'\xd7'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u0430-\u045f]'),
+                                     u(r'\u0450'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u0430-\u045f]'),
+                                     u(r'\u0400'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u0400-\u042f]'),
+                                     u(r'\u0450'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u0400-\u042f]'),
+                                     u(r'\u0400'), re.U | re.I))
+            if sys.maxunicode > 0xffff:
+                self.assertTrue(re.match(u(r'[\U00010428-\U0001044f]'),
+                                         u(r'\U00010428'), re.U | re.I))
+                self.assertTrue(re.match(u(r'[\U00010428-\U0001044f]'),
+                                         u(r'\U00010400'), re.U | re.I))
+                self.assertTrue(re.match(u(r'[\U00010400-\U00010427]'),
+                                         u(r'\U00010428'), re.U | re.I))
+                self.assertTrue(re.match(u(r'[\U00010400-\U00010427]'),
+                                         u(r'\U00010400'), re.U | re.I))
+
+            assert u(r'\u212a').lower() == u'k' # '?'
+            self.assertTrue(re.match(ur'[J-M]', u(r'\u212a'), re.U | re.I))
+            self.assertTrue(re.match(ur'[j-m]', u(r'\u212a'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u2129-\u212b]'), u'K', re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u2129-\u212b]'), u'k', re.U | re.I))
+            assert u(r'\u017f').upper() == u'S' # '?'
+            self.assertTrue(re.match(ur'[R-T]', u(r'\u017f'), re.U | re.I))
+            self.assertTrue(re.match(ur'[r-t]', u(r'\u017f'), re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u017e-\u0180]'), u'S', re.U | re.I))
+            self.assertTrue(re.match(u(r'[\u017e-\u0180]'), u's', re.U | re.I))
+
     def test_category(self):
         self.assertEqual(re.match(r"(\s)", " ").group(1), " ")
 
@@ -409,7 +600,8 @@
         import _sre
         self.assertEqual(_sre.getlower(ord('A'), 0), ord('a'))
         self.assertEqual(_sre.getlower(ord('A'), re.LOCALE), ord('a'))
-        self.assertEqual(_sre.getlower(ord('A'), re.UNICODE), ord('a'))
+        if have_unicode:
+            self.assertEqual(_sre.getlower(ord('A'), re.UNICODE), ord('a'))
 
         self.assertEqual(re.match("abc", "ABC", re.I).group(0), "ABC")
         self.assertEqual(re.match("abc", u"ABC", re.I).group(0), "ABC")
@@ -422,17 +614,63 @@
         self.assertEqual(re.search("\s(b)", " b").group(1), "b")
         self.assertEqual(re.search("a\s", "a ").group(0), "a ")
 
+    def assertMatch(self, pattern, text, match=None, span=None,
+                    matcher=re.match):
+        if match is None and span is None:
+            # the pattern matches the whole text
+            match = text
+            span = (0, len(text))
+        elif match is None or span is None:
+            raise ValueError('If match is not None, span should be specified '
+                             '(and vice versa).')
+        m = matcher(pattern, text)
+        self.assertTrue(m)
+        self.assertEqual(m.group(), match)
+        self.assertEqual(m.span(), span)
+
+    @requires_unicode
     def test_re_escape(self):
-        p=""
-        for i in range(0, 256):
-            p = p + chr(i)
-            self.assertEqual(re.match(re.escape(chr(i)), chr(i)) is not None,
-                             True)
-            self.assertEqual(re.match(re.escape(chr(i)), chr(i)).span(), (0,1))
+        alnum_chars = unicode(string.ascii_letters + string.digits)
+        p = u''.join(unichr(i) for i in range(256))
+        for c in p:
+            if c in alnum_chars:
+                self.assertEqual(re.escape(c), c)
+            elif c == u'\x00':
+                self.assertEqual(re.escape(c), u'\\000')
+            else:
+                self.assertEqual(re.escape(c), u'\\' + c)
+            self.assertMatch(re.escape(c), c)
+        self.assertMatch(re.escape(p), p)
 
-        pat=re.compile(re.escape(p))
-        self.assertEqual(pat.match(p) is not None, True)
-        self.assertEqual(pat.match(p).span(), (0,256))
+    def test_re_escape_byte(self):
+        alnum_chars = string.ascii_letters + string.digits
+        p = ''.join(chr(i) for i in range(256))
+        for b in p:
+            if b in alnum_chars:
+                self.assertEqual(re.escape(b), b)
+            elif b == b'\x00':
+                self.assertEqual(re.escape(b), b'\\000')
+            else:
+                self.assertEqual(re.escape(b), b'\\' + b)
+            self.assertMatch(re.escape(b), b)
+        self.assertMatch(re.escape(p), p)
+
+    @requires_unicode
+    def test_re_escape_non_ascii(self):
+        s = u(r'xxx\u2620\u2620\u2620xxx')
+        s_escaped = re.escape(s)
+        self.assertEqual(s_escaped, u(r'xxx\\\u2620\\\u2620\\\u2620xxx'))
+        self.assertMatch(s_escaped, s)
+        self.assertMatch(u'.%s+.' % re.escape(unichr(0x2620)), s,
+                         u(r'x\u2620\u2620\u2620x'), (2, 7), re.search)
+
+    def test_re_escape_non_ascii_bytes(self):
+        b = b'y\xe2\x98\xa0y\xe2\x98\xa0y'
+        b_escaped = re.escape(b)
+        self.assertEqual(b_escaped, b'y\\\xe2\\\x98\\\xa0y\\\xe2\\\x98\\\xa0y')
+        self.assertMatch(b_escaped, b)
+        res = re.findall(re.escape(b'\xe2\x98\xa0'), b)
+        self.assertEqual(len(res), 2)
 
     def test_pickling(self):
         import pickle
@@ -440,20 +678,17 @@
         import cPickle
         self.pickle_test(cPickle)
         # old pickles expect the _compile() reconstructor in sre module
-        import warnings
-        original_filters = warnings.filters[:]
-        try:
-            warnings.filterwarnings("ignore", "The sre module is deprecated",
-                                    DeprecationWarning)
-            from sre import _compile
-        finally:
-            warnings.filters = original_filters
+        import_module("sre", deprecated=True)
+        from sre import _compile
+        # current pickle expects the _compile() reconstructor in re module
+        from re import _compile
 
     def pickle_test(self, pickle):
         oldpat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)')
-        s = pickle.dumps(oldpat)
-        newpat = pickle.loads(s)
-        self.assertEqual(oldpat, newpat)
+        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+            pickled = pickle.dumps(oldpat, proto)
+            newpat = pickle.loads(pickled)
+            self.assertEqual(newpat, oldpat)
 
     def test_constants(self):
         self.assertEqual(re.I, re.IGNORECASE)
@@ -464,26 +699,26 @@
 
     def test_flags(self):
         for flag in [re.I, re.M, re.X, re.S, re.L]:
-            self.assertNotEqual(re.compile('^pattern$', flag), None)
+            self.assertTrue(re.compile('^pattern$', flag))
 
     def test_sre_character_literals(self):
         for i in [0, 8, 16, 32, 64, 127, 128, 255]:
-            self.assertNotEqual(re.match(r"\%03o" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"\%03o0" % i, chr(i)+"0"), None)
-            self.assertNotEqual(re.match(r"\%03o8" % i, chr(i)+"8"), None)
-            self.assertNotEqual(re.match(r"\x%02x" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"\x%02x0" % i, chr(i)+"0"), None)
-            self.assertNotEqual(re.match(r"\x%02xz" % i, chr(i)+"z"), None)
+            self.assertTrue(re.match(r"\%03o" % i, chr(i)))
+            self.assertTrue(re.match(r"\%03o0" % i, chr(i)+"0"))
+            self.assertTrue(re.match(r"\%03o8" % i, chr(i)+"8"))
+            self.assertTrue(re.match(r"\x%02x" % i, chr(i)))
+            self.assertTrue(re.match(r"\x%02x0" % i, chr(i)+"0"))
+            self.assertTrue(re.match(r"\x%02xz" % i, chr(i)+"z"))
         self.assertRaises(re.error, re.match, "\911", "")
 
     def test_sre_character_class_literals(self):
         for i in [0, 8, 16, 32, 64, 127, 128, 255]:
-            self.assertNotEqual(re.match(r"[\%03o]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\%03o0]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\%03o8]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\x%02x]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\x%02x0]" % i, chr(i)), None)
-            self.assertNotEqual(re.match(r"[\x%02xz]" % i, chr(i)), None)
+            self.assertTrue(re.match(r"[\%03o]" % i, chr(i)))
+            self.assertTrue(re.match(r"[\%03o0]" % i, chr(i)))
+            self.assertTrue(re.match(r"[\%03o8]" % i, chr(i)))
+            self.assertTrue(re.match(r"[\x%02x]" % i, chr(i)))
+            self.assertTrue(re.match(r"[\x%02x0]" % i, chr(i)))
+            self.assertTrue(re.match(r"[\x%02xz]" % i, chr(i)))
         self.assertRaises(re.error, re.match, "[\911]", "")
 
     def test_bug_113254(self):
@@ -493,7 +728,7 @@
 
     def test_bug_527371(self):
         # bug described in patches 527371/672491
-        self.assertEqual(re.match(r'(a)?a','a').lastindex, None)
+        self.assertIsNone(re.match(r'(a)?a','a').lastindex)
         self.assertEqual(re.match(r'(a)(b)?b','ab').lastindex, 1)
         self.assertEqual(re.match(r'(?P<a>a)(?P<b>b)?b','ab').lastgroup, 'a')
         self.assertEqual(re.match("(?P<a>a(b))", "ab").lastgroup, 'a')
@@ -514,22 +749,29 @@
         self.assertEqual(re.match('.*?cd', 20000*'abc'+'de').end(0), 60001)
         # non-simple '*?' still used to hit the recursion limit, before the
         # non-recursive scheme was implemented.
+        self.assertEqual(re.search('(a|b)*?c', 10000*'ab'+'cd').end(0), 20001)
 
-        # does not apply for Jython, since we do not implement the
-        # non-recursive scheme
-        # self.assertEqual(re.search('(a|b)*?c', 10000*'ab'+'cd').end(0), 20001)
-
+    @requires_unicode
     def test_bug_612074(self):
-        pat=u"["+re.escape(u"\u2039")+u"]"
+        pat=u"["+re.escape(unichr(0x2039))+u"]"
         self.assertEqual(re.compile(pat) and 1, 1)
 
-    def not_valid_for_jython_implementation_test_stack_overflow(self):
+    def test_stack_overflow(self):
         # nasty cases that used to overflow the straightforward recursive
         # implementation of repeated groups.
         self.assertEqual(re.match('(x)*', 50000*'x').group(1), 'x')
         self.assertEqual(re.match('(x)*y', 50000*'x'+'y').group(1), 'x')
         self.assertEqual(re.match('(x)*?y', 50000*'x'+'y').group(1), 'x')
 
+    def test_unlimited_zero_width_repeat(self):
+        # Issue #9669
+        self.assertIsNone(re.match(r'(?:a?)*y', 'z'))
+        self.assertIsNone(re.match(r'(?:a?)+y', 'z'))
+        self.assertIsNone(re.match(r'(?:a?){2,}y', 'z'))
+        self.assertIsNone(re.match(r'(?:a?)*?y', 'z'))
+        self.assertIsNone(re.match(r'(?:a?)+?y', 'z'))
+        self.assertIsNone(re.match(r'(?:a?){2,}?y', 'z'))
+
     def test_scanner(self):
         def s_ident(scanner, token): return token
         def s_operator(scanner, token): return "op%s" % token
@@ -544,7 +786,7 @@
             (r"\s+", None),
             ])
 
-        self.assertNotEqual(scanner.scanner.scanner("").pattern, None)
+        self.assertTrue(scanner.scanner.scanner("").pattern)
 
         self.assertEqual(scanner.scan("sum = 3*foo + 312.50 + bar"),
                          (['sum', 'op=', 3, 'op*', 'foo', 'op+', 312.5,
@@ -585,35 +827,26 @@
         self.assertEqual(re.match('(a)((?!(b)*))*', 'abb').groups(),
                          ('a', None, None))
 
+    @requires_unicode
     def test_bug_764548(self):
         # bug 764548, re.compile() barfs on str/unicode subclasses
-        try:
-            unicode
-        except NameError:
-            return  # no problem if we have no unicode
         class my_unicode(unicode): pass
         pat = re.compile(my_unicode("abc"))
-        self.assertEqual(pat.match("xyz"), None)
+        self.assertIsNone(pat.match("xyz"))
 
     def test_finditer(self):
         iter = re.finditer(r":+", "a:b::c:::d")
         self.assertEqual([item.group(0) for item in iter],
                          [":", "::", ":::"])
 
+    @requires_unicode
     def test_bug_926075(self):
-        try:
-            unicode
-        except NameError:
-            return # no problem if we have no unicode
-        self.assert_(re.compile('bug_926075') is not
-                     re.compile(eval("u'bug_926075'")))
+        self.assertIsNot(re.compile('bug_926075'),
+                         re.compile(u'bug_926075'))
 
+    @requires_unicode
     def test_bug_931848(self):
-        try:
-            unicode
-        except NameError:
-            pass
-        pattern = eval('u"[\u002E\u3002\uFF0E\uFF61]"')
+        pattern = u(r"[\u002E\u3002\uFF0E\uFF61]")
         self.assertEqual(re.compile(pattern).split("a.b.c"),
                          ['a','b','c'])
 
@@ -624,7 +857,7 @@
 
         scanner = re.compile(r"\s").scanner("a b")
         self.assertEqual(scanner.search().span(), (1, 2))
-        self.assertEqual(scanner.search(), None)
+        self.assertIsNone(scanner.search())
 
     def test_bug_817234(self):
         iter = re.finditer(r".*", "asdf")
@@ -632,14 +865,40 @@
         self.assertEqual(iter.next().span(), (4, 4))
         self.assertRaises(StopIteration, iter.next)
 
+    @requires_unicode
+    def test_bug_6561(self):
+        # '\d' should match characters in Unicode category 'Nd'
+        # (Number, Decimal Digit), but not those in 'Nl' (Number,
+        # Letter) or 'No' (Number, Other).
+        decimal_digits = [
+            unichr(0x0037), # '\N{DIGIT SEVEN}', category 'Nd'
+            unichr(0x0e58), # '\N{THAI DIGIT SIX}', category 'Nd'
+            unichr(0xff10), # '\N{FULLWIDTH DIGIT ZERO}', category 'Nd'
+            ]
+        for x in decimal_digits:
+            self.assertEqual(re.match('^\d$', x, re.UNICODE).group(0), x)
+
+        not_decimal_digits = [
+            unichr(0x2165), # '\N{ROMAN NUMERAL SIX}', category 'Nl'
+            unichr(0x3039), # '\N{HANGZHOU NUMERAL TWENTY}', category 'Nl'
+            unichr(0x2082), # '\N{SUBSCRIPT TWO}', category 'No'
+            unichr(0x32b4), # '\N{CIRCLED NUMBER THIRTY NINE}', category 'No'
+            ]
+        for x in not_decimal_digits:
+            self.assertIsNone(re.match('^\d$', x, re.UNICODE))
+
     def test_empty_array(self):
         # SF buf 1647541
         import array
-        for typecode in 'cbBuhHiIlLfd':
+        typecodes = 'cbBhHiIlLfd'
+        if have_unicode:
+            typecodes += 'u'
+        for typecode in typecodes:
             a = array.array(typecode)
-            self.assertEqual(re.compile("bla").match(a), None)
+            self.assertIsNone(re.compile("bla").match(a))
             self.assertEqual(re.compile("").match(a).groups(), ())
 
+    @requires_unicode
     def test_inline_flags(self):
         # Bug #1700
         upper_char = unichr(0x1ea0) # Latin Capital Letter A with Dot Bellow
@@ -647,31 +906,234 @@
 
         p = re.compile(upper_char, re.I | re.U)
         q = p.match(lower_char)
-        self.assertNotEqual(q, None)
+        self.assertTrue(q)
 
         p = re.compile(lower_char, re.I | re.U)
         q = p.match(upper_char)
-        self.assertNotEqual(q, None)
+        self.assertTrue(q)
 
         p = re.compile('(?i)' + upper_char, re.U)
         q = p.match(lower_char)
-        self.assertNotEqual(q, None)
+        self.assertTrue(q)
 
         p = re.compile('(?i)' + lower_char, re.U)
         q = p.match(upper_char)
-        self.assertNotEqual(q, None)
+        self.assertTrue(q)
 
         p = re.compile('(?iu)' + upper_char)
         q = p.match(lower_char)
-        self.assertNotEqual(q, None)
+        self.assertTrue(q)
 
         p = re.compile('(?iu)' + lower_char)
         q = p.match(upper_char)
-        self.assertNotEqual(q, None)
+        self.assertTrue(q)
+
+    def test_dollar_matches_twice(self):
+        "$ matches the end of string, and just before the terminating \n"
+        pattern = re.compile('$')
+        self.assertEqual(pattern.sub('#', 'a\nb\n'), 'a\nb#\n#')
+        self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a\nb\nc#')
+        self.assertEqual(pattern.sub('#', '\n'), '#\n#')
+
+        pattern = re.compile('$', re.MULTILINE)
+        self.assertEqual(pattern.sub('#', 'a\nb\n' ), 'a#\nb#\n#' )
+        self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a#\nb#\nc#')
+        self.assertEqual(pattern.sub('#', '\n'), '#\n#')
+
+    def test_dealloc(self):
+        # issue 3299: check for segfault in debug build
+        import _sre
+        # the overflow limit is different on wide and narrow builds and it
+        # depends on the definition of SRE_CODE (see sre.h).
+        # 2**128 should be big enough to overflow on both. For smaller values
+        # a RuntimeError is raised instead of OverflowError.
+        long_overflow = 2**128
+        self.assertRaises(TypeError, re.finditer, "a", {})
+        self.assertRaises(OverflowError, _sre.compile, "abc", 0, [long_overflow])
+
+    def test_compile(self):
+        # Test return value when given string and pattern as parameter
+        pattern = re.compile('random pattern')
+        self.assertIsInstance(pattern, re._pattern_type)
+        same_pattern = re.compile(pattern)
+        self.assertIsInstance(same_pattern, re._pattern_type)
+        self.assertIs(same_pattern, pattern)
+        # Test behaviour when not given a string or pattern as parameter
+        self.assertRaises(TypeError, re.compile, 0)
+
+    def test_bug_13899(self):
+        # Issue #13899: re pattern r"[\A]" should work like "A" but matches
+        # nothing. Ditto B and Z.
+        self.assertEqual(re.findall(r'[\A\B\b\C\Z]', 'AB\bCZ'),
+                         ['A', 'B', '\b', 'C', 'Z'])
+
+    @precisionbigmemtest(size=_2G, memuse=1)
+    def test_large_search(self, size):
+        # Issue #10182: indices were 32-bit-truncated.
+        s = 'a' * size
+        m = re.search('$', s)
+        self.assertIsNotNone(m)
+        self.assertEqual(m.start(), size)
+        self.assertEqual(m.end(), size)
+
+    # The huge memuse is because of re.sub() using a list and a join()
+    # to create the replacement result.
+    @precisionbigmemtest(size=_2G, memuse=16 + 2)
+    def test_large_subn(self, size):
+        # Issue #10182: indices were 32-bit-truncated.
+        s = 'a' * size
+        r, n = re.subn('', '', s)
+        self.assertEqual(r, s)
+        self.assertEqual(n, size + 1)
+
+
+    def test_repeat_minmax_overflow(self):
+        # Issue #13169
+        string = "x" * 100000
+        self.assertEqual(re.match(r".{65535}", string).span(), (0, 65535))
+        self.assertEqual(re.match(r".{,65535}", string).span(), (0, 65535))
+        self.assertEqual(re.match(r".{65535,}?", string).span(), (0, 65535))
+        self.assertEqual(re.match(r".{65536}", string).span(), (0, 65536))
+        self.assertEqual(re.match(r".{,65536}", string).span(), (0, 65536))
+        self.assertEqual(re.match(r".{65536,}?", string).span(), (0, 65536))
+        # 2**128 should be big enough to overflow both SRE_CODE and Py_ssize_t.
+        self.assertRaises(OverflowError, re.compile, r".{%d}" % 2**128)
+        self.assertRaises(OverflowError, re.compile, r".{,%d}" % 2**128)
+        self.assertRaises(OverflowError, re.compile, r".{%d,}?" % 2**128)
+        self.assertRaises(OverflowError, re.compile, r".{%d,%d}" % (2**129, 2**128))
+
+    @cpython_only
+    def test_repeat_minmax_overflow_maxrepeat(self):
+        try:
+            from _sre import MAXREPEAT
+        except ImportError:
+            self.skipTest('requires _sre.MAXREPEAT constant')
+        string = "x" * 100000
+        self.assertIsNone(re.match(r".{%d}" % (MAXREPEAT - 1), string))
+        self.assertEqual(re.match(r".{,%d}" % (MAXREPEAT - 1), string).span(),
+                         (0, 100000))
+        self.assertIsNone(re.match(r".{%d,}?" % (MAXREPEAT - 1), string))
+        self.assertRaises(OverflowError, re.compile, r".{%d}" % MAXREPEAT)
+        self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT)
+        self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT)
+
+    def test_backref_group_name_in_exception(self):
+        # Issue 17341: Poor error message when compiling invalid regex
+        with self.assertRaisesRegexp(sre_constants.error, '<foo>'):
+            re.compile('(?P=<foo>)')
+
+    def test_group_name_in_exception(self):
+        # Issue 17341: Poor error message when compiling invalid regex
+        with self.assertRaisesRegexp(sre_constants.error, '\?foo'):
+            re.compile('(?P<?foo>)')
+
+    def test_issue17998(self):
+        for reps in '*', '+', '?', '{1}':
+            for mod in '', '?':
+                pattern = '.' + reps + mod + 'yz'
+                self.assertEqual(re.compile(pattern, re.S).findall('xyz'),
+                                 ['xyz'], msg=pattern)
+                if have_unicode:
+                    pattern = unicode(pattern)
+                    self.assertEqual(re.compile(pattern, re.S).findall(u'xyz'),
+                                     [u'xyz'], msg=pattern)
+
+
+    def test_bug_2537(self):
+        # issue 2537: empty submatches
+        for outer_op in ('{0,}', '*', '+', '{1,187}'):
+            for inner_op in ('{0,}', '*', '?'):
+                r = re.compile("^((x|y)%s)%s" % (inner_op, outer_op))
+                m = r.match("xyyzy")
+                self.assertEqual(m.group(0), "xyy")
+                self.assertEqual(m.group(1), "")
+                self.assertEqual(m.group(2), "y")
+
+    def test_debug_flag(self):
+        pat = r'(\.)(?:[ch]|py)(?(1)$|: )'
+        with captured_stdout() as out:
+            re.compile(pat, re.DEBUG)
+        dump = '''\
+subpattern 1
+  literal 46
+subpattern None
+  branch
+    in
+      literal 99
+      literal 104
+  or
+    literal 112
+    literal 121
+subpattern None
+  groupref_exists 1
+    at at_end
+  else
+    literal 58
+    literal 32
+'''
+        self.assertEqual(out.getvalue(), dump)
+        # Debug output is output again even a second time (bypassing
+        # the cache -- issue #20426).
+        with captured_stdout() as out:
+            re.compile(pat, re.DEBUG)
+        self.assertEqual(out.getvalue(), dump)
+
+    def test_keyword_parameters(self):
+        # Issue #20283: Accepting the string keyword parameter.
+        pat = re.compile(r'(ab)')
+        self.assertEqual(
+            pat.match(string='abracadabra', pos=7, endpos=10).span(), (7, 9))
+        self.assertEqual(
+            pat.search(string='abracadabra', pos=3, endpos=10).span(), (7, 9))
+        self.assertEqual(
+            pat.findall(string='abracadabra', pos=3, endpos=10), ['ab'])
+        self.assertEqual(
+            pat.split(string='abracadabra', maxsplit=1),
+            ['', 'ab', 'racadabra'])
+
+    def test_match_group_takes_long(self):
+        self.assertEqual(re.match("(foo)", "foo").group(1L), "foo")
+        self.assertRaises(IndexError, re.match("", "").group, sys.maxint + 1)
+
+    def test_locale_caching(self):
+        # Issue #22410
+        oldlocale = locale.setlocale(locale.LC_CTYPE)
+        self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale)
+        for loc in 'en_US.iso88591', 'en_US.utf8':
+            try:
+                locale.setlocale(locale.LC_CTYPE, loc)
+            except locale.Error:
+                # Unsupported locale on this system
+                self.skipTest('test needs %s locale' % loc)
+
+        re.purge()
+        self.check_en_US_iso88591()
+        self.check_en_US_utf8()
+        re.purge()
+        self.check_en_US_utf8()
+        self.check_en_US_iso88591()
+
+    def check_en_US_iso88591(self):
+        locale.setlocale(locale.LC_CTYPE, 'en_US.iso88591')
+        self.assertTrue(re.match(b'\xc5\xe5', b'\xc5\xe5', re.L|re.I))
+        self.assertTrue(re.match(b'\xc5', b'\xe5', re.L|re.I))
+        self.assertTrue(re.match(b'\xe5', b'\xc5', re.L|re.I))
+        self.assertTrue(re.match(b'(?Li)\xc5\xe5', b'\xc5\xe5'))
+        self.assertTrue(re.match(b'(?Li)\xc5', b'\xe5'))
+        self.assertTrue(re.match(b'(?Li)\xe5', b'\xc5'))
+
+    def check_en_US_utf8(self):
+        locale.setlocale(locale.LC_CTYPE, 'en_US.utf8')
+        self.assertTrue(re.match(b'\xc5\xe5', b'\xc5\xe5', re.L|re.I))
+        self.assertIsNone(re.match(b'\xc5', b'\xe5', re.L|re.I))
+        self.assertIsNone(re.match(b'\xe5', b'\xc5', re.L|re.I))
+        self.assertTrue(re.match(b'(?Li)\xc5\xe5', b'\xc5\xe5'))
+        self.assertIsNone(re.match(b'(?Li)\xc5', b'\xe5'))
+        self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5'))
 
 
 def run_re_tests():
-    from test.re_tests import benchmarks, tests, SUCCEED, FAIL, SYNTAX_ERROR
+    from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR
     if verbose:
         print 'Running re_tests test suite'
     else:

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Nov 20 19:54:34 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 21 Nov 2015 00:54:34 +0000
Subject: [Jython-checkins] =?utf-8?q?jython_=28merge_default_-=3E_default?=
 =?utf-8?q?=29=3A_Merge?=
Message-ID: <20151121005433.71401.92441@psf.io>

https://hg.python.org/jython/rev/12201b356322
changeset:   7819:12201b356322
parent:      7818:636b124a7587
parent:      7816:37578797caba
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sat Nov 21 11:54:25 2015 +1100
summary:
  Merge

files:
  Lib/_socket.py                           |    2 +
  Lib/test/test_socket_jy.py               |    3 -
  build.xml                                |   20 +++++-----
  extlibs/netty-buffer-4.0.31.Final.jar    |  Bin 
  extlibs/netty-buffer-4.0.33.Final.jar    |  Bin 
  extlibs/netty-codec-4.0.31.Final.jar     |  Bin 
  extlibs/netty-codec-4.0.33.Final.jar     |  Bin 
  extlibs/netty-common-4.0.31.Final.jar    |  Bin 
  extlibs/netty-common-4.0.33.Final.jar    |  Bin 
  extlibs/netty-handler-4.0.31.Final.jar   |  Bin 
  extlibs/netty-handler-4.0.33.Final.jar   |  Bin 
  extlibs/netty-transport-4.0.31.Final.jar |  Bin 
  extlibs/netty-transport-4.0.33.Final.jar |  Bin 
  13 files changed, 12 insertions(+), 13 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -1080,6 +1080,8 @@
                 self.channel.pipeline().remove(self.python_inbound_handler)
             except NoSuchElementException:
                 pass  # already removed, can safely ignore (presumably)
+            except AttributeError:
+                pass  # inbound handler never set up, also ignore
         if how & SHUT_WR or how & SHUT_RDWR:
             self._can_write = False
 
diff --git a/Lib/test/test_socket_jy.py b/Lib/test/test_socket_jy.py
--- a/Lib/test/test_socket_jy.py
+++ b/Lib/test/test_socket_jy.py
@@ -67,9 +67,6 @@
         """Verify connect_ex states go through EINPROGRESS?, EALREADY*, EISCONN"""
         # Tests fix for http://bugs.jython.org/issue2428; based in part on the
         # code showing failure that was submitted with that bug
-
-        #self.httpd, self.server_thread = start_server()
-        #self.address = self.httpd.server_name, self.httpd.server_port
         for result in self.do_workout():
             self.assertIn(result[0], {errno.EINPROGRESS, errno.EISCONN})
             self.assertEqual(result[-1], errno.EISCONN)
diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -177,11 +177,11 @@
             <pathelement path="${extlibs.dir}/jnr-posix-3.0.20.jar"/>
             <pathelement path="${extlibs.dir}/jnr-constants-0.9.0.jar"/>
             <pathelement path="${extlibs.dir}/jline-2.12.1.jar"/>
-            <pathelement path="${extlibs.dir}/netty-buffer-4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-codec-4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-common-4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-handler--4.0.31.Final.jar"/>
-            <pathelement path="${extlibs.dir}/netty-transport-4.0.31.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-buffer-4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-codec-4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-common-4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-handler--4.0.33.Final.jar"/>
+            <pathelement path="${extlibs.dir}/netty-transport-4.0.33.Final.jar"/>
         </path>
 
         <available property="informix.present" classname="com.informix.jdbc.IfxDriver" classpath="${informix.jar}" />
@@ -587,15 +587,15 @@
             <rule pattern="com.google.**" result="org.python.google. at 1"/>
             <zipfileset src="extlibs/icu4j-54_1_1.jar"/>
             <rule pattern="com.ibm.icu.**" result="org.python.icu. at 1"/>
-            <zipfileset src="extlibs/netty-buffer-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-buffer-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-codec-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-codec-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-common-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-common-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-handler-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-handler-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
-            <zipfileset src="extlibs/netty-transport-4.0.31.Final.jar" excludes="META-INF/**"/>
+            <zipfileset src="extlibs/netty-transport-4.0.33.Final.jar" excludes="META-INF/**"/>
             <rule pattern="io.netty.**" result="org.python.netty. at 1"/>
             <zipfileset src="extlibs/jffi-arm-Linux.jar"/>
             <zipfileset src="extlibs/jffi-Darwin.jar"/>
diff --git a/extlibs/netty-buffer-4.0.31.Final.jar b/extlibs/netty-buffer-4.0.31.Final.jar
deleted file mode 100644
index 759696378f5a57652b4db83462043bc5bca8b542..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-buffer-4.0.33.Final.jar b/extlibs/netty-buffer-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..5e644b4aa8aa4cb736415d71e6a53929fae23005
GIT binary patch
[stripped]
diff --git a/extlibs/netty-codec-4.0.31.Final.jar b/extlibs/netty-codec-4.0.31.Final.jar
deleted file mode 100644
index a943e6bf7361835342c8637c8db3588e87d70e19..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-codec-4.0.33.Final.jar b/extlibs/netty-codec-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b58013e04bcbcdf8eca12e986b1cd4704cadd1c5
GIT binary patch
[stripped]
diff --git a/extlibs/netty-common-4.0.31.Final.jar b/extlibs/netty-common-4.0.31.Final.jar
deleted file mode 100644
index ed507f2d323abfbd593442e32d9ecbd2ace149b4..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-common-4.0.33.Final.jar b/extlibs/netty-common-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b4990abb33b774c96ee0922b2b5505285fade9d5
GIT binary patch
[stripped]
diff --git a/extlibs/netty-handler-4.0.31.Final.jar b/extlibs/netty-handler-4.0.31.Final.jar
deleted file mode 100644
index 1a002fb26d8b46e7e132eabf952ba127d9ac2bb3..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-handler-4.0.33.Final.jar b/extlibs/netty-handler-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..ea26491f786d9d4463e1f09a8f7ddebacd9e8763
GIT binary patch
[stripped]
diff --git a/extlibs/netty-transport-4.0.31.Final.jar b/extlibs/netty-transport-4.0.31.Final.jar
deleted file mode 100644
index 87881ca0beac4e5c5aecb099acffd4d673922d55..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch
[stripped]
diff --git a/extlibs/netty-transport-4.0.33.Final.jar b/extlibs/netty-transport-4.0.33.Final.jar
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7be560332f0c25040e9def2f66ee5f7d7b0cec6c
GIT binary patch
[stripped]

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 21 09:10:42 2015
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 21 Nov 2015 14:10:42 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update/backport_distutils_w?=
 =?utf-8?q?ith_version_from_CPython-2=2E7=2E10?=
Message-ID: <20151121141042.127560.59283@psf.io>

https://hg.python.org/jython/rev/dcc76fd1c837
changeset:   7820:dcc76fd1c837
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sun Nov 22 01:10:34 2015 +1100
summary:
  Update/backport distutils with version from CPython-2.7.10

files:
  Lib/distutils/ccompiler.py                    |   56 +-
  Lib/distutils/command/bdist.py                |   82 +-
  Lib/distutils/command/bdist_dumb.py           |    9 +-
  Lib/distutils/command/build.py                |    6 +-
  Lib/distutils/command/build_scripts.py        |   39 +-
  Lib/distutils/command/install.py              |    4 +-
  Lib/distutils/command/install_scripts.py      |    4 +-
  Lib/distutils/command/sdist.py                |  339 +++++----
  Lib/distutils/file_util.py                    |   34 +-
  Lib/distutils/spawn.py                        |  121 ++-
  Lib/distutils/sysconfig.py                    |  240 +-----
  lib-python/2.7/distutils/__init__.py          |    2 +-
  lib-python/2.7/distutils/command/bdist_rpm.py |    1 +
  lib-python/2.7/distutils/command/build_ext.py |   10 +-
  lib-python/2.7/distutils/command/build_py.py  |    3 +-
  lib-python/2.7/distutils/command/check.py     |    8 +-
  lib-python/2.7/distutils/command/install.py   |    4 +-
  lib-python/2.7/distutils/command/sdist.py     |    2 +-
  lib-python/2.7/distutils/command/upload.py    |   20 +-
  lib-python/2.7/distutils/config.py            |    2 +-
  lib-python/2.7/distutils/core.py              |    7 +-
  lib-python/2.7/distutils/cygwinccompiler.py   |   28 +-
  lib-python/2.7/distutils/dir_util.py          |    6 +-
  lib-python/2.7/distutils/file_util.py         |   32 +-
  lib-python/2.7/distutils/spawn.py             |   63 +-
  lib-python/2.7/distutils/sysconfig.py         |   76 +-
  lib-python/2.7/distutils/text_file.py         |    4 +-
  lib-python/2.7/distutils/util.py              |   23 +-
  28 files changed, 547 insertions(+), 678 deletions(-)


diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py
--- a/Lib/distutils/ccompiler.py
+++ b/Lib/distutils/ccompiler.py
@@ -3,7 +3,7 @@
 Contains CCompiler, an abstract base class that defines the interface
 for the Distutils compiler abstraction model."""
 
-__revision__ = "$Id: ccompiler.py 86238 2010-11-06 04:06:18Z eric.araujo $"
+__revision__ = "$Id$"
 
 import sys
 import os
@@ -17,58 +17,8 @@
 from distutils.dep_util import newer_group
 from distutils.util import split_quoted, execute
 from distutils import log
-
-_sysconfig = __import__('sysconfig')
-
-def customize_compiler(compiler):
-    """Do any platform-specific customization of a CCompiler instance.
-
-    Mainly needed on Unix, so we can plug in the information that
-    varies across Unices and is stored in Python's Makefile.
-    """
-    if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
-            _sysconfig.get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                                       'CCSHARED', 'LDSHARED', 'SO', 'AR',
-                                       'ARFLAGS')
-
-        if 'CC' in os.environ:
-            cc = os.environ['CC']
-        if 'CXX' in os.environ:
-            cxx = os.environ['CXX']
-        if 'LDSHARED' in os.environ:
-            ldshared = os.environ['LDSHARED']
-        if 'CPP' in os.environ:
-            cpp = os.environ['CPP']
-        else:
-            cpp = cc + " -E"           # not always
-        if 'LDFLAGS' in os.environ:
-            ldshared = ldshared + ' ' + os.environ['LDFLAGS']
-        if 'CFLAGS' in os.environ:
-            cflags = opt + ' ' + os.environ['CFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CFLAGS']
-        if 'CPPFLAGS' in os.environ:
-            cpp = cpp + ' ' + os.environ['CPPFLAGS']
-            cflags = cflags + ' ' + os.environ['CPPFLAGS']
-            ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
-        if 'AR' in os.environ:
-            ar = os.environ['AR']
-        if 'ARFLAGS' in os.environ:
-            archiver = ar + ' ' + os.environ['ARFLAGS']
-        else:
-            archiver = ar + ' ' + ar_flags
-
-        cc_cmd = cc + ' ' + cflags
-        compiler.set_executables(
-            preprocessor=cpp,
-            compiler=cc_cmd,
-            compiler_so=cc_cmd + ' ' + ccshared,
-            compiler_cxx=cxx,
-            linker_so=ldshared,
-            linker_exe=cc,
-            archiver=archiver)
-
-        compiler.shared_lib_extension = so_ext
+# following import is for backward compatibility
+from distutils.sysconfig import customize_compiler
 
 class CCompiler:
     """Abstract base class to define the interface that must be implemented
diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py
--- a/Lib/distutils/command/bdist.py
+++ b/Lib/distutils/command/bdist.py
@@ -3,22 +3,20 @@
 Implements the Distutils 'bdist' command (create a built [binary]
 distribution)."""
 
-# This module should be kept compatible with Python 2.1.
-
-__revision__ = "$Id: bdist.py 62197 2008-04-07 01:53:39Z mark.hammond $"
+__revision__ = "$Id$"
 
 import os
-from types import *
+
+from distutils.util import get_platform
 from distutils.core import Command
-from distutils.errors import *
-from distutils.util import get_platform
+from distutils.errors import DistutilsPlatformError, DistutilsOptionError
 
 
-def show_formats ():
+def show_formats():
     """Print list of available formats (arguments to "--format" option).
     """
     from distutils.fancy_getopt import FancyGetopt
-    formats=[]
+    formats = []
     for format in bdist.format_commands:
         formats.append(("formats=" + format, None,
                         bdist.format_command[format][1]))
@@ -26,7 +24,7 @@
     pretty_printer.print_help("List of available distribution formats:")
 
 
-class bdist (Command):
+class bdist(Command):
 
     description = "create a built (binary) distribution"
 
@@ -42,6 +40,12 @@
                      "[default: dist]"),
                     ('skip-build', None,
                      "skip rebuilding everything (for testing/debugging)"),
+                    ('owner=', 'u',
+                     "Owner name used when creating a tar file"
+                     " [default: current user]"),
+                    ('group=', 'g',
+                     "Group name used when creating a tar file"
+                     " [default: current group]"),
                    ]
 
     boolean_options = ['skip-build']
@@ -52,50 +56,42 @@
         ]
 
     # The following commands do not take a format option from bdist
-    no_format_option = ('bdist_rpm',
-                        #'bdist_sdux', 'bdist_pkgtool'
-                        )
+    no_format_option = ('bdist_rpm',)
 
     # This won't do in reality: will need to distinguish RPM-ish Linux,
     # Debian-ish Linux, Solaris, FreeBSD, ..., Windows, Mac OS.
-    default_format = { 'posix': 'gztar',
-                       'java': 'gztar',
-                       'nt': 'zip',
-                       'os2': 'zip', }
+    default_format = {'posix': 'gztar',
+                      'java': 'gztar',
+                      'nt': 'zip',
+                      'os2': 'zip'}
 
     # Establish the preferred order (for the --help-formats option).
     format_commands = ['rpm', 'gztar', 'bztar', 'ztar', 'tar',
-                       'wininst', 'zip',
-                       #'pkgtool', 'sdux'
-                       ]
+                       'wininst', 'zip', 'msi']
 
     # And the real information.
-    format_command = { 'rpm':   ('bdist_rpm',  "RPM distribution"),
-                       'zip':   ('bdist_dumb', "ZIP file"),
-                       'gztar': ('bdist_dumb', "gzip'ed tar file"),
-                       'bztar': ('bdist_dumb', "bzip2'ed tar file"),
-                       'ztar':  ('bdist_dumb', "compressed tar file"),
-                       'tar':   ('bdist_dumb', "tar file"),
-                       'wininst': ('bdist_wininst',
-                                   "Windows executable installer"),
-                       'zip':   ('bdist_dumb', "ZIP file"),
-                       #'pkgtool': ('bdist_pkgtool',
-                       #            "Solaris pkgtool distribution"),
-                       #'sdux':  ('bdist_sdux', "HP-UX swinstall depot"),
+    format_command = {'rpm':   ('bdist_rpm',  "RPM distribution"),
+                      'gztar': ('bdist_dumb', "gzip'ed tar file"),
+                      'bztar': ('bdist_dumb', "bzip2'ed tar file"),
+                      'ztar':  ('bdist_dumb', "compressed tar file"),
+                      'tar':   ('bdist_dumb', "tar file"),
+                      'wininst': ('bdist_wininst',
+                                  "Windows executable installer"),
+                      'zip':   ('bdist_dumb', "ZIP file"),
+                      'msi':   ('bdist_msi',  "Microsoft Installer")
                       }
 
 
-    def initialize_options (self):
+    def initialize_options(self):
         self.bdist_base = None
         self.plat_name = None
         self.formats = None
         self.dist_dir = None
         self.skip_build = 0
+        self.group = None
+        self.owner = None
 
-    # initialize_options()
-
-
-    def finalize_options (self):
+    def finalize_options(self):
         # have to finalize 'plat_name' before 'bdist_base'
         if self.plat_name is None:
             if self.skip_build:
@@ -123,10 +119,7 @@
         if self.dist_dir is None:
             self.dist_dir = "dist"
 
-    # finalize_options()
-
-    def run (self):
-
+    def run(self):
         # Figure out which sub-commands we need to run.
         commands = []
         for format in self.formats:
@@ -142,12 +135,13 @@
             if cmd_name not in self.no_format_option:
                 sub_cmd.format = self.formats[i]
 
+            # passing the owner and group names for tar archiving
+            if cmd_name == 'bdist_dumb':
+                sub_cmd.owner = self.owner
+                sub_cmd.group = self.group
+
             # If we're going to need to run this command again, tell it to
             # keep its temporary files around so subsequent runs go faster.
             if cmd_name in commands[i+1:]:
                 sub_cmd.keep_temp = 1
             self.run_command(cmd_name)
-
-    # run()
-
-# class bdist
diff --git a/Lib/distutils/command/bdist_dumb.py b/Lib/distutils/command/bdist_dumb.py
--- a/Lib/distutils/command/bdist_dumb.py
+++ b/Lib/distutils/command/bdist_dumb.py
@@ -4,7 +4,7 @@
 distribution -- i.e., just an archive to be unpacked under $prefix or
 $exec_prefix)."""
 
-__revision__ = "$Id: bdist_dumb.py 77761 2010-01-26 22:46:15Z tarek.ziade $"
+__revision__ = "$Id$"
 
 import os
 
@@ -50,7 +50,7 @@
     default_format = { 'posix': 'gztar',
                        'java': 'gztar',
                        'nt': 'zip',
-                       'os2': 'zip' }
+                       'os2': 'zip'}
 
 
     def initialize_options (self):
@@ -59,7 +59,7 @@
         self.format = None
         self.keep_temp = 0
         self.dist_dir = None
-        self.skip_build = 0
+        self.skip_build = None
         self.relative = 0
         self.owner = None
         self.group = None
@@ -79,7 +79,8 @@
 
         self.set_undefined_options('bdist',
                                    ('dist_dir', 'dist_dir'),
-                                   ('plat_name', 'plat_name'))
+                                   ('plat_name', 'plat_name'),
+                                   ('skip_build', 'skip_build'))
 
     def run(self):
         if not self.skip_build:
diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py
--- a/Lib/distutils/command/build.py
+++ b/Lib/distutils/command/build.py
@@ -115,11 +115,7 @@
                                               'scripts-' + sys.version[0:3])
 
         if self.executable is None:
-            if not sys.executable is None:
-                self.executable = os.path.normpath(sys.executable)
-            else:
-                from org.python.core import Py
-                self.executable = Py.getDefaultExecutableName()
+            self.executable = os.path.normpath(sys.executable)
 
     def run(self):
         # Run all relevant sub-commands.  This will be some subset of:
diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py
--- a/Lib/distutils/command/build_scripts.py
+++ b/Lib/distutils/command/build_scripts.py
@@ -2,13 +2,10 @@
 
 Implements the Distutils 'build_scripts' command."""
 
-# This module should be kept compatible with Python 2.1.
+__revision__ = "$Id$"
 
-__revision__ = "$Id: build_scripts.py 59668 2008-01-02 18:59:36Z guido.van.rossum $"
-
-import sys, os, re
+import os, re
 from stat import ST_MODE
-from distutils import sysconfig
 from distutils.core import Command
 from distutils.dep_util import newer
 from distutils.util import convert_path
@@ -59,6 +56,7 @@
         ie. starts with "\#!" and contains "python"), then adjust the first
         line to refer to the current Python interpreter as we copy.
         """
+        _sysconfig = __import__('sysconfig')
         self.mkpath(self.build_dir)
         outfiles = []
         for script in self.scripts:
@@ -94,18 +92,23 @@
             if adjust:
                 log.info("copying and adjusting %s -> %s", script,
                          self.build_dir)
-                if not sysconfig.python_build:
-                    executable = self.executable
-                else:
-                    executable = os.path.join(
-                        sysconfig.get_config_var("BINDIR"),
-                        "python" + sysconfig.get_config_var("EXE"))
+                print "###############"
+                print executable
                 executable = fix_jython_executable(executable, post_interp)
+                print executable
                 if not self.dry_run:
                     outf = open(outfile, "w")
-                    outf.write("#!%s%s\n" %
-                               (executable,
-                                post_interp))
+                    if not _sysconfig.is_python_build():
+                        outf.write("#!%s%s\n" %
+                                   (self.executable,
+                                    post_interp))
+                    else:
+                        outf.write("#!%s%s\n" %
+                                   (os.path.join(
+                            _sysconfig.get_config_var("BINDIR"),
+                           "python%s%s" % (_sysconfig.get_config_var("VERSION"),
+                                           _sysconfig.get_config_var("EXE"))),
+                                    post_interp))
                     outf.writelines(f.readlines())
                     outf.close()
                 if f:
@@ -115,7 +118,7 @@
                     f.close()
                 self.copy_file(script, outfile)
 
-        if hasattr(os, 'chmod'):
+        if os.name == 'posix':
             for file in outfiles:
                 if self.dry_run:
                     log.info("changing mode of %s", file)
@@ -150,9 +153,9 @@
         if options:
             # Can't apply the workaround, leave it broken
             log.warn("WARNING: Unable to adapt shebang line for Jython,"
-                             " the following script is NOT executable\n"
+                     " the following script is NOT executable\n"
                      "         see http://bugs.jython.org/issue1112 for"
-                             " more information.")
+                     " more information.")
         else:
             return '/usr/bin/env %s' % executable
-    return executable
+    return executable
\ No newline at end of file
diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py
--- a/Lib/distutils/command/install.py
+++ b/Lib/distutils/command/install.py
@@ -279,8 +279,8 @@
 
         if self.user and (self.prefix or self.exec_prefix or self.home or
                 self.install_base or self.install_platbase):
-            raise DistutilsOptionError("can't combine user with with prefix/"
-                                       "exec_prefix/home or install_(plat)base")
+            raise DistutilsOptionError("can't combine user with prefix, "
+                                       "exec_prefix/home, or install_(plat)base")
 
         # Next, stuff that's wrong (or dubious) only on certain platforms.
         if os.name != "posix":
diff --git a/Lib/distutils/command/install_scripts.py b/Lib/distutils/command/install_scripts.py
--- a/Lib/distutils/command/install_scripts.py
+++ b/Lib/distutils/command/install_scripts.py
@@ -5,7 +5,7 @@
 
 # contributed by Bastian Kleineidam
 
-__revision__ = "$Id: install_scripts.py 68943 2009-01-25 22:09:10Z tarek.ziade $"
+__revision__ = "$Id$"
 
 import os
 from distutils.core import Command
@@ -44,7 +44,7 @@
         if not self.skip_build:
             self.run_command('build_scripts')
         self.outfiles = self.copy_tree(self.build_dir, self.install_dir)
-        if hasattr(os, 'chmod'):
+        if os.name == 'posix':
             # Set the executable bits (owner, group, and world) on
             # all the scripts we just installed.
             for file in self.get_outputs():
diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py
--- a/Lib/distutils/command/sdist.py
+++ b/Lib/distutils/command/sdist.py
@@ -2,40 +2,47 @@
 
 Implements the Distutils 'sdist' command (create a source distribution)."""
 
-# This module should be kept compatible with Python 2.1.
+__revision__ = "$Id$"
 
-__revision__ = "$Id: sdist.py 61268 2008-03-06 07:14:26Z martin.v.loewis $"
+import os
+import string
+import sys
+from glob import glob
+from warnings import warn
 
-import sys, os, string
-from types import *
-from glob import glob
 from distutils.core import Command
 from distutils import dir_util, dep_util, file_util, archive_util
 from distutils.text_file import TextFile
-from distutils.errors import *
+from distutils.errors import (DistutilsPlatformError, DistutilsOptionError,
+                              DistutilsTemplateError)
 from distutils.filelist import FileList
 from distutils import log
+from distutils.util import convert_path
 
-
-def show_formats ():
+def show_formats():
     """Print all possible values for the 'formats' option (used by
     the "--help-formats" command-line option).
     """
     from distutils.fancy_getopt import FancyGetopt
     from distutils.archive_util import ARCHIVE_FORMATS
-    formats=[]
+    formats = []
     for format in ARCHIVE_FORMATS.keys():
         formats.append(("formats=" + format, None,
                         ARCHIVE_FORMATS[format][2]))
     formats.sort()
-    pretty_printer = FancyGetopt(formats)
-    pretty_printer.print_help(
+    FancyGetopt(formats).print_help(
         "List of available source distribution formats:")
 
-class sdist (Command):
+class sdist(Command):
 
     description = "create a source distribution (tarball, zip file, etc.)"
 
+    def checking_metadata(self):
+        """Callable used for the check sub-command.
+
+        Placed here so user_options can view it"""
+        return self.metadata_check
+
     user_options = [
         ('template=', 't',
          "name of manifest template file [default: MANIFEST.in]"),
@@ -56,7 +63,8 @@
          "just regenerate the manifest and then stop "
          "(implies --force-manifest)"),
         ('force-manifest', 'f',
-         "forcibly regenerate the manifest and carry on as usual"),
+         "forcibly regenerate the manifest and carry on as usual. "
+         "Deprecated: now the manifest is always regenerated."),
         ('formats=', None,
          "formats for source distribution (comma-separated list)"),
         ('keep-temp', 'k',
@@ -65,11 +73,18 @@
         ('dist-dir=', 'd',
          "directory to put the source distribution archive(s) in "
          "[default: dist]"),
+        ('metadata-check', None,
+         "Ensure that all required elements of meta-data "
+         "are supplied. Warn if any missing. [default]"),
+        ('owner=', 'u',
+         "Owner name used when creating a tar file [default: current user]"),
+        ('group=', 'g',
+         "Group name used when creating a tar file [default: current group]"),
         ]
 
     boolean_options = ['use-defaults', 'prune',
                        'manifest-only', 'force-manifest',
-                       'keep-temp']
+                       'keep-temp', 'metadata-check']
 
     help_options = [
         ('help-formats', None,
@@ -79,11 +94,13 @@
     negative_opt = {'no-defaults': 'use-defaults',
                     'no-prune': 'prune' }
 
-    default_format = { 'posix': 'gztar',
-                       'java': 'gztar',
-                       'nt': 'zip' }
+    default_format = {'posix': 'gztar',
+                      'java': 'gztar',
+                      'nt': 'zip' }
 
-    def initialize_options (self):
+    sub_commands = [('check', checking_metadata)]
+
+    def initialize_options(self):
         # 'template' and 'manifest' are, respectively, the names of
         # the manifest template and manifest file.
         self.template = None
@@ -102,9 +119,11 @@
         self.dist_dir = None
 
         self.archive_files = None
+        self.metadata_check = 1
+        self.owner = None
+        self.group = None
 
-
-    def finalize_options (self):
+    def finalize_options(self):
         if self.manifest is None:
             self.manifest = "MANIFEST"
         if self.template is None:
@@ -127,16 +146,14 @@
         if self.dist_dir is None:
             self.dist_dir = "dist"
 
-
-    def run (self):
-
+    def run(self):
         # 'filelist' contains the list of files that will make up the
         # manifest
         self.filelist = FileList()
 
-        # Ensure that all required meta-data is given; warn if not (but
-        # don't die, it's not *that* serious!)
-        self.check_metadata()
+        # Run sub commands
+        for cmd_name in self.get_sub_commands():
+            self.run_command(cmd_name)
 
         # Do whatever it takes to get the list of files to process
         # (process the manifest template, read an existing manifest,
@@ -151,116 +168,63 @@
         # or zipfile, or whatever.
         self.make_distribution()
 
+    def check_metadata(self):
+        """Deprecated API."""
+        warn("distutils.command.sdist.check_metadata is deprecated, \
+              use the check command instead", PendingDeprecationWarning)
+        check = self.distribution.get_command_obj('check')
+        check.ensure_finalized()
+        check.run()
 
-    def check_metadata (self):
-        """Ensure that all required elements of meta-data (name, version,
-        URL, (author and author_email) or (maintainer and
-        maintainer_email)) are supplied by the Distribution object; warn if
-        any are missing.
-        """
-        metadata = self.distribution.metadata
-
-        missing = []
-        for attr in ('name', 'version', 'url'):
-            if not (hasattr(metadata, attr) and getattr(metadata, attr)):
-                missing.append(attr)
-
-        if missing:
-            self.warn("missing required meta-data: " +
-                      string.join(missing, ", "))
-
-        if metadata.author:
-            if not metadata.author_email:
-                self.warn("missing meta-data: if 'author' supplied, " +
-                          "'author_email' must be supplied too")
-        elif metadata.maintainer:
-            if not metadata.maintainer_email:
-                self.warn("missing meta-data: if 'maintainer' supplied, " +
-                          "'maintainer_email' must be supplied too")
-        else:
-            self.warn("missing meta-data: either (author and author_email) " +
-                      "or (maintainer and maintainer_email) " +
-                      "must be supplied")
-
-    # check_metadata ()
-
-
-    def get_file_list (self):
+    def get_file_list(self):
         """Figure out the list of files to include in the source
         distribution, and put it in 'self.filelist'.  This might involve
         reading the manifest template (and writing the manifest), or just
         reading the manifest, or just using the default file set -- it all
-        depends on the user's options and the state of the filesystem.
+        depends on the user's options.
         """
-
-        # If we have a manifest template, see if it's newer than the
-        # manifest; if so, we'll regenerate the manifest.
+        # new behavior when using a template:
+        # the file list is recalculated every time because
+        # even if MANIFEST.in or setup.py are not changed
+        # the user might have added some files in the tree that
+        # need to be included.
+        #
+        #  This makes --force the default and only behavior with templates.
         template_exists = os.path.isfile(self.template)
-        if template_exists:
-            template_newer = dep_util.newer(self.template, self.manifest)
-
-        # The contents of the manifest file almost certainly depend on the
-        # setup script as well as the manifest template -- so if the setup
-        # script is newer than the manifest, we'll regenerate the manifest
-        # from the template.  (Well, not quite: if we already have a
-        # manifest, but there's no template -- which will happen if the
-        # developer elects to generate a manifest some other way -- then we
-        # can't regenerate the manifest, so we don't.)
-        self.debug_print("checking if %s newer than %s" %
-                         (self.distribution.script_name, self.manifest))
-        setup_newer = dep_util.newer(self.distribution.script_name,
-                                     self.manifest)
-
-        # cases:
-        #   1) no manifest, template exists: generate manifest
-        #      (covered by 2a: no manifest == template newer)
-        #   2) manifest & template exist:
-        #      2a) template or setup script newer than manifest:
-        #          regenerate manifest
-        #      2b) manifest newer than both:
-        #          do nothing (unless --force or --manifest-only)
-        #   3) manifest exists, no template:
-        #      do nothing (unless --force or --manifest-only)
-        #   4) no manifest, no template: generate w/ warning ("defaults only")
-
-        manifest_outofdate = (template_exists and
-                              (template_newer or setup_newer))
-        force_regen = self.force_manifest or self.manifest_only
-        manifest_exists = os.path.isfile(self.manifest)
-        neither_exists = (not template_exists and not manifest_exists)
-
-        # Regenerate the manifest if necessary (or if explicitly told to)
-        if manifest_outofdate or neither_exists or force_regen:
-            if not template_exists:
-                self.warn(("manifest template '%s' does not exist " +
-                           "(using default file list)") %
-                          self.template)
-            self.filelist.findall()
-
-            if self.use_defaults:
-                self.add_defaults()
-            if template_exists:
-                self.read_template()
-            if self.prune:
-                self.prune_file_list()
-
+        if not template_exists and self._manifest_is_not_generated():
+            self.read_manifest()
             self.filelist.sort()
             self.filelist.remove_duplicates()
-            self.write_manifest()
+            return
 
-        # Don't regenerate the manifest, just read it in.
-        else:
-            self.read_manifest()
+        if not template_exists:
+            self.warn(("manifest template '%s' does not exist " +
+                        "(using default file list)") %
+                        self.template)
+        self.filelist.findall()
 
-    # get_file_list ()
+        if self.use_defaults:
+            self.add_defaults()
 
+        if template_exists:
+            self.read_template()
 
-    def add_defaults (self):
+        if self.prune:
+            self.prune_file_list()
+
+        self.filelist.sort()
+        self.filelist.remove_duplicates()
+        self.write_manifest()
+
+    def add_defaults(self):
         """Add all the default files to self.filelist:
           - README or README.txt
           - setup.py
           - test/test*.py
           - all pure Python modules mentioned in setup script
+          - all files pointed by package_data (build_py)
+          - all files defined in data_files.
+          - all files defined as scripts.
           - all C sources listed as part of extensions or C libraries
             in the setup script (doesn't catch C headers!)
         Warns if (README or README.txt) or setup.py are missing; everything
@@ -269,7 +233,7 @@
 
         standards = [('README', 'README.txt'), self.distribution.script_name]
         for fn in standards:
-            if type(fn) is TupleType:
+            if isinstance(fn, tuple):
                 alts = fn
                 got_it = 0
                 for fn in alts:
@@ -293,10 +257,35 @@
             if files:
                 self.filelist.extend(files)
 
+        # build_py is used to get:
+        #  - python modules
+        #  - files defined in package_data
+        build_py = self.get_finalized_command('build_py')
+
+        # getting python files
         if self.distribution.has_pure_modules():
-            build_py = self.get_finalized_command('build_py')
             self.filelist.extend(build_py.get_source_files())
 
+        # getting package_data files
+        # (computed in build_py.data_files by build_py.finalize_options)
+        for pkg, src_dir, build_dir, filenames in build_py.data_files:
+            for filename in filenames:
+                self.filelist.append(os.path.join(src_dir, filename))
+
+        # getting distribution.data_files
+        if self.distribution.has_data_files():
+            for item in self.distribution.data_files:
+                if isinstance(item, str): # plain file
+                    item = convert_path(item)
+                    if os.path.isfile(item):
+                        self.filelist.append(item)
+                else:    # a (dirname, filenames) tuple
+                    dirname, filenames = item
+                    for f in filenames:
+                        f = convert_path(f)
+                        if os.path.isfile(f):
+                            self.filelist.append(f)
+
         if self.distribution.has_ext_modules():
             build_ext = self.get_finalized_command('build_ext')
             self.filelist.extend(build_ext.get_source_files())
@@ -309,10 +298,7 @@
             build_scripts = self.get_finalized_command('build_scripts')
             self.filelist.extend(build_scripts.get_source_files())
 
-    # add_defaults ()
-
-
-    def read_template (self):
+    def read_template(self):
         """Read and parse manifest template file named by self.template.
 
         (usually "MANIFEST.in") The parsing and processing is done by
@@ -327,22 +313,25 @@
                             rstrip_ws=1,
                             collapse_join=1)
 
-        while 1:
-            line = template.readline()
-            if line is None:            # end of file
-                break
+        try:
+            while 1:
+                line = template.readline()
+                if line is None:            # end of file
+                    break
 
-            try:
-                self.filelist.process_template_line(line)
-            except DistutilsTemplateError, msg:
-                self.warn("%s, line %d: %s" % (template.filename,
-                                               template.current_line,
-                                               msg))
+                try:
+                    self.filelist.process_template_line(line)
+                # the call above can raise a DistutilsTemplateError for
+                # malformed lines, or a ValueError from the lower-level
+                # convert_path function
+                except (DistutilsTemplateError, ValueError) as msg:
+                    self.warn("%s, line %d: %s" % (template.filename,
+                                                   template.current_line,
+                                                   msg))
+        finally:
+            template.close()
 
-    # read_template ()
-
-
-    def prune_file_list (self):
+    def prune_file_list(self):
         """Prune off branches that might slip into the file list as created
         by 'read_template()', but really don't belong there:
           * the build tree (typically "build")
@@ -355,43 +344,62 @@
 
         self.filelist.exclude_pattern(None, prefix=build.build_base)
         self.filelist.exclude_pattern(None, prefix=base_dir)
-        self.filelist.exclude_pattern(r'(^|/)(RCS|CVS|\.svn|\.hg|\.git|\.bzr|_darcs)/.*', is_regex=1)
 
+        # pruning out vcs directories
+        # both separators are used under win32
+        if sys.platform == 'win32':
+            seps = r'/|\\'
+        else:
+            seps = '/'
 
-    def write_manifest (self):
+        vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr',
+                    '_darcs']
+        vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps)
+        self.filelist.exclude_pattern(vcs_ptrn, is_regex=1)
+
+    def write_manifest(self):
         """Write the file list in 'self.filelist' (presumably as filled in
         by 'add_defaults()' and 'read_template()') to the manifest file
         named by 'self.manifest'.
         """
-        self.execute(file_util.write_file,
-                     (self.manifest, self.filelist.files),
+        if self._manifest_is_not_generated():
+            log.info("not writing to manually maintained "
+                     "manifest file '%s'" % self.manifest)
+            return
+
+        content = self.filelist.files[:]
+        content.insert(0, '# file GENERATED by distutils, do NOT edit')
+        self.execute(file_util.write_file, (self.manifest, content),
                      "writing manifest file '%s'" % self.manifest)
 
-    # write_manifest ()
+    def _manifest_is_not_generated(self):
+        # check for special comment used in 2.7.1 and higher
+        if not os.path.isfile(self.manifest):
+            return False
 
+        fp = open(self.manifest, 'rU')
+        try:
+            first_line = fp.readline()
+        finally:
+            fp.close()
+        return first_line != '# file GENERATED by distutils, do NOT edit\n'
 
-    def read_manifest (self):
+    def read_manifest(self):
         """Read the manifest file (named by 'self.manifest') and use it to
         fill in 'self.filelist', the list of files to include in the source
         distribution.
         """
         log.info("reading manifest file '%s'", self.manifest)
         manifest = open(self.manifest)
-        try:
-            while 1:
-                line = manifest.readline()
-                if line == '':              # end of file
-                    break
-                if line[-1] == '\n':
-                    line = line[0:-1]
-                self.filelist.append(line)
-        finally:
-            manifest.close()
+        for line in manifest:
+            # ignore comments and blank lines
+            line = line.strip()
+            if line.startswith('#') or not line:
+                continue
+            self.filelist.append(line)
+        manifest.close()
 
-    # read_manifest ()
-
-
-    def make_release_tree (self, base_dir, files):
+    def make_release_tree(self, base_dir, files):
         """Create the directory tree that will become the source
         distribution archive.  All directories implied by the filenames in
         'files' are created under 'base_dir', and then we hard link or copy
@@ -433,9 +441,7 @@
 
         self.distribution.metadata.write_pkg_info(base_dir)
 
-    # make_release_tree ()
-
-    def make_distribution (self):
+    def make_distribution(self):
         """Create the source distribution(s).  First, we create the release
         tree with 'make_release_tree()'; then, we create all required
         archive files (according to 'self.formats') from the release tree.
@@ -450,8 +456,13 @@
 
         self.make_release_tree(base_dir, self.filelist.files)
         archive_files = []              # remember names of files we create
+        # tar archive must be created last to avoid overwrite and remove
+        if 'tar' in self.formats:
+            self.formats.append(self.formats.pop(self.formats.index('tar')))
+
         for fmt in self.formats:
-            file = self.make_archive(base_name, fmt, base_dir=base_dir)
+            file = self.make_archive(base_name, fmt, base_dir=base_dir,
+                                     owner=self.owner, group=self.group)
             archive_files.append(file)
             self.distribution.dist_files.append(('sdist', '', file))
 
@@ -460,10 +471,8 @@
         if not self.keep_temp:
             dir_util.remove_tree(base_dir, dry_run=self.dry_run)
 
-    def get_archive_files (self):
+    def get_archive_files(self):
         """Return the list of archive files created when the command
         was run, or None if the command hasn't run yet.
         """
         return self.archive_files
-
-# class sdist
diff --git a/Lib/distutils/file_util.py b/Lib/distutils/file_util.py
--- a/Lib/distutils/file_util.py
+++ b/Lib/distutils/file_util.py
@@ -3,7 +3,7 @@
 Utility functions for operating on single files.
 """
 
-__revision__ = "$Id: file_util.py 86238 2010-11-06 04:06:18Z eric.araujo $"
+__revision__ = "$Id$"
 
 import os
 from distutils.errors import DistutilsFileError
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher <da at ski.org>, utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode and hasattr(os, 'chmod'):
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher <da at ski.org>, utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py
--- a/Lib/distutils/spawn.py
+++ b/Lib/distutils/spawn.py
@@ -6,12 +6,13 @@
 executable name.
 """
 
-__revision__ = "$Id: spawn.py 73147 2009-06-02 15:58:43Z tarek.ziade $"
+__revision__ = "$Id$"
 
 import sys
 import os
 
 from distutils.errors import DistutilsPlatformError, DistutilsExecError
+from distutils.debug import DEBUG
 from distutils import log
 
 def spawn(cmd, search_path=1, verbose=0, dry_run=0):
@@ -30,6 +31,9 @@
     Raise DistutilsExecError if running the program fails in any way; just
     return on success.
     """
+    # cmd is documented as a list, but just in case some code passes a tuple
+    # in, protect our %-formatting code against horrible death
+    cmd = list(cmd)
     if os.name == 'posix':
         _spawn_posix(cmd, search_path, dry_run=dry_run)
     elif os.name == 'nt':
@@ -71,12 +75,16 @@
             rc = os.spawnv(os.P_WAIT, executable, cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
+            if not DEBUG:
+                cmd = executable
             raise DistutilsExecError, \
-                  "command '%s' failed: %s" % (cmd[0], exc[-1])
+                  "command %r failed: %s" % (cmd, exc[-1])
         if rc != 0:
             # and this reflects the command running but failing
+            if not DEBUG:
+                cmd = executable
             raise DistutilsExecError, \
-                  "command '%s' failed with exit status %d" % (cmd[0], rc)
+                  "command %r failed with exit status %d" % (cmd, rc)
 
 def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0):
     executable = cmd[0]
@@ -90,31 +98,68 @@
             rc = os.spawnv(os.P_WAIT, executable, cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
+            if not DEBUG:
+                cmd = executable
             raise DistutilsExecError, \
-                  "command '%s' failed: %s" % (cmd[0], exc[-1])
+                  "command %r failed: %s" % (cmd, exc[-1])
         if rc != 0:
             # and this reflects the command running but failing
-            log.debug("command '%s' failed with exit status %d" % (cmd[0], rc))
+            if not DEBUG:
+                cmd = executable
+            log.debug("command %r failed with exit status %d" % (cmd, rc))
             raise DistutilsExecError, \
-                  "command '%s' failed with exit status %d" % (cmd[0], rc)
+                  "command %r failed with exit status %d" % (cmd, rc)
 
+if sys.platform == 'darwin':
+    from distutils import sysconfig
+    _cfg_target = None
+    _cfg_target_split = None
 
 def _spawn_posix(cmd, search_path=1, verbose=0, dry_run=0):
     log.info(' '.join(cmd))
     if dry_run:
         return
+    executable = cmd[0]
     exec_fn = search_path and os.execvp or os.execv
+    env = None
+    if sys.platform == 'darwin':
+        global _cfg_target, _cfg_target_split
+        if _cfg_target is None:
+            _cfg_target = sysconfig.get_config_var(
+                                  'MACOSX_DEPLOYMENT_TARGET') or ''
+            if _cfg_target:
+                _cfg_target_split = [int(x) for x in _cfg_target.split('.')]
+        if _cfg_target:
+            # ensure that the deployment target of build process is not less
+            # than that used when the interpreter was built. This ensures
+            # extension modules are built with correct compatibility values
+            cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target)
+            if _cfg_target_split > [int(x) for x in cur_target.split('.')]:
+                my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: '
+                          'now "%s" but "%s" during configure'
+                                % (cur_target, _cfg_target))
+                raise DistutilsPlatformError(my_msg)
+            env = dict(os.environ,
+                       MACOSX_DEPLOYMENT_TARGET=cur_target)
+            exec_fn = search_path and os.execvpe or os.execve
     pid = os.fork()
 
     if pid == 0:  # in the child
         try:
-            exec_fn(cmd[0], cmd)
+            if env is None:
+                exec_fn(executable, cmd)
+            else:
+                exec_fn(executable, cmd, env)
         except OSError, e:
-            sys.stderr.write("unable to execute %s: %s\n" %
-                             (cmd[0], e.strerror))
+            if not DEBUG:
+                cmd = executable
+            sys.stderr.write("unable to execute %r: %s\n" %
+                             (cmd, e.strerror))
             os._exit(1)
 
-        sys.stderr.write("unable to execute %s for unknown reasons" % cmd[0])
+        if not DEBUG:
+            cmd = executable
+        sys.stderr.write("unable to execute %r for unknown reasons" % cmd)
         os._exit(1)
     else:   # in the parent
         # Loop until the child either exits or is terminated by a signal
@@ -126,50 +171,58 @@
                 import errno
                 if exc.errno == errno.EINTR:
                     continue
+                if not DEBUG:
+                    cmd = executable
                 raise DistutilsExecError, \
-                      "command '%s' failed: %s" % (cmd[0], exc[-1])
+                      "command %r failed: %s" % (cmd, exc[-1])
             if os.WIFSIGNALED(status):
+                if not DEBUG:
+                    cmd = executable
                 raise DistutilsExecError, \
-                      "command '%s' terminated by signal %d" % \
-                      (cmd[0], os.WTERMSIG(status))
+                      "command %r terminated by signal %d" % \
+                      (cmd, os.WTERMSIG(status))
 
             elif os.WIFEXITED(status):
                 exit_status = os.WEXITSTATUS(status)
                 if exit_status == 0:
                     return   # hey, it succeeded!
                 else:
+                    if not DEBUG:
+                        cmd = executable
                     raise DistutilsExecError, \
-                          "command '%s' failed with exit status %d" % \
-                          (cmd[0], exit_status)
+                          "command %r failed with exit status %d" % \
+                          (cmd, exit_status)
 
             elif os.WIFSTOPPED(status):
                 continue
 
             else:
+                if not DEBUG:
+                    cmd = executable
                 raise DistutilsExecError, \
-                      "unknown error executing '%s': termination status %d" % \
-                      (cmd[0], status)
+                      "unknown error executing %r: termination status %d" % \
+                      (cmd, status)
 
-def _spawn_java(cmd,
-                search_path=1,
-                verbose=0,
-                dry_run=0):
+def _spawn_java(cmd, search_path=1, verbose=0, dry_run=0):
+    log.info(' '.join(cmd))
+    if dry_run:
+        return
+
     executable = cmd[0]
     cmd = ' '.join(_nt_quote_args(cmd))
-    log.info(cmd)
-    if not dry_run:
-        try:
-            rc = os.system(cmd) >> 8
-        except OSError, exc:
-            # this seems to happen when the command isn't found
-            raise DistutilsExecError, \
-                  "command '%s' failed: %s" % (executable, exc[-1])
-        if rc != 0:
-            # and this reflects the command running but failing
-            print "command '%s' failed with exit status %d" % (executable, rc)
-            raise DistutilsExecError, \
-                  "command '%s' failed with exit status %d" % (executable, rc)
 
+    try:
+        rc = os.system(cmd) >> 8
+    except OSError as exc:
+        # this seems to happen when the command isn't found
+        raise DistutilsExecError, \
+            "command '%s' failed: %s" % (executable, exc[-1])
+
+    if rc != 0:
+        # and this reflects the command running but failing
+        print "command '%s' failed with exit status %d" % (executable, rc)
+        raise DistutilsExecError, \
+            "command '%s' failed with exit status %d" % (executable, rc)
 
 def find_executable(executable, path=None):
     """Tries to find 'executable' in the directories listed in 'path'.
diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py
--- a/Lib/distutils/sysconfig.py
+++ b/Lib/distutils/sysconfig.py
@@ -9,7 +9,7 @@
 Email:        <fdrake at acm.org>
 """
 
-__revision__ = "$Id: sysconfig.py 83688 2010-08-03 21:18:06Z mark.dickinson $"
+__revision__ = "$Id$"
 
 import os
 import re
@@ -27,6 +27,7 @@
 # live in project/PCBuild9.  If we're dealing with an x64 Windows build,
 # it'll live in project/PCbuild/amd64.
 
+
 def getJythonBinDir():
     if not sys.executable is None:
         return os.path.dirname(os.path.realpath(sys.executable))
@@ -45,6 +46,11 @@
     project_base = os.path.abspath(os.path.join(project_base, os.path.pardir,
                                                 os.path.pardir))
 
+# set for cross builds
+if "_PYTHON_PROJECT_BASE" in os.environ:
+    # this is the build directory, at least for posix
+    project_base = os.path.normpath(os.environ["_PYTHON_PROJECT_BASE"])
+
 # python_build: (Boolean) if true, we're either building Python or
 # building an extension with an un-installed Python, so we use
 # different (hard-wired) directories.
@@ -96,11 +102,6 @@
         return os.path.join(prefix, "include", "python" + get_python_version())
     elif os.name == "nt":
         return os.path.join(prefix, "include")
-    elif os.name == "mac":
-        if plat_specific:
-            return os.path.join(prefix, "Mac", "Include")
-        else:
-            return os.path.join(prefix, "Include")
     elif os.name == "os2" or os.name == "java":
         return os.path.join(prefix, "Include")
     else:
@@ -143,18 +144,6 @@
             else:
                 return os.path.join(prefix, "Lib", "site-packages")
 
-    elif os.name == "mac":
-        if plat_specific:
-            if standard_lib:
-                return os.path.join(prefix, "Lib", "lib-dynload")
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-        else:
-            if standard_lib:
-                return os.path.join(prefix, "Lib")
-            else:
-                return os.path.join(prefix, "Lib", "site-packages")
-
     elif os.name == "os2" or os.name == "java":
         if standard_lib:
             return os.path.join(prefix, "Lib")
@@ -167,6 +156,7 @@
             "on platform '%s'" % os.name)
 
 
+
 def customize_compiler(compiler):
     """Do any platform-specific customization of a CCompiler instance.
 
@@ -174,12 +164,36 @@
     varies across Unices and is stored in Python's Makefile.
     """
     if compiler.compiler_type == "unix":
-        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext) = \
+        if sys.platform == "darwin":
+            # Perform first-time customization of compiler-related
+            # config vars on OS X now that we know we need a compiler.
+            # This is primarily to support Pythons from binary
+            # installers.  The kind and paths to build tools on
+            # the user system may vary significantly from the system
+            # that Python itself was built on.  Also the user OS
+            # version and build tools may not support the same set
+            # of CPU architectures for universal builds.
+            global _config_vars
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
+                import _osx_support
+                _osx_support.customize_compiler(_config_vars)
+                _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
+
+        (cc, cxx, opt, cflags, ccshared, ldshared, so_ext, ar, ar_flags) = \
             get_config_vars('CC', 'CXX', 'OPT', 'CFLAGS',
-                            'CCSHARED', 'LDSHARED', 'SO')
+                            'CCSHARED', 'LDSHARED', 'SO', 'AR',
+                            'ARFLAGS')
 
         if 'CC' in os.environ:
-            cc = os.environ['CC']
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
         if 'LDSHARED' in os.environ:
@@ -197,6 +211,12 @@
             cpp = cpp + ' ' + os.environ['CPPFLAGS']
             cflags = cflags + ' ' + os.environ['CPPFLAGS']
             ldshared = ldshared + ' ' + os.environ['CPPFLAGS']
+        if 'AR' in os.environ:
+            ar = os.environ['AR']
+        if 'ARFLAGS' in os.environ:
+            archiver = ar + ' ' + os.environ['ARFLAGS']
+        else:
+            archiver = ar + ' ' + ar_flags
 
         cc_cmd = cc + ' ' + cflags
         compiler.set_executables(
@@ -205,7 +225,8 @@
             compiler_so=cc_cmd + ' ' + ccshared,
             compiler_cxx=cxx,
             linker_so=ldshared,
-            linker_exe=cc)
+            linker_exe=cc,
+            archiver=archiver)
 
         compiler.shared_lib_extension = so_ext
 
@@ -230,8 +251,7 @@
 def get_makefile_filename():
     """Return full pathname of installed Makefile from the Python build."""
     if python_build:
-        return os.path.join(getJythonBinDir(),
-                            "Makefile")
+        return os.path.join(getJythonBinDir(), "Makefile")
     lib_dir = get_python_lib(plat_specific=1, standard_lib=1)
     return os.path.join(lib_dir, "config", "Makefile")
 
@@ -344,6 +364,11 @@
 
     fp.close()
 
+    # strip spurious spaces
+    for k, v in done.items():
+        if isinstance(v, str):
+            done[k] = v.strip()
+
     # save the results in the global dictionary
     g.update(done)
     return g
@@ -378,81 +403,11 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On MacOSX we need to check the setting of the environment variable
-    # MACOSX_DEPLOYMENT_TARGET: configure bases some choices on it so
-    # it needs to be compatible.
-    # If it isn't set we set it to the configure-time value
-    if sys.platform == 'darwin' and 'MACOSX_DEPLOYMENT_TARGET' in g:
-        cfg_target = g['MACOSX_DEPLOYMENT_TARGET']
-        cur_target = os.getenv('MACOSX_DEPLOYMENT_TARGET', '')
-        if cur_target == '':
-            cur_target = cfg_target
-            os.putenv('MACOSX_DEPLOYMENT_TARGET', cfg_target)
-        elif map(int, cfg_target.split('.')) > map(int, cur_target.split('.')):
-            my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: now "%s" but "%s" during configure'
-                % (cur_target, cfg_target))
-            raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
+    # _sysconfigdata is generated at build time, see the sysconfig module
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars = {}
+    _config_vars.update(build_time_vars)
 
 
 def _init_nt():
@@ -474,32 +429,6 @@
     _config_vars = g
 
 
-def _init_mac():
-    """Initialize the module as appropriate for Macintosh systems"""
-    g = {}
-    # set basic install directories
-    g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1)
-    g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1)
-
-    # XXX hmmm.. a normal install puts include files here
-    g['INCLUDEPY'] = get_python_inc(plat_specific=0)
-
-    import MacOS
-    if not hasattr(MacOS, 'runtimemodel'):
-        g['SO'] = '.ppc.slb'
-    else:
-        g['SO'] = '.%s.slb' % MacOS.runtimemodel
-
-    # XXX are these used anywhere?
-    g['install_lib'] = os.path.join(EXEC_PREFIX, "Lib")
-    g['install_platlib'] = os.path.join(EXEC_PREFIX, "Mac", "Lib")
-
-    # These are used by the extension module build
-    g['srcdir'] = ':'
-    global _config_vars
-    _config_vars = g
-
-
 def _init_os2():
     """Initialize the module as appropriate for OS/2"""
     g = {}
@@ -553,66 +482,11 @@
         _config_vars['prefix'] = PREFIX
         _config_vars['exec_prefix'] = EXEC_PREFIX
 
+        # OS X platforms require special customization to handle
+        # multi-architecture, multi-os-version installers
         if sys.platform == 'darwin':
-            kernel_version = os.uname()[2] # Kernel version (8.4.3)
-            major_version = int(kernel_version.split('.')[0])
-
-            if major_version < 8:
-                # On Mac OS X before 10.4, check if -arch and -isysroot
-                # are in CFLAGS or LDFLAGS and remove them if they are.
-                # This is needed when building extensions on a 10.3 system
-                # using a universal build of python.
-                for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-                    flags = _config_vars[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                    flags = re.sub('-isysroot [^ \t]*', ' ', flags)
-                    _config_vars[key] = flags
-
-            else:
-
-                # Allow the user to override the architecture flags using
-                # an environment variable.
-                # NOTE: This name was introduced by Apple in OSX 10.5 and
-                # is used by several scripting languages distributed with
-                # that OS release.
-
-                if 'ARCHFLAGS' in os.environ:
-                    arch = os.environ['ARCHFLAGS']
-                    for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                        # a number of derived variables. These need to be
-                        # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                        flags = _config_vars[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
-                        flags = flags + ' ' + arch
-                        _config_vars[key] = flags
-
-                # If we're on OSX 10.5 or later and the user tries to
-                # compiles an extension using an SDK that is not present
-                # on the current machine it is better to not use an SDK
-                # than to fail.
-                #
-                # The major usecase for this is users using a Python.org
-                # binary installer  on OSX 10.6: that installer uses
-                # the 10.4u SDK, but that SDK is not installed by default
-                # when you install Xcode.
-                #
-                m = re.search('-isysroot\s+(\S+)', _config_vars['CFLAGS'])
-                if m is not None:
-                    sdk = m.group(1)
-                    if not os.path.exists(sdk):
-                        for key in ('LDFLAGS', 'BASECFLAGS', 'LDSHARED',
-                             # a number of derived variables. These need to be
-                             # patched up as well.
-                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
-
-                            flags = _config_vars[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
-                            _config_vars[key] = flags
+            import _osx_support
+            _osx_support.customize_config_vars(_config_vars)
 
     if args:
         vals = []
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.3"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/bdist_rpm.py b/lib-python/2.7/distutils/command/bdist_rpm.py
--- a/lib-python/2.7/distutils/command/bdist_rpm.py
+++ b/lib-python/2.7/distutils/command/bdist_rpm.py
@@ -12,6 +12,7 @@
 from distutils.core import Command
 from distutils.debug import DEBUG
 from distutils.file_util import write_file
+from distutils.sysconfig import get_python_version
 from distutils.errors import (DistutilsOptionError, DistutilsPlatformError,
                               DistutilsFileError, DistutilsExecError)
 from distutils import log
diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py
--- a/lib-python/2.7/distutils/command/build_ext.py
+++ b/lib-python/2.7/distutils/command/build_ext.py
@@ -231,13 +231,11 @@
                 # building python standard extensions
                 self.library_dirs.append('.')
 
-        # for extensions under Linux or Solaris with a shared Python library,
+        # For building extensions with a shared Python library,
         # Python's library directory must be appended to library_dirs
-        sysconfig.get_config_var('Py_ENABLE_SHARED')
-        if ((sys.platform.startswith('linux') or sys.platform.startswith('gnu')
-             or sys.platform.startswith('sunos'))
-            and sysconfig.get_config_var('Py_ENABLE_SHARED')):
-            if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")):
+        # See Issues: #1600860, #4366
+        if (sysconfig.get_config_var('Py_ENABLE_SHARED')):
+            if not sysconfig.python_build:
                 # building third party extensions
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
diff --git a/lib-python/2.7/distutils/command/build_py.py b/lib-python/2.7/distutils/command/build_py.py
--- a/lib-python/2.7/distutils/command/build_py.py
+++ b/lib-python/2.7/distutils/command/build_py.py
@@ -128,7 +128,8 @@
             # Each pattern has to be converted to a platform-specific path
             filelist = glob(os.path.join(src_dir, convert_path(pattern)))
             # Files that match more than one pattern are only added once
-            files.extend([fn for fn in filelist if fn not in files])
+            files.extend([fn for fn in filelist if fn not in files
+                and os.path.isfile(fn)])
         return files
 
     def build_package_data(self):
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/command/install.py b/lib-python/2.7/distutils/command/install.py
--- a/lib-python/2.7/distutils/command/install.py
+++ b/lib-python/2.7/distutils/command/install.py
@@ -265,8 +265,8 @@
 
         if self.user and (self.prefix or self.exec_prefix or self.home or
                 self.install_base or self.install_platbase):
-            raise DistutilsOptionError("can't combine user with with prefix/"
-                                       "exec_prefix/home or install_(plat)base")
+            raise DistutilsOptionError("can't combine user with prefix, "
+                                       "exec_prefix/home, or install_(plat)base")
 
         # Next, stuff that's wrong (or dubious) only on certain platforms.
         if os.name != "posix":
diff --git a/lib-python/2.7/distutils/command/sdist.py b/lib-python/2.7/distutils/command/sdist.py
--- a/lib-python/2.7/distutils/command/sdist.py
+++ b/lib-python/2.7/distutils/command/sdist.py
@@ -183,7 +183,7 @@
         depends on the user's options.
         """
         # new behavior when using a template:
-        # the file list is recalculated everytime because
+        # the file list is recalculated every time because
         # even if MANIFEST.in or setup.py are not changed
         # the user might have added some files in the tree that
         # need to be included.
diff --git a/lib-python/2.7/distutils/command/upload.py b/lib-python/2.7/distutils/command/upload.py
--- a/lib-python/2.7/distutils/command/upload.py
+++ b/lib-python/2.7/distutils/command/upload.py
@@ -10,7 +10,7 @@
 import cStringIO as StringIO
 from hashlib import md5
 
-from distutils.errors import DistutilsOptionError
+from distutils.errors import DistutilsError, DistutilsOptionError
 from distutils.core import PyPIRCCommand
 from distutils.spawn import spawn
 from distutils import log
@@ -136,8 +136,8 @@
 
         # Build up the MIME payload for the POST data
         boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
-        sep_boundary = '\n--' + boundary
-        end_boundary = sep_boundary + '--'
+        sep_boundary = '\r\n--' + boundary
+        end_boundary = sep_boundary + '--\r\n'
         body = StringIO.StringIO()
         for key, value in data.items():
             # handle multiple entries for the same name
@@ -151,14 +151,13 @@
                     fn = ""
 
                 body.write(sep_boundary)
-                body.write('\nContent-Disposition: form-data; name="%s"'%key)
+                body.write('\r\nContent-Disposition: form-data; name="%s"' % key)
                 body.write(fn)
-                body.write("\n\n")
+                body.write("\r\n\r\n")
                 body.write(value)
                 if value and value[-1] == '\r':
                     body.write('\n')  # write an extra newline (lurve Macs)
         body.write(end_boundary)
-        body.write("\n")
         body = body.getvalue()
 
         self.announce("Submitting %s to %s" % (filename, self.repository), log.INFO)
@@ -177,11 +176,11 @@
             status = result.getcode()
             reason = result.msg
             if self.show_response:
-                msg = '\n'.join(('-' * 75, r.read(), '-' * 75))
+                msg = '\n'.join(('-' * 75, result.read(), '-' * 75))
                 self.announce(msg, log.INFO)
         except socket.error, e:
             self.announce(str(e), log.ERROR)
-            return
+            raise
         except HTTPError, e:
             status = e.code
             reason = e.msg
@@ -190,5 +189,6 @@
             self.announce('Server response (%s): %s' % (status, reason),
                           log.INFO)
         else:
-            self.announce('Upload failed (%s): %s' % (status, reason),
-                          log.ERROR)
+            msg = 'Upload failed (%s): %s' % (status, reason)
+            self.announce(msg, log.ERROR)
+            raise DistutilsError(msg)
diff --git a/lib-python/2.7/distutils/config.py b/lib-python/2.7/distutils/config.py
--- a/lib-python/2.7/distutils/config.py
+++ b/lib-python/2.7/distutils/config.py
@@ -21,7 +21,7 @@
 class PyPIRCCommand(Command):
     """Base command that knows how to handle the .pypirc file
     """
-    DEFAULT_REPOSITORY = 'http://pypi.python.org/pypi'
+    DEFAULT_REPOSITORY = 'https://pypi.python.org/pypi'
     DEFAULT_REALM = 'pypi'
     repository = None
     realm = None
diff --git a/lib-python/2.7/distutils/core.py b/lib-python/2.7/distutils/core.py
--- a/lib-python/2.7/distutils/core.py
+++ b/lib-python/2.7/distutils/core.py
@@ -14,7 +14,6 @@
 from distutils.debug import DEBUG
 from distutils.errors import (DistutilsSetupError, DistutilsArgError,
                               DistutilsError, CCompilerError)
-from distutils.util import grok_environment_error
 
 # Mainly import these so setup scripts can "from distutils.core import" them.
 from distutils.dist import Distribution
@@ -153,13 +152,11 @@
         except KeyboardInterrupt:
             raise SystemExit, "interrupted"
         except (IOError, os.error), exc:
-            error = grok_environment_error(exc)
-
             if DEBUG:
-                sys.stderr.write(error + "\n")
+                sys.stderr.write("error: %s\n" % (exc,))
                 raise
             else:
-                raise SystemExit, error
+                raise SystemExit, "error: %s" % (exc,)
 
         except (DistutilsError,
                 CCompilerError), msg:
diff --git a/lib-python/2.7/distutils/cygwinccompiler.py b/lib-python/2.7/distutils/cygwinccompiler.py
--- a/lib-python/2.7/distutils/cygwinccompiler.py
+++ b/lib-python/2.7/distutils/cygwinccompiler.py
@@ -319,13 +319,18 @@
         else:
             entry_point = ''
 
-        self.set_executables(compiler='gcc -mno-cygwin -O -Wall',
-                             compiler_so='gcc -mno-cygwin -mdll -O -Wall',
-                             compiler_cxx='g++ -mno-cygwin -O -Wall',
-                             linker_exe='gcc -mno-cygwin',
-                             linker_so='%s -mno-cygwin %s %s'
-                                        % (self.linker_dll, shared_option,
-                                           entry_point))
+        if self.gcc_version < '4' or is_cygwingcc():
+            no_cygwin = ' -mno-cygwin'
+        else:
+            no_cygwin = ''
+
+        self.set_executables(compiler='gcc%s -O -Wall' % no_cygwin,
+                             compiler_so='gcc%s -mdll -O -Wall' % no_cygwin,
+                             compiler_cxx='g++%s -O -Wall' % no_cygwin,
+                             linker_exe='gcc%s' % no_cygwin,
+                             linker_so='%s%s %s %s'
+                                    % (self.linker_dll, no_cygwin,
+                                       shared_option, entry_point))
         # Maybe we should also append -mthreads, but then the finished
         # dlls need another dll (mingwm10.dll see Mingw32 docs)
         # (-mthreads: Support thread-safe exception handling on `Mingw32')
@@ -447,3 +452,12 @@
     else:
         dllwrap_version = None
     return (gcc_version, ld_version, dllwrap_version)
+
+def is_cygwingcc():
+    '''Try to determine if the gcc that would be used is from cygwin.'''
+    out = os.popen('gcc -dumpmachine', 'r')
+    out_string = out.read()
+    out.close()
+    # out_string is the target triplet cpu-vendor-os
+    # Cygwin's gcc sets the os to 'cygwin'
+    return out_string.strip().endswith('cygwin')
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
@@ -185,7 +185,6 @@
     Any errors are ignored (apart from being reported to stdout if 'verbose'
     is true).
     """
-    from distutils.util import grok_environment_error
     global _path_created
 
     if verbose >= 1:
@@ -202,8 +201,7 @@
             if abspath in _path_created:
                 del _path_created[abspath]
         except (IOError, OSError), exc:
-            log.warn(grok_environment_error(
-                    exc, "error removing %s: " % directory))
+            log.warn("error removing %s: %s", directory, exc)
 
 def ensure_relative(path):
     """Take the full path 'path', and make it a relative path.
diff --git a/lib-python/2.7/distutils/file_util.py b/lib-python/2.7/distutils/file_util.py
--- a/lib-python/2.7/distutils/file_util.py
+++ b/lib-python/2.7/distutils/file_util.py
@@ -85,7 +85,8 @@
     (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
     None (the default), files are copied.  Don't set 'link' on systems that
     don't support it: 'copy_file()' doesn't check if hard or symbolic
-    linking is available.
+    linking is available. If hardlink fails, falls back to
+    _copy_file_contents().
 
     Under Mac OS, uses the native file copy function in macostools; on
     other systems, uses '_copy_file_contents()' to copy file contents.
@@ -137,24 +138,31 @@
     # (Unix only, of course, but that's the caller's responsibility)
     if link == 'hard':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
-            os.link(src, dst)
+            try:
+                os.link(src, dst)
+                return (dst, 1)
+            except OSError:
+                # If hard linking fails, fall back on copying file
+                # (some special filesystems don't support hard linking
+                #  even under Unix, see issue #8876).
+                pass
     elif link == 'sym':
         if not (os.path.exists(dst) and os.path.samefile(src, dst)):
             os.symlink(src, dst)
+            return (dst, 1)
 
     # Otherwise (non-Mac, not linking), copy the file contents and
     # (optionally) copy the times and mode.
-    else:
-        _copy_file_contents(src, dst)
-        if preserve_mode or preserve_times:
-            st = os.stat(src)
+    _copy_file_contents(src, dst)
+    if preserve_mode or preserve_times:
+        st = os.stat(src)
 
-            # According to David Ascher <da at ski.org>, utime() should be done
-            # before chmod() (at least under NT).
-            if preserve_times:
-                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
-            if preserve_mode:
-                os.chmod(dst, S_IMODE(st[ST_MODE]))
+        # According to David Ascher <da at ski.org>, utime() should be done
+        # before chmod() (at least under NT).
+        if preserve_times:
+            os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
+        if preserve_mode:
+            os.chmod(dst, S_IMODE(st[ST_MODE]))
 
     return (dst, 1)
 
diff --git a/lib-python/2.7/distutils/spawn.py b/lib-python/2.7/distutils/spawn.py
--- a/lib-python/2.7/distutils/spawn.py
+++ b/lib-python/2.7/distutils/spawn.py
@@ -12,6 +12,7 @@
 import os
 
 from distutils.errors import DistutilsPlatformError, DistutilsExecError
+from distutils.debug import DEBUG
 from distutils import log
 
 def spawn(cmd, search_path=1, verbose=0, dry_run=0):
@@ -30,6 +31,9 @@
     Raise DistutilsExecError if running the program fails in any way; just
     return on success.
     """
+    # cmd is documented as a list, but just in case some code passes a tuple
+    # in, protect our %-formatting code against horrible death
+    cmd = list(cmd)
     if os.name == 'posix':
         _spawn_posix(cmd, search_path, dry_run=dry_run)
     elif os.name == 'nt':
@@ -69,12 +73,16 @@
             rc = os.spawnv(os.P_WAIT, executable, cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
+            if not DEBUG:
+                cmd = executable
             raise DistutilsExecError, \
-                  "command '%s' failed: %s" % (cmd[0], exc[-1])
+                  "command %r failed: %s" % (cmd, exc[-1])
         if rc != 0:
             # and this reflects the command running but failing
+            if not DEBUG:
+                cmd = executable
             raise DistutilsExecError, \
-                  "command '%s' failed with exit status %d" % (cmd[0], rc)
+                  "command %r failed with exit status %d" % (cmd, rc)
 
 def _spawn_os2(cmd, search_path=1, verbose=0, dry_run=0):
     executable = cmd[0]
@@ -88,13 +96,17 @@
             rc = os.spawnv(os.P_WAIT, executable, cmd)
         except OSError, exc:
             # this seems to happen when the command isn't found
+            if not DEBUG:
+                cmd = executable
             raise DistutilsExecError, \
-                  "command '%s' failed: %s" % (cmd[0], exc[-1])
+                  "command %r failed: %s" % (cmd, exc[-1])
         if rc != 0:
             # and this reflects the command running but failing
-            log.debug("command '%s' failed with exit status %d" % (cmd[0], rc))
+            if not DEBUG:
+                cmd = executable
+            log.debug("command %r failed with exit status %d" % (cmd, rc))
             raise DistutilsExecError, \
-                  "command '%s' failed with exit status %d" % (cmd[0], rc)
+                  "command %r failed with exit status %d" % (cmd, rc)
 
 if sys.platform == 'darwin':
     from distutils import sysconfig
@@ -105,8 +117,9 @@
     log.info(' '.join(cmd))
     if dry_run:
         return
+    executable = cmd[0]
     exec_fn = search_path and os.execvp or os.execv
-    exec_args = [cmd[0], cmd]
+    env = None
     if sys.platform == 'darwin':
         global _cfg_target, _cfg_target_split
         if _cfg_target is None:
@@ -127,18 +140,24 @@
             env = dict(os.environ,
                        MACOSX_DEPLOYMENT_TARGET=cur_target)
             exec_fn = search_path and os.execvpe or os.execve
-            exec_args.append(env)
     pid = os.fork()
 
     if pid == 0:  # in the child
         try:
-            exec_fn(*exec_args)
+            if env is None:
+                exec_fn(executable, cmd)
+            else:
+                exec_fn(executable, cmd, env)
         except OSError, e:
-            sys.stderr.write("unable to execute %s: %s\n" %
-                             (cmd[0], e.strerror))
+            if not DEBUG:
+                cmd = executable
+            sys.stderr.write("unable to execute %r: %s\n" %
+                             (cmd, e.strerror))
             os._exit(1)
 
-        sys.stderr.write("unable to execute %s for unknown reasons" % cmd[0])
+        if not DEBUG:
+            cmd = executable
+        sys.stderr.write("unable to execute %r for unknown reasons" % cmd)
         os._exit(1)
     else:   # in the parent
         # Loop until the child either exits or is terminated by a signal
@@ -150,29 +169,37 @@
                 import errno
                 if exc.errno == errno.EINTR:
                     continue
+                if not DEBUG:
+                    cmd = executable
                 raise DistutilsExecError, \
-                      "command '%s' failed: %s" % (cmd[0], exc[-1])
+                      "command %r failed: %s" % (cmd, exc[-1])
             if os.WIFSIGNALED(status):
+                if not DEBUG:
+                    cmd = executable
                 raise DistutilsExecError, \
-                      "command '%s' terminated by signal %d" % \
-                      (cmd[0], os.WTERMSIG(status))
+                      "command %r terminated by signal %d" % \
+                      (cmd, os.WTERMSIG(status))
 
             elif os.WIFEXITED(status):
                 exit_status = os.WEXITSTATUS(status)
                 if exit_status == 0:
                     return   # hey, it succeeded!
                 else:
+                    if not DEBUG:
+                        cmd = executable
                     raise DistutilsExecError, \
-                          "command '%s' failed with exit status %d" % \
-                          (cmd[0], exit_status)
+                          "command %r failed with exit status %d" % \
+                          (cmd, exit_status)
 
             elif os.WIFSTOPPED(status):
                 continue
 
             else:
+                if not DEBUG:
+                    cmd = executable
                 raise DistutilsExecError, \
-                      "unknown error executing '%s': termination status %d" % \
-                      (cmd[0], status)
+                      "unknown error executing %r: termination status %d" % \
+                      (cmd, status)
 
 def find_executable(executable, path=None):
     """Tries to find 'executable' in the directories listed in 'path'.
diff --git a/lib-python/2.7/distutils/sysconfig.py b/lib-python/2.7/distutils/sysconfig.py
--- a/lib-python/2.7/distutils/sysconfig.py
+++ b/lib-python/2.7/distutils/sysconfig.py
@@ -165,7 +165,8 @@
             # version and build tools may not support the same set
             # of CPU architectures for universal builds.
             global _config_vars
-            if not _config_vars.get('CUSTOMIZED_OSX_COMPILER', ''):
+            # Use get_config_var() to ensure _config_vars is initialized.
+            if not get_config_var('CUSTOMIZED_OSX_COMPILER'):
                 import _osx_support
                 _osx_support.customize_compiler(_config_vars)
                 _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True'
@@ -175,9 +176,15 @@
                             'CCSHARED', 'LDSHARED', 'SO', 'AR',
                             'ARFLAGS')
 
-        newcc = None
         if 'CC' in os.environ:
-            cc = os.environ['CC']
+            newcc = os.environ['CC']
+            if (sys.platform == 'darwin'
+                    and 'LDSHARED' not in os.environ
+                    and ldshared.startswith(cc)):
+                # On OS X, if CC is overridden, use that as the default
+                #       command for LDSHARED as well
+                ldshared = newcc + ldshared[len(cc):]
+            cc = newcc
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
         if 'LDSHARED' in os.environ:
@@ -387,66 +394,11 @@
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    g = {}
-    # load the installed Makefile:
-    try:
-        filename = get_makefile_filename()
-        parse_makefile(filename, g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # load the installed pyconfig.h:
-    try:
-        filename = get_config_h_filename()
-        parse_config_h(file(filename), g)
-    except IOError, msg:
-        my_msg = "invalid Python installation: unable to open %s" % filename
-        if hasattr(msg, "strerror"):
-            my_msg = my_msg + " (%s)" % msg.strerror
-
-        raise DistutilsPlatformError(my_msg)
-
-    # On AIX, there are wrong paths to the linker scripts in the Makefile
-    # -- these paths are relative to the Python source, but when installed
-    # the scripts are in another directory.
-    if python_build:
-        g['LDSHARED'] = g['BLDSHARED']
-
-    elif get_python_version() < '2.1':
-        # The following two branches are for 1.5.2 compatibility.
-        if sys.platform == 'aix4':          # what about AIX 3.x ?
-            # Linker script is in the config directory, not in Modules as the
-            # Makefile says.
-            python_lib = get_python_lib(standard_lib=1)
-            ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix')
-            python_exp = os.path.join(python_lib, 'config', 'python.exp')
-
-            g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp)
-
-        elif sys.platform == 'beos':
-            # Linker script is in the config directory.  In the Makefile it is
-            # relative to the srcdir, which after installation no longer makes
-            # sense.
-            python_lib = get_python_lib(standard_lib=1)
-            linkerscript_path = string.split(g['LDSHARED'])[0]
-            linkerscript_name = os.path.basename(linkerscript_path)
-            linkerscript = os.path.join(python_lib, 'config',
-                                        linkerscript_name)
-
-            # XXX this isn't the right place to do this: adding the Python
-            # library to the link, if needed, should be in the "build_ext"
-            # command.  (It's also needed for non-MS compilers on Windows, and
-            # it's taken care of for them by the 'build_ext.get_libraries()'
-            # method.)
-            g['LDSHARED'] = ("%s -L%s/lib -lpython%s" %
-                             (linkerscript, PREFIX, get_python_version()))
-
+    # _sysconfigdata is generated at build time, see the sysconfig module
+    from _sysconfigdata import build_time_vars
     global _config_vars
-    _config_vars = g
+    _config_vars = {}
+    _config_vars.update(build_time_vars)
 
 
 def _init_nt():
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/distutils/util.py b/lib-python/2.7/distutils/util.py
--- a/lib-python/2.7/distutils/util.py
+++ b/lib-python/2.7/distutils/util.py
@@ -213,25 +213,10 @@
 
 
 def grok_environment_error (exc, prefix="error: "):
-    """Generate a useful error message from an EnvironmentError (IOError or
-    OSError) exception object.  Handles Python 1.5.1 and 1.5.2 styles, and
-    does what it can to deal with exception objects that don't have a
-    filename (which happens when the error is due to a two-file operation,
-    such as 'rename()' or 'link()'.  Returns the error message as a string
-    prefixed with 'prefix'.
-    """
-    # check for Python 1.5.2-style {IO,OS}Error exception objects
-    if hasattr(exc, 'filename') and hasattr(exc, 'strerror'):
-        if exc.filename:
-            error = prefix + "%s: %s" % (exc.filename, exc.strerror)
-        else:
-            # two-argument functions in posix module don't
-            # include the filename in the exception object!
-            error = prefix + "%s" % exc.strerror
-    else:
-        error = prefix + str(exc[-1])
-
-    return error
+    # Function kept for backward compatibility.
+    # Used to try clever things with EnvironmentErrors,
+    # but nowadays str(exception) produces good messages.
+    return prefix + str(exc)
 
 
 # Needed by 'split_quoted()'

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Nov 21 11:59:17 2015
From: jython-checkins at python.org (jim.baker)
Date: Sat, 21 Nov 2015 16:59:17 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Use_value_equality=2C_not_i?=
 =?utf-8?q?dentity=2C_when_working_with_SRE_ops=2E_Fixes_=232392=2E?=
Message-ID: <20151121165916.9714.14421@psf.io>

https://hg.python.org/jython/rev/f048badad7f7
changeset:   7821:f048badad7f7
user:        Jim Baker <jim.baker at rackspace.com>
date:        Sat Nov 21 09:59:10 2015 -0700
summary:
  Use value equality, not identity, when working with SRE ops. Fixes #2392.

The modules sre_compile.py and sre_parse.py, which implement
compilation and parsing of the SRE regex engine used by Jython (and
ported from CPython), use identity comparisons against operator
constants. But if these modules are reloaded, these are no longer
singletons the resulting mismatch causes spurious failures.

files:
  Lib/sre_compile.py |   74 +-
  Lib/sre_parse.py   |  801 +++++++++++++++++++++++++++++++++
  2 files changed, 838 insertions(+), 37 deletions(-)


diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -51,7 +51,7 @@
             else:
                 emit(OPCODES[op])
                 emit(av)
-        elif op is IN:
+        elif op == IN:
             if flags & SRE_FLAG_IGNORECASE:
                 emit(OPCODES[OP_IGNORE[op]])
                 def fixup(literal, flags=flags):
@@ -62,7 +62,7 @@
             skip = _len(code); emit(0)
             _compile_charset(av, flags, code, fixup)
             code[skip] = _len(code) - skip
-        elif op is ANY:
+        elif op == ANY:
             if flags & SRE_FLAG_DOTALL:
                 emit(OPCODES[ANY_ALL])
             else:
@@ -77,8 +77,8 @@
                 _compile(code, av[2], flags)
                 emit(OPCODES[SUCCESS])
                 code[skip] = _len(code) - skip
-            elif _simple(av) and op is not REPEAT:
-                if op is MAX_REPEAT:
+            elif _simple(av) and op != REPEAT:
+                if op == MAX_REPEAT:
                     emit(OPCODES[REPEAT_ONE])
                 else:
                     emit(OPCODES[MIN_REPEAT_ONE])
@@ -95,11 +95,11 @@
                 emit(av[1])
                 _compile(code, av[2], flags)
                 code[skip] = _len(code) - skip
-                if op is MAX_REPEAT:
+                if op == MAX_REPEAT:
                     emit(OPCODES[MAX_UNTIL])
                 else:
                     emit(OPCODES[MIN_UNTIL])
-        elif op is SUBPATTERN:
+        elif op == SUBPATTERN:
             if av[0]:
                 emit(OPCODES[MARK])
                 emit((av[0]-1)*2)
@@ -123,13 +123,13 @@
             _compile(code, av[1], flags)
             emit(OPCODES[SUCCESS])
             code[skip] = _len(code) - skip
-        elif op is CALL:
+        elif op == CALL:
             emit(OPCODES[op])
             skip = _len(code); emit(0)
             _compile(code, av, flags)
             emit(OPCODES[SUCCESS])
             code[skip] = _len(code) - skip
-        elif op is AT:
+        elif op == AT:
             emit(OPCODES[op])
             if flags & SRE_FLAG_MULTILINE:
                 av = AT_MULTILINE.get(av, av)
@@ -138,7 +138,7 @@
             elif flags & SRE_FLAG_UNICODE:
                 av = AT_UNICODE.get(av, av)
             emit(ATCODES[av])
-        elif op is BRANCH:
+        elif op == BRANCH:
             emit(OPCODES[op])
             tail = []
             tailappend = tail.append
@@ -152,20 +152,20 @@
             emit(0) # end of branch
             for tail in tail:
                 code[tail] = _len(code) - tail
-        elif op is CATEGORY:
+        elif op == CATEGORY:
             emit(OPCODES[op])
             if flags & SRE_FLAG_LOCALE:
                 av = CH_LOCALE[av]
             elif flags & SRE_FLAG_UNICODE:
                 av = CH_UNICODE[av]
             emit(CHCODES[av])
-        elif op is GROUPREF:
+        elif op == GROUPREF:
             if flags & SRE_FLAG_IGNORECASE:
                 emit(OPCODES[OP_IGNORE[op]])
             else:
                 emit(OPCODES[op])
             emit(av-1)
-        elif op is GROUPREF_EXISTS:
+        elif op == GROUPREF_EXISTS:
             emit(OPCODES[op])
             emit(av[0]-1)
             skipyes = _len(code); emit(0)
@@ -188,18 +188,18 @@
         fixup = _identityfunction
     for op, av in _optimize_charset(charset, fixup):
         emit(OPCODES[op])
-        if op is NEGATE:
+        if op == NEGATE:
             pass
-        elif op is LITERAL:
+        elif op == LITERAL:
             emit(fixup(av))
-        elif op is RANGE:
+        elif op == RANGE:
             emit(fixup(av[0]))
             emit(fixup(av[1]))
-        elif op is CHARSET:
+        elif op == CHARSET:
             code.extend(av)
-        elif op is BIGCHARSET:
+        elif op == BIGCHARSET:
             code.extend(av)
-        elif op is CATEGORY:
+        elif op == CATEGORY:
             if flags & SRE_FLAG_LOCALE:
                 emit(CHCODES[CH_LOCALE[av]])
             elif flags & SRE_FLAG_UNICODE:
@@ -217,14 +217,14 @@
     charmap = [0]*256
     try:
         for op, av in charset:
-            if op is NEGATE:
+            if op == NEGATE:
                 outappend((op, av))
-            elif op is LITERAL:
+            elif op == LITERAL:
                 charmap[fixup(av)] = 1
-            elif op is RANGE:
+            elif op == RANGE:
                 for i in range(fixup(av[0]), fixup(av[1])+1):
                     charmap[i] = 1
-            elif op is CATEGORY:
+            elif op == CATEGORY:
                 # XXX: could append to charmap tail
                 return charset # cannot compress
     except IndexError:
@@ -279,7 +279,7 @@
     return data
 
 # To represent a big charset, first a bitmap of all characters in the
-# set is constructed. Then, this bitmap is sliced into chunks of 256
+# set is constructed. Then, this bitmap == sliced into chunks of 256
 # characters, duplicate chunks are eliminated, and each chunk is
 # given a number. In the compiled expression, the charset is
 # represented by a 16-bit word sequence, consisting of one word for
@@ -316,14 +316,14 @@
     negate = 0
     try:
         for op, av in charset:
-            if op is NEGATE:
+            if op == NEGATE:
                 negate = 1
-            elif op is LITERAL:
+            elif op == LITERAL:
                 charmap[fixup(av)] = 1
-            elif op is RANGE:
+            elif op == RANGE:
                 for i in xrange(fixup(av[0]), fixup(av[1])+1):
                     charmap[i] = 1
-            elif op is CATEGORY:
+            elif op == CATEGORY:
                 # XXX: could expand category
                 return charset # cannot compress
     except IndexError:
@@ -385,13 +385,13 @@
     if not (flags & SRE_FLAG_IGNORECASE):
         # look for literal prefix
         for op, av in pattern.data:
-            if op is LITERAL:
+            if op == LITERAL:
                 if len(prefix) == prefix_skip:
                     prefix_skip = prefix_skip + 1
                 prefixappend(av)
-            elif op is SUBPATTERN and len(av[1]) == 1:
+            elif op == SUBPATTERN and len(av[1]) == 1:
                 op, av = av[1][0]
-                if op is LITERAL:
+                if op == LITERAL:
                     prefixappend(av)
                 else:
                     break
@@ -400,37 +400,37 @@
         # if no prefix, look for charset prefix
         if not prefix and pattern.data:
             op, av = pattern.data[0]
-            if op is SUBPATTERN and av[1]:
+            if op == SUBPATTERN and av[1]:
                 op, av = av[1][0]
-                if op is LITERAL:
+                if op == LITERAL:
                     charsetappend((op, av))
-                elif op is BRANCH:
+                elif op == BRANCH:
                     c = []
                     cappend = c.append
                     for p in av[1]:
                         if not p:
                             break
                         op, av = p[0]
-                        if op is LITERAL:
+                        if op == LITERAL:
                             cappend((op, av))
                         else:
                             break
                     else:
                         charset = c
-            elif op is BRANCH:
+            elif op == BRANCH:
                 c = []
                 cappend = c.append
                 for p in av[1]:
                     if not p:
                         break
                     op, av = p[0]
-                    if op is LITERAL:
+                    if op == LITERAL:
                         cappend((op, av))
                     else:
                         break
                 else:
                     charset = c
-            elif op is IN:
+            elif op == IN:
                 charset = av
 ##     if prefix:
 ##         print "*** PREFIX", prefix, prefix_skip
diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py
new file mode 100644
--- /dev/null
+++ b/Lib/sre_parse.py
@@ -0,0 +1,801 @@
+#
+# Secret Labs' Regular Expression Engine
+#
+# convert re-style regular expression to sre pattern
+#
+# Copyright (c) 1998-2001 by Secret Labs AB.  All rights reserved.
+#
+# See the sre.py file for information on usage and redistribution.
+#
+
+"""Internal support module for sre"""
+
+# XXX: show string offset and offending character for all errors
+
+import sys
+
+from sre_constants import *
+from _sre import MAXREPEAT
+
+SPECIAL_CHARS = ".\\[{()*+?^$|"
+REPEAT_CHARS = "*+?{"
+
+DIGITS = set("0123456789")
+
+OCTDIGITS = set("01234567")
+HEXDIGITS = set("0123456789abcdefABCDEF")
+
+WHITESPACE = set(" \t\n\r\v\f")
+
+ESCAPES = {
+    r"\a": (LITERAL, ord("\a")),
+    r"\b": (LITERAL, ord("\b")),
+    r"\f": (LITERAL, ord("\f")),
+    r"\n": (LITERAL, ord("\n")),
+    r"\r": (LITERAL, ord("\r")),
+    r"\t": (LITERAL, ord("\t")),
+    r"\v": (LITERAL, ord("\v")),
+    r"\\": (LITERAL, ord("\\"))
+}
+
+CATEGORIES = {
+    r"\A": (AT, AT_BEGINNING_STRING), # start of string
+    r"\b": (AT, AT_BOUNDARY),
+    r"\B": (AT, AT_NON_BOUNDARY),
+    r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]),
+    r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]),
+    r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]),
+    r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]),
+    r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]),
+    r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]),
+    r"\Z": (AT, AT_END_STRING), # end of string
+}
+
+FLAGS = {
+    # standard flags
+    "i": SRE_FLAG_IGNORECASE,
+    "L": SRE_FLAG_LOCALE,
+    "m": SRE_FLAG_MULTILINE,
+    "s": SRE_FLAG_DOTALL,
+    "x": SRE_FLAG_VERBOSE,
+    # extensions
+    "t": SRE_FLAG_TEMPLATE,
+    "u": SRE_FLAG_UNICODE,
+}
+
+class Pattern:
+    # master pattern object.  keeps track of global attributes
+    def __init__(self):
+        self.flags = 0
+        self.open = []
+        self.groups = 1
+        self.groupdict = {}
+    def opengroup(self, name=None):
+        gid = self.groups
+        self.groups = gid + 1
+        if name is not None:
+            ogid = self.groupdict.get(name, None)
+            if ogid is not None:
+                raise error, ("redefinition of group name %s as group %d; "
+                              "was group %d" % (repr(name), gid,  ogid))
+            self.groupdict[name] = gid
+        self.open.append(gid)
+        return gid
+    def closegroup(self, gid):
+        self.open.remove(gid)
+    def checkgroup(self, gid):
+        return gid < self.groups and gid not in self.open
+
+class SubPattern:
+    # a subpattern, in intermediate form
+    def __init__(self, pattern, data=None):
+        self.pattern = pattern
+        if data is None:
+            data = []
+        self.data = data
+        self.width = None
+    def dump(self, level=0):
+        nl = 1
+        seqtypes = type(()), type([])
+        for op, av in self.data:
+            print level*"  " + op,; nl = 0
+            if op == "in":
+                # member sublanguage
+                print; nl = 1
+                for op, a in av:
+                    print (level+1)*"  " + op, a
+            elif op == "branch":
+                print; nl = 1
+                i = 0
+                for a in av[1]:
+                    if i > 0:
+                        print level*"  " + "or"
+                    a.dump(level+1); nl = 1
+                    i = i + 1
+            elif type(av) in seqtypes:
+                for a in av:
+                    if isinstance(a, SubPattern):
+                        if not nl: print
+                        a.dump(level+1); nl = 1
+                    else:
+                        print a, ; nl = 0
+            else:
+                print av, ; nl = 0
+            if not nl: print
+    def __repr__(self):
+        return repr(self.data)
+    def __len__(self):
+        return len(self.data)
+    def __delitem__(self, index):
+        del self.data[index]
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            return SubPattern(self.pattern, self.data[index])
+        return self.data[index]
+    def __setitem__(self, index, code):
+        self.data[index] = code
+    def insert(self, index, code):
+        self.data.insert(index, code)
+    def append(self, code):
+        self.data.append(code)
+    def getwidth(self):
+        # determine the width (min, max) for this subpattern
+        if self.width:
+            return self.width
+        lo = hi = 0L
+        UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY)
+        REPEATCODES = (MIN_REPEAT, MAX_REPEAT)
+        for op, av in self.data:
+            if op == BRANCH:
+                i = sys.maxint
+                j = 0
+                for av in av[1]:
+                    l, h = av.getwidth()
+                    i = min(i, l)
+                    j = max(j, h)
+                lo = lo + i
+                hi = hi + j
+            elif op == CALL:
+                i, j = av.getwidth()
+                lo = lo + i
+                hi = hi + j
+            elif op == SUBPATTERN:
+                i, j = av[1].getwidth()
+                lo = lo + i
+                hi = hi + j
+            elif op in REPEATCODES:
+                i, j = av[2].getwidth()
+                lo = lo + long(i) * av[0]
+                hi = hi + long(j) * av[1]
+            elif op in UNITCODES:
+                lo = lo + 1
+                hi = hi + 1
+            elif op == SUCCESS:
+                break
+        self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint))
+        return self.width
+
+class Tokenizer:
+    def __init__(self, string):
+        self.string = string
+        self.index = 0
+        self.__next()
+    def __next(self):
+        if self.index >= len(self.string):
+            self.next = None
+            return
+        char = self.string[self.index]
+        if char[0] == "\\":
+            try:
+                c = self.string[self.index + 1]
+            except IndexError:
+                raise error, "bogus escape (end of line)"
+            char = char + c
+        self.index = self.index + len(char)
+        self.next = char
+    def match(self, char, skip=1):
+        if char == self.next:
+            if skip:
+                self.__next()
+            return 1
+        return 0
+    def get(self):
+        this = self.next
+        self.__next()
+        return this
+    def tell(self):
+        return self.index, self.next
+    def seek(self, index):
+        self.index, self.next = index
+
+def isident(char):
+    return "a" <= char <= "z" or "A" <= char <= "Z" or char == "_"
+
+def isdigit(char):
+    return "0" <= char <= "9"
+
+def isname(name):
+    # check that group name is a valid string
+    if not isident(name[0]):
+        return False
+    for char in name[1:]:
+        if not isident(char) and not isdigit(char):
+            return False
+    return True
+
+def _class_escape(source, escape):
+    # handle escape code inside character class
+    code = ESCAPES.get(escape)
+    if code:
+        return code
+    code = CATEGORIES.get(escape)
+    if code and code[0] == IN:
+        return code
+    try:
+        c = escape[1:2]
+        if c == "x":
+            # hexadecimal escape (exactly two digits)
+            while source.next in HEXDIGITS and len(escape) < 4:
+                escape = escape + source.get()
+            escape = escape[2:]
+            if len(escape) != 2:
+                raise error, "bogus escape: %s" % repr("\\" + escape)
+            return LITERAL, int(escape, 16) & 0xff
+        elif c in OCTDIGITS:
+            # octal escape (up to three digits)
+            while source.next in OCTDIGITS and len(escape) < 4:
+                escape = escape + source.get()
+            escape = escape[1:]
+            return LITERAL, int(escape, 8) & 0xff
+        elif c in DIGITS:
+            raise error, "bogus escape: %s" % repr(escape)
+        if len(escape) == 2:
+            return LITERAL, ord(escape[1])
+    except ValueError:
+        pass
+    raise error, "bogus escape: %s" % repr(escape)
+
+def _escape(source, escape, state):
+    # handle escape code in expression
+    code = CATEGORIES.get(escape)
+    if code:
+        return code
+    code = ESCAPES.get(escape)
+    if code:
+        return code
+    try:
+        c = escape[1:2]
+        if c == "x":
+            # hexadecimal escape
+            while source.next in HEXDIGITS and len(escape) < 4:
+                escape = escape + source.get()
+            if len(escape) != 4:
+                raise ValueError
+            return LITERAL, int(escape[2:], 16) & 0xff
+        elif c == "0":
+            # octal escape
+            while source.next in OCTDIGITS and len(escape) < 4:
+                escape = escape + source.get()
+            return LITERAL, int(escape[1:], 8) & 0xff
+        elif c in DIGITS:
+            # octal escape *or* decimal group reference (sigh)
+            if source.next in DIGITS:
+                escape = escape + source.get()
+                if (escape[1] in OCTDIGITS and escape[2] in OCTDIGITS and
+                    source.next in OCTDIGITS):
+                    # got three octal digits; this is an octal escape
+                    escape = escape + source.get()
+                    return LITERAL, int(escape[1:], 8) & 0xff
+            # not an octal escape, so this is a group reference
+            group = int(escape[1:])
+            if group < state.groups:
+                if not state.checkgroup(group):
+                    raise error, "cannot refer to open group"
+                return GROUPREF, group
+            raise ValueError
+        if len(escape) == 2:
+            return LITERAL, ord(escape[1])
+    except ValueError:
+        pass
+    raise error, "bogus escape: %s" % repr(escape)
+
+def _parse_sub(source, state, nested=1):
+    # parse an alternation: a|b|c
+
+    items = []
+    itemsappend = items.append
+    sourcematch = source.match
+    while 1:
+        itemsappend(_parse(source, state))
+        if sourcematch("|"):
+            continue
+        if not nested:
+            break
+        if not source.next or sourcematch(")", 0):
+            break
+        else:
+            raise error, "pattern not properly closed"
+
+    if len(items) == 1:
+        return items[0]
+
+    subpattern = SubPattern(state)
+    subpatternappend = subpattern.append
+
+    # check if all items share a common prefix
+    while 1:
+        prefix = None
+        for item in items:
+            if not item:
+                break
+            if prefix is None:
+                prefix = item[0]
+            elif item[0] != prefix:
+                break
+        else:
+            # all subitems start with a common "prefix".
+            # move it out of the branch
+            for item in items:
+                del item[0]
+            subpatternappend(prefix)
+            continue # check next one
+        break
+
+    # check if the branch can be replaced by a character set
+    for item in items:
+        if len(item) != 1 or item[0][0] != LITERAL:
+            break
+    else:
+        # we can store this as a character set instead of a
+        # branch (the compiler may optimize this even more)
+        set = []
+        setappend = set.append
+        for item in items:
+            setappend(item[0])
+        subpatternappend((IN, set))
+        return subpattern
+
+    subpattern.append((BRANCH, (None, items)))
+    return subpattern
+
+def _parse_sub_cond(source, state, condgroup):
+    item_yes = _parse(source, state)
+    if source.match("|"):
+        item_no = _parse(source, state)
+        if source.match("|"):
+            raise error, "conditional backref with more than two branches"
+    else:
+        item_no = None
+    if source.next and not source.match(")", 0):
+        raise error, "pattern not properly closed"
+    subpattern = SubPattern(state)
+    subpattern.append((GROUPREF_EXISTS, (condgroup, item_yes, item_no)))
+    return subpattern
+
+_PATTERNENDERS = set("|)")
+_ASSERTCHARS = set("=!<")
+_LOOKBEHINDASSERTCHARS = set("=!")
+_REPEATCODES = set([MIN_REPEAT, MAX_REPEAT])
+
+def _parse(source, state):
+    # parse a simple pattern
+    subpattern = SubPattern(state)
+
+    # precompute constants into local variables
+    subpatternappend = subpattern.append
+    sourceget = source.get
+    sourcematch = source.match
+    _len = len
+    PATTERNENDERS = _PATTERNENDERS
+    ASSERTCHARS = _ASSERTCHARS
+    LOOKBEHINDASSERTCHARS = _LOOKBEHINDASSERTCHARS
+    REPEATCODES = _REPEATCODES
+
+    while 1:
+
+        if source.next in PATTERNENDERS:
+            break # end of subpattern
+        this = sourceget()
+        if this is None:
+            break # end of pattern
+
+        if state.flags & SRE_FLAG_VERBOSE:
+            # skip whitespace and comments
+            if this in WHITESPACE:
+                continue
+            if this == "#":
+                while 1:
+                    this = sourceget()
+                    if this in (None, "\n"):
+                        break
+                continue
+
+        if this and this[0] not in SPECIAL_CHARS:
+            subpatternappend((LITERAL, ord(this)))
+
+        elif this == "[":
+            # character set
+            set = []
+            setappend = set.append
+##          if sourcematch(":"):
+##              pass # handle character classes
+            if sourcematch("^"):
+                setappend((NEGATE, None))
+            # check remaining characters
+            start = set[:]
+            while 1:
+                this = sourceget()
+                if this == "]" and set != start:
+                    break
+                elif this and this[0] == "\\":
+                    code1 = _class_escape(source, this)
+                elif this:
+                    code1 = LITERAL, ord(this)
+                else:
+                    raise error, "unexpected end of regular expression"
+                if sourcematch("-"):
+                    # potential range
+                    this = sourceget()
+                    if this == "]":
+                        if code1[0] == IN:
+                            code1 = code1[1][0]
+                        setappend(code1)
+                        setappend((LITERAL, ord("-")))
+                        break
+                    elif this:
+                        if this[0] == "\\":
+                            code2 = _class_escape(source, this)
+                        else:
+                            code2 = LITERAL, ord(this)
+                        if code1[0] != LITERAL or code2[0] != LITERAL:
+                            raise error, "bad character range"
+                        lo = code1[1]
+                        hi = code2[1]
+                        if hi < lo:
+                            raise error, "bad character range"
+                        setappend((RANGE, (lo, hi)))
+                    else:
+                        raise error, "unexpected end of regular expression"
+                else:
+                    if code1[0] == IN:
+                        code1 = code1[1][0]
+                    setappend(code1)
+
+            # XXX: <fl> should move set optimization to compiler!
+            if _len(set)==1 and set[0][0] == LITERAL:
+                subpatternappend(set[0]) # optimization
+            elif _len(set)==2 and set[0][0] == NEGATE and set[1][0] == LITERAL:
+                subpatternappend((NOT_LITERAL, set[1][1])) # optimization
+            else:
+                # XXX: <fl> should add charmap optimization here
+                subpatternappend((IN, set))
+
+        elif this and this[0] in REPEAT_CHARS:
+            # repeat previous item
+            if this == "?":
+                min, max = 0, 1
+            elif this == "*":
+                min, max = 0, MAXREPEAT
+
+            elif this == "+":
+                min, max = 1, MAXREPEAT
+            elif this == "{":
+                if source.next == "}":
+                    subpatternappend((LITERAL, ord(this)))
+                    continue
+                here = source.tell()
+                min, max = 0, MAXREPEAT
+                lo = hi = ""
+                while source.next in DIGITS:
+                    lo = lo + source.get()
+                if sourcematch(","):
+                    while source.next in DIGITS:
+                        hi = hi + sourceget()
+                else:
+                    hi = lo
+                if not sourcematch("}"):
+                    subpatternappend((LITERAL, ord(this)))
+                    source.seek(here)
+                    continue
+                if lo:
+                    min = int(lo)
+                    if min >= MAXREPEAT:
+                        raise OverflowError("the repetition number is too large")
+                if hi:
+                    max = int(hi)
+                    if max >= MAXREPEAT:
+                        raise OverflowError("the repetition number is too large")
+                    if max < min:
+                        raise error("bad repeat interval")
+            else:
+                raise error, "not supported"
+            # figure out which item to repeat
+            if subpattern:
+                item = subpattern[-1:]
+            else:
+                item = None
+            if not item or (_len(item) == 1 and item[0][0] == AT):
+                raise error, "nothing to repeat"
+            if item[0][0] in REPEATCODES:
+                raise error, "multiple repeat"
+            if sourcematch("?"):
+                subpattern[-1] = (MIN_REPEAT, (min, max, item))
+            else:
+                subpattern[-1] = (MAX_REPEAT, (min, max, item))
+
+        elif this == ".":
+            subpatternappend((ANY, None))
+
+        elif this == "(":
+            group = 1
+            name = None
+            condgroup = None
+            if sourcematch("?"):
+                group = 0
+                # options
+                if sourcematch("P"):
+                    # python extensions
+                    if sourcematch("<"):
+                        # named group: skip forward to end of name
+                        name = ""
+                        while 1:
+                            char = sourceget()
+                            if char is None:
+                                raise error, "unterminated name"
+                            if char == ">":
+                                break
+                            name = name + char
+                        group = 1
+                        if not name:
+                            raise error("missing group name")
+                        if not isname(name):
+                            raise error, "bad character in group name"
+                    elif sourcematch("="):
+                        # named backreference
+                        name = ""
+                        while 1:
+                            char = sourceget()
+                            if char is None:
+                                raise error, "unterminated name"
+                            if char == ")":
+                                break
+                            name = name + char
+                        if not name:
+                            raise error("missing group name")
+                        if not isname(name):
+                            raise error, "bad character in group name"
+                        gid = state.groupdict.get(name)
+                        if gid is None:
+                            raise error, "unknown group name"
+                        subpatternappend((GROUPREF, gid))
+                        continue
+                    else:
+                        char = sourceget()
+                        if char is None:
+                            raise error, "unexpected end of pattern"
+                        raise error, "unknown specifier: ?P%s" % char
+                elif sourcematch(":"):
+                    # non-capturing group
+                    group = 2
+                elif sourcematch("#"):
+                    # comment
+                    while 1:
+                        if source.next is None or source.next == ")":
+                            break
+                        sourceget()
+                    if not sourcematch(")"):
+                        raise error, "unbalanced parenthesis"
+                    continue
+                elif source.next in ASSERTCHARS:
+                    # lookahead assertions
+                    char = sourceget()
+                    dir = 1
+                    if char == "<":
+                        if source.next not in LOOKBEHINDASSERTCHARS:
+                            raise error, "syntax error"
+                        dir = -1 # lookbehind
+                        char = sourceget()
+                    p = _parse_sub(source, state)
+                    if not sourcematch(")"):
+                        raise error, "unbalanced parenthesis"
+                    if char == "=":
+                        subpatternappend((ASSERT, (dir, p)))
+                    else:
+                        subpatternappend((ASSERT_NOT, (dir, p)))
+                    continue
+                elif sourcematch("("):
+                    # conditional backreference group
+                    condname = ""
+                    while 1:
+                        char = sourceget()
+                        if char is None:
+                            raise error, "unterminated name"
+                        if char == ")":
+                            break
+                        condname = condname + char
+                    group = 2
+                    if not condname:
+                        raise error("missing group name")
+                    if isname(condname):
+                        condgroup = state.groupdict.get(condname)
+                        if condgroup is None:
+                            raise error, "unknown group name"
+                    else:
+                        try:
+                            condgroup = int(condname)
+                        except ValueError:
+                            raise error, "bad character in group name"
+                else:
+                    # flags
+                    if not source.next in FLAGS:
+                        raise error, "unexpected end of pattern"
+                    while source.next in FLAGS:
+                        state.flags = state.flags | FLAGS[sourceget()]
+            if group:
+                # parse group contents
+                if group == 2:
+                    # anonymous group
+                    group = None
+                else:
+                    group = state.opengroup(name)
+                if condgroup:
+                    p = _parse_sub_cond(source, state, condgroup)
+                else:
+                    p = _parse_sub(source, state)
+                if not sourcematch(")"):
+                    raise error, "unbalanced parenthesis"
+                if group is not None:
+                    state.closegroup(group)
+                subpatternappend((SUBPATTERN, (group, p)))
+            else:
+                while 1:
+                    char = sourceget()
+                    if char is None:
+                        raise error, "unexpected end of pattern"
+                    if char == ")":
+                        break
+                    raise error, "unknown extension"
+
+        elif this == "^":
+            subpatternappend((AT, AT_BEGINNING))
+
+        elif this == "$":
+            subpattern.append((AT, AT_END))
+
+        elif this and this[0] == "\\":
+            code = _escape(source, this, state)
+            subpatternappend(code)
+
+        else:
+            raise error, "parser error"
+
+    return subpattern
+
+def parse(str, flags=0, pattern=None):
+    # parse 're' pattern into list of (opcode, argument) tuples
+
+    source = Tokenizer(str)
+
+    if pattern is None:
+        pattern = Pattern()
+    pattern.flags = flags
+    pattern.str = str
+
+    p = _parse_sub(source, pattern, 0)
+
+    tail = source.get()
+    if tail == ")":
+        raise error, "unbalanced parenthesis"
+    elif tail:
+        raise error, "bogus characters at end of regular expression"
+
+    if flags & SRE_FLAG_DEBUG:
+        p.dump()
+
+    if not (flags & SRE_FLAG_VERBOSE) and p.pattern.flags & SRE_FLAG_VERBOSE:
+        # the VERBOSE flag was switched on inside the pattern.  to be
+        # on the safe side, we'll parse the whole thing again...
+        return parse(str, p.pattern.flags)
+
+    return p
+
+def parse_template(source, pattern):
+    # parse 're' replacement string into list of literals and
+    # group references
+    s = Tokenizer(source)
+    sget = s.get
+    p = []
+    a = p.append
+    def literal(literal, p=p, pappend=a):
+        if p and p[-1][0] == LITERAL:
+            p[-1] = LITERAL, p[-1][1] + literal
+        else:
+            pappend((LITERAL, literal))
+    sep = source[:0]
+    if type(sep) == type(""):
+        makechar = chr
+    else:
+        makechar = unichr
+    while 1:
+        this = sget()
+        if this is None:
+            break # end of replacement string
+        if this and this[0] == "\\":
+            # group
+            c = this[1:2]
+            if c == "g":
+                name = ""
+                if s.match("<"):
+                    while 1:
+                        char = sget()
+                        if char is None:
+                            raise error, "unterminated group name"
+                        if char == ">":
+                            break
+                        name = name + char
+                if not name:
+                    raise error, "missing group name"
+                try:
+                    index = int(name)
+                    if index < 0:
+                        raise error, "negative group number"
+                except ValueError:
+                    if not isname(name):
+                        raise error, "bad character in group name"
+                    try:
+                        index = pattern.groupindex[name]
+                    except KeyError:
+                        raise IndexError, "unknown group name"
+                a((MARK, index))
+            elif c == "0":
+                if s.next in OCTDIGITS:
+                    this = this + sget()
+                    if s.next in OCTDIGITS:
+                        this = this + sget()
+                literal(makechar(int(this[1:], 8) & 0xff))
+            elif c in DIGITS:
+                isoctal = False
+                if s.next in DIGITS:
+                    this = this + sget()
+                    if (c in OCTDIGITS and this[2] in OCTDIGITS and
+                        s.next in OCTDIGITS):
+                        this = this + sget()
+                        isoctal = True
+                        literal(makechar(int(this[1:], 8) & 0xff))
+                if not isoctal:
+                    a((MARK, int(this[1:])))
+            else:
+                try:
+                    this = makechar(ESCAPES[this][1])
+                except KeyError:
+                    pass
+                literal(this)
+        else:
+            literal(this)
+    # convert template to groups and literals lists
+    i = 0
+    groups = []
+    groupsappend = groups.append
+    literals = [None] * len(p)
+    for c, s in p:
+        if c == MARK:
+            groupsappend((i, s))
+            # literal[i] is already None
+        else:
+            literals[i] = s
+        i = i + 1
+    return groups, literals
+
+def expand_template(template, match):
+    g = match.group
+    sep = match.string[:0]
+    groups, literals = template
+    literals = literals[:]
+    try:
+        for index, group in groups:
+            literals[index] = s = g(group)
+            if s is None:
+                raise error, "unmatched group"
+    except IndexError:
+        raise error, "invalid group reference"
+    return sep.join(literals)

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Mon Nov 23 16:32:55 2015
From: jython-checkins at python.org (jim.baker)
Date: Mon, 23 Nov 2015 21:32:55 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Define_socket=2ESOL=5FTCP?=
 =?utf-8?q?=2E_Fixes_=232436=2E?=
Message-ID: <20151123213255.21501.39080@psf.io>

https://hg.python.org/jython/rev/b8fec161bcbb
changeset:   7822:b8fec161bcbb
user:        Jim Baker <jim.baker at rackspace.com>
date:        Mon Nov 23 14:32:48 2015 -0700
summary:
  Define socket.SOL_TCP. Fixes #2436.

files:
  Lib/_socket.py                                        |    2 +-
  Lib/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl |  Bin 
  Lib/socket.py                                         |    1 +
  Lib/test/test_socket_jy.py                            |   10 +++++++++-
  4 files changed, 11 insertions(+), 2 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -135,7 +135,7 @@
 IPPROTO_PUP      =  12 # not supported
 IPPROTO_RAW      = 255 # not supported
 IPPROTO_ROUTING  =  43 # not supported
-IPPROTO_TCP      =   6
+SOL_TCP = IPPROTO_TCP = 6
 IPPROTO_UDP      =  17
 
 SO_ACCEPTCONN  = 1
diff --git a/Lib/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl
index 5e490155f0ca7f4ddb64c93c39fb2efb8795cd08..610a36c5f1b1a4357784474766d3e785b45d9ed9
GIT binary patch
[stripped]
diff --git a/Lib/socket.py b/Lib/socket.py
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -55,6 +55,7 @@
     SOCK_SEQPACKET,
     
     SOL_SOCKET,
+    SOL_TCP,
     # not supported, but here for apparent completeness
     IPPROTO_AH,
     IPPROTO_DSTOPTS,
diff --git a/Lib/test/test_socket_jy.py b/Lib/test/test_socket_jy.py
--- a/Lib/test/test_socket_jy.py
+++ b/Lib/test/test_socket_jy.py
@@ -74,8 +74,16 @@
                 self.assertEqual(code, errno.EALREADY)
 
 
+class SocketOptionsTest(unittest.TestCase):
+
+    def test_socket_options_defined(self):
+        # Basic existence test to verify trivial fix for
+        # http://bugs.jython.org/issue2436
+        self.assertEqual(socket.SOL_TCP, socket.IPPROTO_TCP)
+
+
 def test_main():
-    test_support.run_unittest(SocketConnectTest)
+    test_support.run_unittest(SocketConnectTest, SocketOptionsTest)
 
 
 if __name__ == "__main__":

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Mon Nov 23 16:36:55 2015
From: jython-checkins at python.org (jim.baker)
Date: Mon, 23 Nov 2015 21:36:55 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Rollback_inadvertent_ensure?=
 =?utf-8?q?pip_bundled_wheel_change_from_prev_change_set?=
Message-ID: <20151123213655.21503.23584@psf.io>

https://hg.python.org/jython/rev/2a848d435269
changeset:   7823:2a848d435269
user:        Jim Baker <jim.baker at rackspace.com>
date:        Mon Nov 23 14:36:50 2015 -0700
summary:
  Rollback inadvertent ensurepip bundled wheel change from prev change set

files:
  Lib/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl |  Bin 
  1 files changed, 0 insertions(+), 0 deletions(-)


diff --git a/Lib/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-7.1.2-py2.py3-none-any.whl
index 610a36c5f1b1a4357784474766d3e785b45d9ed9..5e490155f0ca7f4ddb64c93c39fb2efb8795cd08
GIT binary patch
[stripped]

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Mon Nov 23 17:42:07 2015
From: jython-checkins at python.org (jim.baker)
Date: Mon, 23 Nov 2015 22:42:07 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Remove_socket=2ESO=5FEXCLUS?=
 =?utf-8?q?IVEADDRUSE=2E_Fixes_=232435=2E?=
Message-ID: <20151123224205.81884.73673@psf.io>

https://hg.python.org/jython/rev/f528d540f4b3
changeset:   7824:f528d540f4b3
user:        Jim Baker <jim.baker at rackspace.com>
date:        Mon Nov 23 15:42:00 2015 -0700
summary:
  Remove socket.SO_EXCLUSIVEADDRUSE. Fixes #2435.

files:
  Lib/_socket.py          |  1 -
  Lib/socket.py           |  1 -
  Lib/test/test_socket.py |  9 ++++++++-
  3 files changed, 8 insertions(+), 3 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -156,7 +156,6 @@
 
 SO_DEBUG            = -1
 SO_DONTROUTE        = -1
-SO_EXCLUSIVEADDRUSE = -8
 SO_RCVLOWAT         = -16
 SO_RCVTIMEO         = -32
 SO_REUSEPORT        = -64
diff --git a/Lib/socket.py b/Lib/socket.py
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -99,7 +99,6 @@
     SO_OOBINLINE,
     SO_DEBUG,
     SO_DONTROUTE,
-    SO_EXCLUSIVEADDRUSE,
     SO_RCVLOWAT,
     SO_RCVTIMEO,
     SO_REUSEPORT,
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -1002,7 +1002,14 @@
         # this is an MS specific option that will not be appearing on java
         # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6421091
         # http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=6402335
-        self.failUnless(hasattr(socket, 'SO_EXCLUSIVEADDRUSE'))
+        # 
+        # As of 2.7.1, we no longer support this option - it is also
+        # not not necessary since Java implements BSD semantics for
+        # address reuse even on Windows. See
+        # http://bugs.jython.org/issue2435 and
+        # http://sourceforge.net/p/jython/mailman/message/34642295/
+        # for discussion
+        self.assertFalse(hasattr(socket, 'SO_EXCLUSIVEADDRUSE'))
 
     def testSO_RCVLOWAT(self):
         self.failUnless(hasattr(socket, 'SO_RCVLOWAT'))

-- 
Repository URL: https://hg.python.org/jython