From jython-checkins at python.org Wed Jul 2 00:43:42 2014 From: jython-checkins at python.org (jim.baker) Date: Wed, 2 Jul 2014 00:43:42 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Change_bz2=2EBZ2File=2Eread?= =?utf-8?q?_such_that_it_reads_the_number_of_requested_bytes=2C?= Message-ID: <3h30wZ0k2qz7Ljd@mail.python.org> http://hg.python.org/jython/rev/91b39451dc89 changeset: 7345:91b39451dc89 user: Jim Baker date: Tue Jul 01 16:43:36 2014 -0600 summary: Change bz2.BZ2File.read such that it reads the number of requested bytes, even if the underlying TextIOBase may return less. Fixes http://bugs.jython.org/issue2176 files: build.xml | 2 +- extlibs/commons-compress-1.8.1.jar | Bin extlibs/commons-compress-1.8.jar | Bin src/org/python/modules/bz2/PyBZ2File.java | 13 ++++++++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -148,7 +148,7 @@ - + diff --git a/extlibs/commons-compress-1.8.1.jar b/extlibs/commons-compress-1.8.1.jar new file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..66b0a56bd8b24478ddeb2d6eb2824280de1c1a9d GIT binary patch [stripped] diff --git a/extlibs/commons-compress-1.8.jar b/extlibs/commons-compress-1.8.jar deleted file mode 100644 index 940b0687323428c7ef6063a07a43eb77fb9cbea1..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] diff --git a/src/org/python/modules/bz2/PyBZ2File.java b/src/org/python/modules/bz2/PyBZ2File.java --- a/src/org/python/modules/bz2/PyBZ2File.java +++ b/src/org/python/modules/bz2/PyBZ2File.java @@ -180,9 +180,18 @@ new String[] { "size" }, 0); int size = ap.getInt(0, -1); - final String data = buffer.read(size); - return new PyString(data); + if (size == 0) { return Py.EmptyString; } + if (size < 0) { return new PyString(buffer.readall()); } + StringBuilder data = new StringBuilder(size); + while (data.length() < size) { + String chunk = buffer.read(size - data.length()); + if (chunk.length() == 0) { + break; + } + data.append(chunk); + } + return new PyString(data.toString()); } @ExposedMethod -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Wed Jul 2 22:50:24 2014 From: jython-checkins at python.org (jim.baker) Date: Wed, 2 Jul 2014 22:50:24 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Java_since_Java_6_supports_?= =?utf-8?q?JSR_223_scripting=2C_so_no_need_to_include?= Message-ID: <3h3ZMN5Ypbz7Ljc@mail.python.org> http://hg.python.org/jython/rev/a00987ea4da6 changeset: 7346:a00987ea4da6 user: Jim Baker date: Wed Jul 02 14:50:21 2014 -0600 summary: Java since Java 6 supports JSR 223 scripting, so no need to include supporting jar. files: build.xml | 1 - extlibs/livetribe-jsr223-2.0.6.jar | Bin 2 files changed, 0 insertions(+), 1 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -147,7 +147,6 @@ - diff --git a/extlibs/livetribe-jsr223-2.0.6.jar b/extlibs/livetribe-jsr223-2.0.6.jar deleted file mode 100644 index b35c0977f409090ccb831226a426403e11410e54..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 GIT binary patch [stripped] -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Thu Jul 10 01:49:39 2014 From: jython-checkins at python.org (jim.baker) Date: Thu, 10 Jul 2014 01:49:39 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_Apache_Commons_Compr?= =?utf-8?q?ession_to_1=2E8=2E1_for_jar-complete_builds=2E?= Message-ID: <3h7y0z2jM8z7Lnf@mail.python.org> http://hg.python.org/jython/rev/0c21916a620a changeset: 7347:0c21916a620a user: Jim Baker date: Wed Jul 09 17:46:41 2014 -0600 summary: Update Apache Commons Compression to 1.8.1 for jar-complete builds. Fixes http://bugs.jython.org/issue2178 files: build.xml | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -576,7 +576,7 @@ - + -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Mon Jul 14 22:19:48 2014 From: jython-checkins at python.org (jim.baker) Date: Mon, 14 Jul 2014 22:19:48 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_bugs_in_select=2Epoll?= =?utf-8?q?=2C_threadpool_group_closing=2C_and_SSL_handshaking=2E?= Message-ID: <3hBx6X2RTvz7LjY@mail.python.org> http://hg.python.org/jython/rev/2c45f75a5406 changeset: 7348:2c45f75a5406 user: Jim Baker date: Mon Jul 14 14:19:57 2014 -0600 summary: Fix bugs in select.poll, threadpool group closing, and SSL handshaking. Does not yet support Start TLS for the server side of a SSL connection, which has been deferred indefinitely. Merged from https://bitbucket.org/jimbaker/jython-fix-socket-bugs Fixes http://bugs.jython.org/issue2094, http://bugs.jython.org/issue2147, http://bugs.jython.org/issue2174 files: Lib/_socket.py | 280 ++++++++++++++++++--------- Lib/select.py | 2 +- Lib/ssl.py | 211 ++++++++++++++++---- Lib/test/test_socket.py | 28 +- 4 files changed, 369 insertions(+), 152 deletions(-) diff --git a/Lib/_socket.py b/Lib/_socket.py --- a/Lib/_socket.py +++ b/Lib/_socket.py @@ -3,7 +3,9 @@ import errno import jarray import logging +import numbers import pprint +import struct import sys import time import _google_ipaddr_r234 @@ -15,18 +17,19 @@ from StringIO import StringIO from threading import Condition, Lock from types import MethodType, NoneType +from weakref import WeakKeyDictionary import java from java.io import IOException, InterruptedIOException -from java.lang import Thread +from java.lang import Thread, IllegalStateException from java.net import InetAddress, InetSocketAddress from java.nio.channels import ClosedChannelException from java.util import NoSuchElementException from java.util.concurrent import ( ArrayBlockingQueue, CopyOnWriteArrayList, CountDownLatch, LinkedBlockingQueue, RejectedExecutionException, ThreadFactory, TimeUnit) -from java.util.concurrent.atomic import AtomicBoolean -from javax.net.ssl import SSLPeerUnverifiedException +from java.util.concurrent.atomic import AtomicBoolean, AtomicLong +from javax.net.ssl import SSLPeerUnverifiedException, SSLException try: # jarjar-ed version @@ -53,6 +56,8 @@ FORMAT = '%(asctime)-15s %(threadName)s %(levelname)s %(funcName)s %(message)s %(sock)s' logging.basicConfig(format=FORMAT, level=logging.DEBUG) +# _debug() # UNCOMMENT to get logging of socket activity + # Constants ########### @@ -185,21 +190,27 @@ # because these threads only handle ephemeral data, such as performing # SSL wrap/unwrap. + class DaemonThreadFactory(ThreadFactory): + + thread_count = AtomicLong() + + def __init__(self, label): + self.label = label + def newThread(self, runnable): t = Thread(runnable) t.daemon = True + t.name = self.label % (self.thread_count.getAndIncrement()) return t -# This number should be configurable by the user. 10 is the default -# number as of 4.0.17 of Netty. FIXME this default may be based on core count. +NIO_GROUP = NioEventLoopGroup(10, DaemonThreadFactory("Jython-Netty-Client-%s")) -NIO_GROUP = NioEventLoopGroup(10, DaemonThreadFactory()) -def _check_threadpool_for_pending_threads(): +def _check_threadpool_for_pending_threads(group): pending_threads = [] - for t in NIO_GROUP: + for t in group: pending_count = t.pendingTasks() if pending_count > 0: pending_threads.append((t, pending_count)) @@ -272,6 +283,7 @@ IOException : lambda x: error(errno.ECONNRESET, 'Software caused connection abort'), InterruptedIOException : lambda x: timeout(None, 'timed out'), + IllegalStateException : lambda x: error(errno.EPIPE, 'Illegal state exception'), java.net.BindException : lambda x: error(errno.EADDRINUSE, 'Address already in use'), java.net.ConnectException : lambda x: error(errno.ECONNREFUSED, 'Connection refused'), @@ -299,7 +311,8 @@ java.nio.channels.UnresolvedAddressException : lambda x: gaierror(errno.EGETADDRINFOFAILED, 'getaddrinfo failed'), java.nio.channels.UnsupportedAddressTypeException : None, - SSLPeerUnverifiedException: lambda x: SSLError(SSL_ERROR_SSL, "FIXME"), + SSLPeerUnverifiedException: lambda x: SSLError(SSL_ERROR_SSL, x.message), + SSLException: lambda x: SSLError(SSL_ERROR_SSL, x.message), } @@ -391,7 +404,9 @@ # shortcircuiting if the socket was in fact ready for # reading/writing/exception before the select call if selected_rlist or selected_wlist: - return sorted(selected_rlist), sorted(selected_wlist), sorted(selected_xlist) + completed = sorted(selected_rlist), sorted(selected_wlist), sorted(selected_xlist) + log.debug("Completed select %s", completed, extra={"sock": "*"}) + return completed elif timeout is not None and time.time() - started >= timeout: return [], [], [] self.cv.wait(timeout) @@ -400,7 +415,12 @@ # poll support ############## -_PollNotification = namedtuple("_PollNotification", ["sock", "fd", "exception", "hangup"]) +_PollNotification = namedtuple( + "_PollNotification", + ["sock", # the real socket + "fd", # could be the real socket (as returned by fileno) or a wrapping socket object + "exception", + "hangup"]) class poll(object): @@ -408,30 +428,41 @@ def __init__(self): self.queue = LinkedBlockingQueue() self.registered = dict() # fd -> eventmask + self.socks2fd = WeakKeyDictionary() # sock -> fd def notify(self, sock, exception=None, hangup=False): notification = _PollNotification( sock=sock, - fd=sock.fileno(), + fd=self.socks2fd.get(sock), exception=exception, hangup=hangup) + log.debug("Notify %s", notification, extra={"sock": "*"}) + self.queue.put(notification) def register(self, fd, eventmask=POLLIN|POLLPRI|POLLOUT): + if not hasattr(fd, "fileno"): + raise TypeError("argument must have a fileno() method") + sock = fd.fileno() + log.debug("Register fd=%s eventmask=%s", fd, eventmask, extra={"sock": sock}) self.registered[fd] = eventmask - # NOTE in case fd != sock in a future release, modifiy accordingly - sock = fd + self.socks2fd[sock] = fd sock._register_selector(self) self.notify(sock) # Ensure we get an initial notification def modify(self, fd, eventmask): + if not hasattr(fd, "fileno"): + raise TypeError("argument must have a fileno() method") if fd not in self.registered: raise error(errno.ENOENT, "No such file or directory") self.registered[fd] = eventmask def unregister(self, fd): + if not hasattr(fd, "fileno"): + raise TypeError("argument must have a fileno() method") + log.debug("Unregister socket fd=%s", fd, extra={"sock": fd.fileno()}) del self.registered[fd] - sock = fd + sock = fd.fileno() sock._unregister_selector(self) def _event_test(self, notification): @@ -439,7 +470,8 @@ # edges around errors and hangup if notification is None: return None, 0 - mask = self.registered.get(notification.sock, 0) # handle if concurrently removed, by simply ignoring + mask = self.registered.get(notification.fd, 0) # handle if concurrently removed, by simply ignoring + log.debug("Testing notification=%s mask=%s", notification, mask, extra={"sock": "*"}) event = 0 if mask & POLLIN and notification.sock._readable(): event |= POLLIN @@ -451,53 +483,58 @@ event |= POLLHUP if mask & POLLNVAL and not notification.sock.peer_closed: event |= POLLNVAL + log.debug("Tested notification=%s event=%s", notification, event, extra={"sock": "*"}) return notification.fd, event + def _handle_poll(self, poller): + notification = poller() + if notification is None: + return [] + + # Pull as many outstanding notifications as possible out + # of the queue + notifications = [notification] + self.queue.drainTo(notifications) + log.debug("Got notification(s) %s", notifications, extra={"sock": "MODULE"}) + result = [] + socks = set() + + # But given how we notify, it's possible to see possible + # multiple notifications. Just return one (fd, event) for a + # given socket + for notification in notifications: + if notification.sock not in socks: + fd, event = self._event_test(notification) + if event: + result.append((fd, event)) + socks.add(notification.sock) + + # Repump sockets to pick up a subsequent level change + for sock in socks: + self.notify(sock) + + return result + def poll(self, timeout=None): - if not timeout or timeout < 0: - # Simplify logic around timeout resets + if not (timeout is None or isinstance(timeout, numbers.Real)): + raise TypeError("timeout must be a number or None, got %r" % (timeout,)) + if timeout < 0: timeout = None + log.debug("Polling timeout=%s", timeout, extra={"sock": "*"}) + if timeout is None: + return self._handle_poll(self.queue.take) + elif timeout == 0: + return self._handle_poll(self.queue.poll) else: - timeout /= 1000. # convert from milliseconds to seconds - - while True: - if timeout is None: - notification = self.queue.take() - elif timeout > 0: + timeout = float(timeout) / 1000. # convert from milliseconds to seconds + while timeout > 0: started = time.time() timeout_in_ns = int(timeout * _TO_NANOSECONDS) - notification = self.queue.poll(timeout_in_ns, TimeUnit.NANOSECONDS) - # Need to reset the timeout, because this notification - # may not be of interest when masked out + result = self._handle_poll(partial(self.queue.poll, timeout_in_ns, TimeUnit.NANOSECONDS)) + if result: + return result timeout = timeout - (time.time() - started) - else: - return [] - - if notification is None: - continue - - # Pull as many outstanding notifications as possible out - # of the queue - notifications = [notification] - self.queue.drainTo(notifications) - log.debug("Got notification(s) %s", notifications, extra={"sock": "MODULE"}) - result = [] - socks = set() - - # But given how we notify, it's possible to see possible - # multiple notifications. Just return one (fd, event) for a - # given socket - for notification in notifications: - if notification.sock not in socks: - fd, event = self._event_test(notification) - if event: - result.append((fd, event)) - socks.add(notification.sock) - # Repump sockets to pick up a subsequent level change - for sock in socks: - self.notify(sock) - if result: - return result + return [] # integration with Netty @@ -538,7 +575,7 @@ self.parent_socket = parent_socket def initChannel(self, child_channel): - child = ChildSocket() + child = ChildSocket(self.parent_socket) child.proto = IPPROTO_TCP child._init_client_mode(child_channel) @@ -551,7 +588,7 @@ log.debug("Setting inherited options %s", child.options, extra={"sock": child}) config = child_channel.config() for option, value in child.options.iteritems(): - config.setOption(option, value) + _set_option(config.setOption, option, value) log.debug("Notifing listeners of parent socket %s", self.parent_socket, extra={"sock": child}) self.parent_socket.child_queue.put(child) @@ -578,6 +615,35 @@ # FIXME raise exceptions for ops not permitted on client socket, server socket UNKNOWN_SOCKET, CLIENT_SOCKET, SERVER_SOCKET, DATAGRAM_SOCKET = range(4) +_socket_types = { + UNKNOWN_SOCKET: "unknown", + CLIENT_SOCKET: "client", + SERVER_SOCKET: "server", + DATAGRAM_SOCKET: "datagram" +} + + + + +def _identity(value): + return value + + +def _set_option(setter, option, value): + if option in (ChannelOption.SO_LINGER, ChannelOption.SO_TIMEOUT): + # FIXME consider implementing these options. Note these are not settable + # via config.setOption in any event: + # + # * SO_TIMEOUT does not work for NIO sockets, need to use + # IdleStateHandler instead + # + # * SO_LINGER does not work for nonblocking sockets, so need + # to emulate in calling close on the socket by attempting to + # send any unsent data (it's not clear this actually is + # needed in Netty however...) + return + else: + setter(option, value) # These are the only socket protocols we currently support, so it's easy to map as follows: @@ -585,19 +651,15 @@ _socket_options = { IPPROTO_TCP: { (SOL_SOCKET, SO_KEEPALIVE): (ChannelOption.SO_KEEPALIVE, bool), - (SOL_SOCKET, SO_LINGER): (ChannelOption.SO_LINGER, int), + (SOL_SOCKET, SO_LINGER): (ChannelOption.SO_LINGER, _identity), (SOL_SOCKET, SO_RCVBUF): (ChannelOption.SO_RCVBUF, int), (SOL_SOCKET, SO_REUSEADDR): (ChannelOption.SO_REUSEADDR, bool), (SOL_SOCKET, SO_SNDBUF): (ChannelOption.SO_SNDBUF, int), - # FIXME SO_TIMEOUT needs to be handled by an IdleStateHandler - - # ChannelOption.SO_TIMEOUT really only applies to OIO (old) socket channels, - # we want to use NIO ones (SOL_SOCKET, SO_TIMEOUT): (ChannelOption.SO_TIMEOUT, int), (IPPROTO_TCP, TCP_NODELAY): (ChannelOption.TCP_NODELAY, bool), }, IPPROTO_UDP: { (SOL_SOCKET, SO_BROADCAST): (ChannelOption.SO_BROADCAST, bool), - (SOL_SOCKET, SO_LINGER): (ChannelOption.SO_LINGER, int), (SOL_SOCKET, SO_RCVBUF): (ChannelOption.SO_RCVBUF, int), (SOL_SOCKET, SO_REUSEADDR): (ChannelOption.SO_REUSEADDR, bool), (SOL_SOCKET, SO_SNDBUF): (ChannelOption.SO_SNDBUF, int), @@ -629,6 +691,7 @@ proto = IPPROTO_UDP self.proto = proto + self._sock = self # some Python code wants to see a socket self._last_error = 0 # supports SO_ERROR self.connected = False self.timeout = _defaulttimeout @@ -654,7 +717,7 @@ def __repr__(self): return "<_realsocket at {:#x} type={} open_count={} channel={} timeout={}>".format( - id(self), self.socket_type, self.open_count, self.channel, self.timeout) + id(self), _socket_types[self.socket_type], self.open_count, self.channel, self.timeout) def _unlatch(self): pass # no-op once mutated from ChildSocket to normal _socketobject @@ -689,6 +752,7 @@ elif self.timeout: self._handle_timeout(future.await, reason) if not future.isSuccess(): + log.exception("Got this failure %s during %s", future.cause(), reason, extra={"sock": self}) raise future.cause() return future else: @@ -757,7 +821,7 @@ self.python_inbound_handler = PythonInboundHandler(self) bootstrap = Bootstrap().group(NIO_GROUP).channel(NioSocketChannel) for option, value in self.options.iteritems(): - bootstrap.option(option, value) + _set_option(bootstrap.option, option, value) # FIXME really this is just for SSL handling, so make more # specific than a list of connect_handlers @@ -772,13 +836,12 @@ bind_future = bootstrap.bind(self.bind_addr) self._handle_channel_future(bind_future, "local bind") self.channel = bind_future.channel() - future = self.channel.connect(addr) else: log.debug("Connect to %s", addr, extra={"sock": self}) - future = bootstrap.connect(addr) - self.channel = future.channel() - - self._handle_channel_future(future, "connect") + self.channel = bootstrap.channel() + + connect_future = self.channel.connect(addr) + self._handle_channel_future(connect_future, "connect") self.bind_timestamp = time.time() def _post_connect(self): @@ -818,19 +881,21 @@ def listen(self, backlog): self.socket_type = SERVER_SOCKET + self.child_queue = ArrayBlockingQueue(backlog) + self.accepted_children = 1 # include the parent as well to simplify close logic b = ServerBootstrap() - self.group = NioEventLoopGroup(10, DaemonThreadFactory()) - b.group(self.group) + self.parent_group = NioEventLoopGroup(2, DaemonThreadFactory("Jython-Netty-Parent-%s")) + self.child_group = NioEventLoopGroup(2, DaemonThreadFactory("Jython-Netty-Child-%s")) + b.group(self.parent_group, self.child_group) b.channel(NioServerSocketChannel) b.option(ChannelOption.SO_BACKLOG, backlog) for option, value in self.options.iteritems(): - b.option(option, value) + _set_option(b.option, option, value) # Note that child options are set in the child handler so # that they can take into account any subsequent changes, # plus have shadow support - self.child_queue = ArrayBlockingQueue(backlog) self.child_handler = ChildSocketHandler(self) b.childHandler(self.child_handler) @@ -854,6 +919,9 @@ raise error(errno.EWOULDBLOCK, "Resource temporarily unavailable") peername = child.getpeername() if child else None log.debug("Got child %s connected to %s", child, peername, extra={"sock": self}) + child.accepted = True + with self.open_lock: + self.accepted_children += 1 return child, peername # DATAGRAM METHODS @@ -870,7 +938,7 @@ bootstrap = Bootstrap().group(NIO_GROUP).channel(NioDatagramChannel) bootstrap.handler(self.python_inbound_handler) for option, value in self.options.iteritems(): - bootstrap.option(option, value) + _set_option(bootstrap.option, option, value) future = bootstrap.register() self._handle_channel_future(future, "register") @@ -903,14 +971,19 @@ self._handle_channel_future(future, "sendto") return len(string) - - # FIXME implement these methods - def recvfrom_into(self, buffer, nbytes=0, flags=0): - raise NotImplementedError() + if nbytes == 0: + nbytes = len(buffer) + data, remote_addr = self.recvfrom(nbytes, flags) + buffer[0:len(data)] = data + return len(data), remote_addr def recv_into(self, buffer, nbytes=0, flags=0): - raise NotImplementedError() + if nbytes == 0: + nbytes = len(buffer) + data = self.recv(nbytes, flags) + buffer[0:len(data)] = data + return len(data) # GENERAL METHODS @@ -930,7 +1003,9 @@ # Do not care about tasks that attempt to schedule after close pass if self.socket_type == SERVER_SOCKET: - self.group.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS) + log.debug("Shutting down server socket parent group", extra={"sock": self}) + self.parent_group.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS) + self.accepted_children -= 1 while True: child = self.child_queue.poll() if child is None: @@ -941,6 +1016,7 @@ log.debug("Closed socket", extra={"sock": self}) def shutdown(self, how): + log.debug("Got request to shutdown socket how=%s", how, extra={"sock": self}) self._verify_channel() if how & SHUT_RD: try: @@ -952,8 +1028,10 @@ def _readable(self): if self.socket_type == CLIENT_SOCKET or self.socket_type == DATAGRAM_SOCKET: - return ((self.incoming_head is not None and self.incoming_head.readableBytes()) or - self.incoming.peek()) + log.debug("Incoming head=%s queue=%s", self.incoming_head, self.incoming, extra={"sock": self}) + return ( + (self.incoming_head is not None and self.incoming_head.readableBytes()) or + self.incoming.peek()) elif self.socket_type == SERVER_SOCKET: return bool(self.child_queue.peek()) else: @@ -986,6 +1064,7 @@ raise error(errno.ENOTCONN, 'Socket not connected') future = self.channel.writeAndFlush(Unpooled.wrappedBuffer(data)) self._handle_channel_future(future, "send") + log.debug("Sent data <<<{!r:.20}>>>".format(data), extra={"sock": self}) # FIXME are we sure we are going to be able to send this much data, especially async? return len(data) @@ -1008,9 +1087,9 @@ log.debug("No data yet for socket", extra={"sock": self}) raise error(errno.EAGAIN, "Resource temporarily unavailable") - # Only return _PEER_CLOSED once msg = self.incoming_head if msg is _PEER_CLOSED: + # Only return _PEER_CLOSED once self.incoming_head = None self.peer_closed = True return msg @@ -1073,13 +1152,11 @@ except KeyError: raise error(errno.ENOPROTOOPT, "Protocol not available") - # FIXME for NIO sockets, SO_TIMEOUT doesn't work - should use - # IdleStateHandler instead cast_value = cast(value) self.options[option] = cast_value log.debug("Setting option %s to %s", optname, value, extra={"sock": self}) if self.channel: - self.channel.config().setOption(option, cast(value)) + _set_option(self.channel.config().setOption, option, cast_value) def getsockopt(self, level, optname, buflen=None): # Pseudo options for interrogating the status of this socket @@ -1247,10 +1324,12 @@ class ChildSocket(_realsocket): - def __init__(self): + def __init__(self, parent_socket): super(ChildSocket, self).__init__() + self.parent_socket = parent_socket self.active = AtomicBoolean() self.active_latch = CountDownLatch(1) + self.accepted = False def _ensure_post_connect(self): do_post_connect = not self.active.getAndSet(True) @@ -1293,6 +1372,14 @@ def close(self): self._ensure_post_connect() super(ChildSocket, self).close() + if self.open_count > 0: + return + if self.accepted: + with self.parent_socket.open_lock: + self.parent_socket.accepted_children -= 1 + if self.parent_socket.accepted_children == 0: + log.debug("Shutting down child group for parent socket=%s", self.parent_socket, extra={"sock": self}) + self.parent_socket.child_group.shutdownGracefully(0, 100, TimeUnit.MILLISECONDS) def shutdown(self, how): self._ensure_post_connect() @@ -1413,6 +1500,12 @@ def _get_jsockaddr(address_object, family, sock_type, proto, flags): + if family is None: + family = AF_UNSPEC + if sock_type is None: + sock_type = 0 + if proto is None: + proto = 0 addr = _get_jsockaddr2(address_object, family, sock_type, proto, flags) log.debug("Address %s for %s", addr, address_object, extra={"sock": "*"}) return addr @@ -1427,9 +1520,10 @@ address_object = ("", 0) error_message = "Address must be a 2-tuple (ipv4: (host, port)) or a 4-tuple (ipv6: (host, port, flow, scope))" if not isinstance(address_object, tuple) or \ - ((family == AF_INET and len(address_object) != 2) or (family == AF_INET6 and len(address_object) not in [2,4] )) or \ - not isinstance(address_object[0], (basestring, NoneType)) or \ - not isinstance(address_object[1], (int, long)): + ((family == AF_INET and len(address_object) != 2) or \ + (family == AF_INET6 and len(address_object) not in [2,4] )) or \ + not isinstance(address_object[0], (basestring, NoneType)) or \ + not isinstance(address_object[1], (int, long)): raise TypeError(error_message) if len(address_object) == 4 and not isinstance(address_object[3], (int, long)): raise TypeError(error_message) @@ -1520,8 +1614,10 @@ @raises_java_exception def getaddrinfo(host, port, family=AF_UNSPEC, socktype=0, proto=0, flags=0): - if _ipv4_addresses_only: - family = AF_INET + if family is None: + family = AF_UNSPEC + if socktype is None: + socktype = 0 if not family in [AF_INET, AF_INET6, AF_UNSPEC]: raise gaierror(errno.EIO, 'ai_family not supported') host = _getaddrinfo_get_host(host, family, flags) diff --git a/Lib/select.py b/Lib/select.py --- a/Lib/select.py +++ b/Lib/select.py @@ -8,5 +8,5 @@ POLLHUP, POLLNVAL, error, - #poll, + poll, select) diff --git a/Lib/ssl.py b/Lib/ssl.py --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -1,5 +1,10 @@ +import base64 +import errno import logging +import os.path +import textwrap import time +import threading try: # jarjar-ed version @@ -20,27 +25,33 @@ SSL_ERROR_ZERO_RETURN, SSL_ERROR_WANT_CONNECT, SSL_ERROR_EOF, - SSL_ERROR_INVALID_ERROR_CODE) + SSL_ERROR_INVALID_ERROR_CODE, + error as socket_error) from _sslcerts import _get_ssl_context from java.text import SimpleDateFormat -from java.util import Locale, TimeZone +from java.util import ArrayList, Locale, TimeZone +from java.util.concurrent import CountDownLatch from javax.naming.ldap import LdapName from javax.security.auth.x500 import X500Principal -log = logging.getLogger("socket") +log = logging.getLogger("_socket") +# Pretend to be OpenSSL +OPENSSL_VERSION = "OpenSSL 1.0.0 (as emulated by Java SSL)" +OPENSSL_VERSION_NUMBER = 0x1000000L +OPENSSL_VERSION_INFO = (1, 0, 0, 0, 0) + CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED = range(3) -# FIXME need to map to java names as well; there's also possibility some difference between -# SSLv2 (Java) and PROTOCOL_SSLv23 (Python) but reading the docs suggest not -# http://docs.oracle.com/javase/7/docs/technotes/guides/security/StandardNames.html#SSLContext - -# Currently ignored, since we just use the default in Java. FIXME -PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1 = range(4) -_PROTOCOL_NAMES = {PROTOCOL_SSLv2: 'SSLv2', PROTOCOL_SSLv3: 'SSLv3', PROTOCOL_SSLv23: 'SSLv23', PROTOCOL_TLSv1: 'TLSv1'} +# Do not support PROTOCOL_SSLv2, it is highly insecure and it is optional +_, PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1 = range(4) +_PROTOCOL_NAMES = { + PROTOCOL_SSLv3: 'SSLv3', + PROTOCOL_SSLv23: 'SSLv23', + PROTOCOL_TLSv1: 'TLSv1'} _rfc2822_date_format = SimpleDateFormat("MMM dd HH:mm:ss yyyy z", Locale.US) _rfc2822_date_format.setTimeZone(TimeZone.getTimeZone("GMT")) @@ -59,8 +70,7 @@ } _cert_name_types = [ - # FIXME only entry 2 - DNS - has been confirmed w/ cpython; - # everything else is coming from this doc: + # Fields documented in # http://docs.oracle.com/javase/7/docs/api/java/security/cert/X509Certificate.html#getSubjectAlternativeNames() "other", "rfc822", @@ -80,7 +90,7 @@ def initChannel(self, ch): pipeline = ch.pipeline() - pipeline.addLast("ssl", self.ssl_handler) + pipeline.addFirst("ssl", self.ssl_handler) class SSLSocket(object): @@ -89,47 +99,61 @@ keyfile, certfile, ca_certs, do_handshake_on_connect, server_side): self.sock = sock + self.do_handshake_on_connect = do_handshake_on_connect self._sock = sock._sock # the real underlying socket self.context = _get_ssl_context(keyfile, certfile, ca_certs) self.engine = self.context.createSSLEngine() + self.server_side = server_side self.engine.setUseClientMode(not server_side) - self.ssl_handler = SslHandler(self.engine) - self.already_handshaked = False - self.do_handshake_on_connect = do_handshake_on_connect + self.ssl_handler = None + # _sslobj is used to follow CPython convention that an object + # means we have handshaked, as used by existing code that + # looks at this internal + self._sslobj = None + self.handshake_count = 0 - if self.do_handshake_on_connect and hasattr(self._sock, "connected") and self._sock.connected: - self.already_handshaked = True - log.debug("Adding SSL handler to pipeline after connection", extra={"sock": self._sock}) - self._sock.channel.pipeline().addFirst("ssl", self.ssl_handler) - self._sock._post_connect() - self._sock._notify_selectors() - self._sock._unlatch() - - def handshake_step(result): - log.debug("SSL handshaking %s", result, extra={"sock": self._sock}) - if not hasattr(self._sock, "activity_latch"): # need a better discriminant - self._sock._post_connect() - self._sock._notify_selectors() - - self.ssl_handler.handshakeFuture().addListener(handshake_step) - if self.do_handshake_on_connect and self.already_handshaked: - time.sleep(0.1) # FIXME do we need this sleep - self.ssl_handler.handshakeFuture().sync() - log.debug("SSL handshaking completed", extra={"sock": self._sock}) + if self.do_handshake_on_connect and self.sock._sock.connected: + self.do_handshake() def connect(self, addr): log.debug("Connect SSL with handshaking %s", self.do_handshake_on_connect, extra={"sock": self._sock}) self._sock._connect(addr) if self.do_handshake_on_connect: - self.already_handshaked = True - if self._sock.connected: - log.debug("Already connected, adding SSL handler to pipeline...", extra={"sock": self._sock}) + self.do_handshake() + + def unwrap(self): + self._sock.channel.pipeline().remove("ssl") + self.ssl_handler.close() + return self._sock + + def do_handshake(self): + log.debug("SSL handshaking", extra={"sock": self._sock}) + + def handshake_step(result): + log.debug("SSL handshaking completed %s", result, extra={"sock": self._sock}) + if not hasattr(self._sock, "active_latch"): + log.debug("Post connect step", extra={"sock": self._sock}) + self._sock._post_connect() + self._sock._unlatch() + self._sslobj = object() # we have now handshaked + self._notify_selectors() + + if self.ssl_handler is None: + self.ssl_handler = SslHandler(self.engine) + self.ssl_handler.handshakeFuture().addListener(handshake_step) + + if hasattr(self._sock, "connected") and self._sock.connected: + # The underlying socket is already connected, so some extra work to manage + log.debug("Adding SSL handler to pipeline after connection", extra={"sock": self._sock}) self._sock.channel.pipeline().addFirst("ssl", self.ssl_handler) else: log.debug("Not connected, adding SSL initializer...", extra={"sock": self._sock}) self._sock.connect_handlers.append(SSLInitializer(self.ssl_handler)) - # Various pass through methods to the wrapper socket + handshake = self.ssl_handler.handshakeFuture() + self._sock._handle_channel_future(handshake, "SSL handshake") + + # Various pass through methods to the wrapped socket def send(self, data): return self.sock.send(data) @@ -140,6 +164,18 @@ def recv(self, bufsize, flags=0): return self.sock.recv(bufsize, flags) + def recvfrom(self, bufsize, flags=0): + return self.sock.recvfrom(bufsize, flags) + + def recvfrom_into(self, buffer, nbytes=0, flags=0): + return self.sock.recvfrom_into(buffer, nbytes, flags) + + def recv_into(self, buffer, nbytes=0, flags=0): + return self.sock.recv_into(buffer, nbytes, flags) + + def sendto(self, string, arg1, arg2=None): + raise socket_error(errno.EPROTO) + def close(self): self.sock.close() @@ -175,12 +211,6 @@ def _notify_selectors(self): self._sock._notify_selectors() - def do_handshake(self): - if not self.already_handshaked: - log.debug("Not handshaked, so adding SSL handler", extra={"sock": self._sock}) - self.already_handshaked = True - self._sock.channel.pipeline().addFirst("ssl", self.ssl_handler) - def getpeername(self): return self.sock.getpeername() @@ -240,9 +270,91 @@ do_handshake_on_connect=do_handshake_on_connect) -def unwrap_socket(sock): - # FIXME removing SSL handler from pipeline should suffice, but low pri for now - raise NotImplemented() +# some utility functions + +def cert_time_to_seconds(cert_time): + + """Takes a date-time string in standard ASN1_print form + ("MON DAY 24HOUR:MINUTE:SEC YEAR TIMEZONE") and return + a Python time value in seconds past the epoch.""" + + import time + return time.mktime(time.strptime(cert_time, "%b %d %H:%M:%S %Y GMT")) + +PEM_HEADER = "-----BEGIN CERTIFICATE-----" +PEM_FOOTER = "-----END CERTIFICATE-----" + +def DER_cert_to_PEM_cert(der_cert_bytes): + + """Takes a certificate in binary DER format and returns the + PEM version of it as a string.""" + + if hasattr(base64, 'standard_b64encode'): + # preferred because older API gets line-length wrong + f = base64.standard_b64encode(der_cert_bytes) + return (PEM_HEADER + '\n' + + textwrap.fill(f, 64) + '\n' + + PEM_FOOTER + '\n') + else: + return (PEM_HEADER + '\n' + + base64.encodestring(der_cert_bytes) + + PEM_FOOTER + '\n') + +def PEM_cert_to_DER_cert(pem_cert_string): + + """Takes a certificate in ASCII PEM format and returns the + DER-encoded version of it as a byte sequence""" + + if not pem_cert_string.startswith(PEM_HEADER): + raise ValueError("Invalid PEM encoding; must start with %s" + % PEM_HEADER) + if not pem_cert_string.strip().endswith(PEM_FOOTER): + raise ValueError("Invalid PEM encoding; must end with %s" + % PEM_FOOTER) + d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)] + return base64.decodestring(d) + +def get_server_certificate(addr, ssl_version=PROTOCOL_SSLv3, ca_certs=None): + + """Retrieve the certificate from the server at the specified address, + and return it as a PEM-encoded string. + If 'ca_certs' is specified, validate the server cert against it. + If 'ssl_version' is specified, use it in the connection attempt.""" + + host, port = addr + if (ca_certs is not None): + cert_reqs = CERT_REQUIRED + else: + cert_reqs = CERT_NONE + s = wrap_socket(socket(), ssl_version=ssl_version, + cert_reqs=cert_reqs, ca_certs=ca_certs) + s.connect(addr) + dercert = s.getpeercert(True) + s.close() + return DER_cert_to_PEM_cert(dercert) + +def get_protocol_name(protocol_code): + return _PROTOCOL_NAMES.get(protocol_code, '') + +# a replacement for the old socket.ssl function + +def sslwrap_simple(sock, keyfile=None, certfile=None): + + """A replacement for the old socket.ssl function. Designed + for compability with Python 2.5 and earlier. Will disappear in + Python 3.0.""" + + ssl_sock = wrap_socket(sock, keyfile=keyfile, certfile=certfile, ssl_version=PROTOCOL_SSLv23) + try: + sock.getpeername() + except socket_error: + # no, no connection yet + pass + else: + # yes, do the handshake + ssl_sock.do_handshake() + + return ssl_sock # Underlying Java does a good job of managing entropy, so these are just no-ops @@ -251,7 +363,8 @@ return True def RAND_egd(path): - pass + if os.path.abspath(str(path)) != path: + raise TypeError("Must be an absolute path, but ignoring it regardless") def RAND_add(bytes, entropy): pass diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -16,7 +16,7 @@ import thread, threading from weakref import proxy from StringIO import StringIO -from _socket import _check_threadpool_for_pending_threads +from _socket import _check_threadpool_for_pending_threads, NIO_GROUP PORT = 50100 HOST = 'localhost' @@ -126,6 +126,17 @@ if not self.server_ready.isSet(): self.server_ready.set() self.client_ready.wait() + + def _assert_no_pending_threads(self, group, msg): + # Wait up to one second for there not to be pending threads + for i in xrange(10): + pending_threads = _check_threadpool_for_pending_threads(group) + if len(pending_threads) == 0: + break + time.sleep(0.1) + + if pending_threads: + self.fail("Pending threads in Netty msg={} pool={}".format(msg, pprint.pformat(pending_threads))) def _tearDown(self): self.done.wait() # wait for the client to exit @@ -134,16 +145,13 @@ msg = None if not self.queue.empty(): msg = self.queue.get() + + self._assert_no_pending_threads(NIO_GROUP, "Client thread pool") + if hasattr(self, "srv"): + self._assert_no_pending_threads(self.srv.group, "Server thread pool") - # Wait up to one second for there not to be pending threads - for i in xrange(10): - pending_threads = _check_threadpool_for_pending_threads() - if len(pending_threads) == 0: - break - time.sleep(0.1) - - if pending_threads or msg: - self.fail("msg={} Pending threads in Netty pool={}".format(msg, pprint.pformat(pending_threads))) + if msg: + self.fail("msg={}".format(msg)) def clientRun(self, test_func): -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Thu Jul 17 18:06:51 2014 From: jython-checkins at python.org (jim.baker) Date: Thu, 17 Jul 2014 18:06:51 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Support_UTF-8_encoded_Java_?= =?utf-8?q?source_files_in_builds_on_systems_where_the?= Message-ID: <3hDgMH0WKNz7LjQ@mail.python.org> http://hg.python.org/jython/rev/f52f81d83a74 changeset: 7349:f52f81d83a74 user: Werner Mendizabal date: Thu Jul 17 10:06:12 2014 -0600 summary: Support UTF-8 encoded Java source files in builds on systems where the default encoding is not UTF-8. Fixes http://bugs.jython.org/issue2180 files: ACKNOWLEDGMENTS | 1 + build.xml | 3 ++- 2 files changed, 3 insertions(+), 1 deletions(-) diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS --- a/ACKNOWLEDGMENTS +++ b/ACKNOWLEDGMENTS @@ -110,6 +110,7 @@ Richard Eckart de Castilho Timoth?e Lecomte Peter Holloway + Werner Mendizabal Local Variables: mode: indented-text diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -474,7 +474,8 @@ deprecation="${deprecation}" nowarn="${nowarn}" memoryMaximumSize="192m" - fork="true"> + fork="true" + encoding="UTF-8"> -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Jul 26 11:03:31 2014 From: jython-checkins at python.org (jim.baker) Date: Sat, 26 Jul 2014 11:03:31 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_max_such_that_any_raise?= =?utf-8?q?d_ValueError_uses_the_correct_error_message_text=2E?= Message-ID: <3hL1Xg6NH1z7LjZ@mail.python.org> http://hg.python.org/jython/rev/8f4fef31df0b changeset: 7350:8f4fef31df0b user: Henning Jacobs date: Sat Jul 26 10:51:23 2014 +0200 summary: Fix max such that any raised ValueError uses the correct error message text. Fixes http://bugs.jython.org/issue2130 files: ACKNOWLEDGMENTS | 1 + Lib/test/test_builtin_jy.py | 9 +++++++++ src/org/python/core/__builtin__.java | 2 +- 3 files changed, 11 insertions(+), 1 deletions(-) diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS --- a/ACKNOWLEDGMENTS +++ b/ACKNOWLEDGMENTS @@ -111,6 +111,7 @@ Timoth?e Lecomte Peter Holloway Werner Mendizabal + Henning Jacobs Local Variables: mode: indented-text diff --git a/Lib/test/test_builtin_jy.py b/Lib/test/test_builtin_jy.py --- a/Lib/test/test_builtin_jy.py +++ b/Lib/test/test_builtin_jy.py @@ -42,6 +42,15 @@ self.assertTrue(numeric < Ellipsis) self.assertTrue(Ellipsis > numeric) + def test_max_error_message(self): + 'fix for http://bugs.jython.org/issue2130' + try: + max([]) + except ValueError, e: + self.assertEqual(str(e), 'max of empty sequence') + else: + self.fail('max with empty sequence should raise a proper ValueError') + class LoopTest(unittest.TestCase): def test_break(self): diff --git a/src/org/python/core/__builtin__.java b/src/org/python/core/__builtin__.java --- a/src/org/python/core/__builtin__.java +++ b/src/org/python/core/__builtin__.java @@ -1526,7 +1526,7 @@ } } if (max == null) { - throw Py.ValueError("min of empty sequence"); + throw Py.ValueError("max of empty sequence"); } return max; } -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Jul 26 11:34:54 2014 From: jython-checkins at python.org (jim.baker) Date: Sat, 26 Jul 2014 11:34:54 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_HTML_reporting_for_JUni?= =?utf-8?q?t_XML_test_results_as_an_ant_command?= Message-ID: <3hL2Dt1hZWz7LjS@mail.python.org> http://hg.python.org/jython/rev/d22384ceddcd changeset: 7351:d22384ceddcd user: Darjus Loktevic date: Sat Jul 26 11:34:37 2014 +0200 summary: Add HTML reporting for JUnit XML test results as an ant command files: ACKNOWLEDGMENTS | 1 + build.xml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 0 deletions(-) diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS --- a/ACKNOWLEDGMENTS +++ b/ACKNOWLEDGMENTS @@ -112,6 +112,7 @@ Peter Holloway Werner Mendizabal Henning Jacobs + Darjus Loktevic Local Variables: mode: indented-text diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -135,6 +135,8 @@ + + @@ -1042,6 +1044,16 @@ + + + + + + + + + + -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Jul 26 16:59:15 2014 From: jython-checkins at python.org (oti.humbel) Date: Sat, 26 Jul 2014 16:59:15 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_up_maxmem_to_512m_for_compi?= =?utf-8?q?le_target?= Message-ID: <3hL9R7119Wz7LkD@mail.python.org> http://hg.python.org/jython/rev/b0e5c8c77a25 changeset: 7352:b0e5c8c77a25 user: larsbutler date: Sat Jul 26 16:57:02 2014 +0200 summary: up maxmem to 512m for compile target files: build.xml | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/build.xml b/build.xml --- a/build.xml +++ b/build.xml @@ -475,7 +475,7 @@ debug="${debug}" deprecation="${deprecation}" nowarn="${nowarn}" - memoryMaximumSize="192m" + memoryMaximumSize="512m" fork="true" encoding="UTF-8"> -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Jul 26 17:38:22 2014 From: jython-checkins at python.org (oti.humbel) Date: Sat, 26 Jul 2014 17:38:22 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_add_=2Egitignore_=28using_i?= =?utf-8?q?nformation_from_=2Ehgignore=29?= Message-ID: <3hLBJG4r3wz7Lk6@mail.python.org> http://hg.python.org/jython/rev/895d67fdffaa changeset: 7353:895d67fdffaa user: Henning Jacobs date: Sat Jul 26 17:37:26 2014 +0200 summary: add .gitignore (using information from .hgignore) (https://github.com/jythontools/jython/pull/3) files: .gitignore | 39 +++++++++++++++++++++++++++++++++++++++ 1 files changed, 39 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore new file mode 100644 --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +*.class +*.pyc +*.pyd +*.pyo +*.orig +*.rej +*.swp +\#* +*~ +# IntelliJ files +*.eml +*.ipr +*.iml +*.iws +.idea/* + +# Eclipse files +.classpath +.externalToolBuilders/* +.project +.pydevproject + +# Netbeans files +nbproject +nbbuild.xml + +.vagrant + +.AppleDouble +.DS_Store +.settings +__pycache__ +ant.properties +bin +build +cachedir +dist +target +profile.txt -- Repository URL: http://hg.python.org/jython From jython-checkins at python.org Sat Jul 26 17:44:14 2014 From: jython-checkins at python.org (oti.humbel) Date: Sat, 26 Jul 2014 17:44:14 +0200 (CEST) Subject: [Jython-checkins] =?utf-8?q?jython=3A_add_helper_script_to_apply_?= =?utf-8?q?github_pull_requests_to_the_hg=2Epython=2Eorg_repo?= Message-ID: <3hLBR26y5Xz7Lk6@mail.python.org> http://hg.python.org/jython/rev/744d673392b4 changeset: 7354:744d673392b4 user: Henning Jacobs date: Sat Jul 26 17:43:01 2014 +0200 summary: add helper script to apply github pull requests to the hg.python.org repo (https://github.com/jythontools/jython/pull/5) files: Misc/apply-github-pull-request.py | 126 ++++++++++++++++++ 1 files changed, 126 insertions(+), 0 deletions(-) diff --git a/Misc/apply-github-pull-request.py b/Misc/apply-github-pull-request.py new file mode 100755 --- /dev/null +++ b/Misc/apply-github-pull-request.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +''' +Helper script to apply pull requests from github.com to local hg working copy. + +Example: + +cd jython-hg-workdir +apply-github-pull-request.py 4 + +''' + +import argparse +import getpass +import os +import requests +import subprocess +import sys +import tempfile + + +def get_pull_url(repo, pr_id): + return 'https://github.com/{}/pull/{}'.format(repo, pr_id) + + +def get_added_files(diff): + '''hacky approach to extract added files from github diff output''' + + prefix = '+++ b/' + lastline = None + for line in diff.splitlines(): + line = line.strip() + if line.startswith(prefix) and lastline and lastline == '--- /dev/null': + yield line[len(prefix):] + lastline = line + + +def main(args): + + if not os.path.exists('.hg'): + print 'ERROR: No .hg folder found.' + print 'Please make sure you run this script from within your local hg.python.org/jython checkout.' + return 1 + + password = args.github_password + + if password and password.startswith('@'): + # if the command line password starts with "@", we read it from a file + # (this prevents exposing the password on the command line / bash history) + with open(password[1:]) as fd: + password = fd.read().strip() + + if not password: + password = getpass.getpass('{}@github.com: '.format(args.github_user)) + + from requests.auth import HTTPBasicAuth + auth = HTTPBasicAuth(args.github_user, password) + r = requests.get('https://api.github.com/repos/{}/pulls/{}'.format(args.github_repo, args.pull_request_id), + auth=auth) + if r.status_code != 200: + print 'ERROR:' + print r.json() + return 1 + + data = r.json() + + r = requests.get('https://api.github.com/users/{}'.format(data['user']['login']), auth=auth) + if r.status_code != 200: + print 'ERROR:' + print r.json() + return 1 + + user_data = r.json() + commiter = '{} <{}>'.format(user_data['name'], user_data['email']) + + print 'Pull Request {} by {}:'.format(data['number'], commiter) + print data['title'] + print data['body'] + print '-' * 40 + + r = requests.get('{}.diff'.format(get_pull_url(args.github_repo, args.pull_request_id))) + + patch_contents = r.text + + print patch_contents + + added_files = set(get_added_files(patch_contents)) + + with tempfile.NamedTemporaryFile(suffix='-github-patch-{}'.format(args.pull_request_id)) as fd: + fd.write(patch_contents) + fd.flush() + cmd = [ + 'patch', + '-p1', + '--batch', + '--forward', + '-i', + fd.name, + ] + if args.dry_run: + cmd.append('--dry-run') + p = subprocess.Popen(cmd) + p.communicate() + + print '-' * 40 + print 'Applied pull request {} into your current working directory.'.format(args.pull_request_id) + print 'Please check the changes (run unit tests) and commit...' + print '-' * 40 + for fn in sorted(added_files): + print 'hg add {}'.format(fn) + print 'hg ci -u "{}" -m "{} ({})"'.format(commiter, data['title'], get_pull_url(args.github_repo, + args.pull_request_id)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--github-repo', default='jythontools/jython') + parser.add_argument('-u', '--github-user', default=getpass.getuser()) + parser.add_argument('-p', '--github-password', + help='Your github password (you can use "@" to read the password from a file)') + parser.add_argument('--dry-run', action='store_true', help='Dry-run mode: only show what would be done') + parser.add_argument('pull_request_id', help='Pull request ID on github.com') + + args = parser.parse_args() + sys.exit(main(args)) -- Repository URL: http://hg.python.org/jython