[Python-checkins] cpython (merge 2.7 -> 2.7): merge heads
petri.lehtinen
python-checkins at python.org
Mon Oct 24 20:33:33 CEST 2011
http://hg.python.org/cpython/rev/817946aadecb
changeset: 73107:817946aadecb
branch: 2.7
parent: 73102:5d7164febff1
parent: 73099:3465a9b2d25c
user: Petri Lehtinen <petri at digip.org>
date: Mon Oct 24 21:29:20 2011 +0300
summary:
merge heads
files:
Doc/library/2to3.rst | 4 +-
Doc/library/socketserver.rst | 48 ++-
Doc/library/subprocess.rst | 255 +++++++++++-------
Lib/multiprocessing/pool.py | 6 +-
Lib/test/test_multiprocessing.py | 14 +
Misc/NEWS | 11 +
6 files changed, 220 insertions(+), 118 deletions(-)
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst
--- a/Doc/library/2to3.rst
+++ b/Doc/library/2to3.rst
@@ -123,7 +123,9 @@
.. 2to3fixer:: callable
Converts ``callable(x)`` to ``isinstance(x, collections.Callable)``, adding
- an import to :mod:`collections` if needed.
+ an import to :mod:`collections` if needed. Note ``callable(x)`` has returned
+ in Python 3.2, so if you do not intend to support Python 3.1, you can disable
+ this fixer.
.. 2to3fixer:: dict
diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst
--- a/Doc/library/socketserver.rst
+++ b/Doc/library/socketserver.rst
@@ -225,6 +225,7 @@
desired. If :meth:`handle_request` receives no incoming requests within the
timeout period, the :meth:`handle_timeout` method is called.
+
There are various server methods that can be overridden by subclasses of base
server classes like :class:`TCPServer`; these methods aren't useful to external
users of the server object.
@@ -355,7 +356,7 @@
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
- print "%s wrote:" % self.client_address[0]
+ print "{} wrote:".format(self.client_address[0])
print self.data
# just send back the same data, but upper-cased
self.request.send(self.data.upper())
@@ -379,7 +380,7 @@
# self.rfile is a file-like object created by the handler;
# we can now use e.g. readline() instead of raw recv() calls
self.data = self.rfile.readline().strip()
- print "%s wrote:" % self.client_address[0]
+ print "{} wrote:".format(self.client_address[0])
print self.data
# Likewise, self.wfile is a file-like object used to write back
# to the client
@@ -402,16 +403,18 @@
# Create a socket (SOCK_STREAM means a TCP socket)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- # Connect to server and send data
- sock.connect((HOST, PORT))
- sock.send(data + "\n")
+ try:
+ # Connect to server and send data
+ sock.connect((HOST, PORT))
+ sock.send(data + "\n")
- # Receive data from the server and shut down
- received = sock.recv(1024)
- sock.close()
+ # Receive data from the server and shut down
+ received = sock.recv(1024)
+ finally:
+ sock.close()
- print "Sent: %s" % data
- print "Received: %s" % received
+ print "Sent: {}".format(data)
+ print "Received: {}".format(received)
The output of the example should look something like this:
@@ -452,7 +455,7 @@
def handle(self):
data = self.request[0].strip()
socket = self.request[1]
- print "%s wrote:" % self.client_address[0]
+ print "{} wrote:".format(self.client_address[0])
print data
socket.sendto(data.upper(), self.client_address)
@@ -477,8 +480,8 @@
sock.sendto(data + "\n", (HOST, PORT))
received = sock.recv(1024)
- print "Sent: %s" % data
- print "Received: %s" % received
+ print "Sent: {}".format(data)
+ print "Received: {}".format(received)
The output of the example should look exactly like for the TCP server example.
@@ -499,8 +502,8 @@
def handle(self):
data = self.request.recv(1024)
- cur_thread = threading.currentThread()
- response = "%s: %s" % (cur_thread.getName(), data)
+ cur_thread = threading.current_thread()
+ response = "{}: {}".format(cur_thread.name, data)
self.request.send(response)
class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
@@ -509,10 +512,12 @@
def client(ip, port, message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((ip, port))
- sock.send(message)
- response = sock.recv(1024)
- print "Received: %s" % response
- sock.close()
+ try:
+ sock.send(message)
+ response = sock.recv(1024)
+ print "Received: {}".format(response)
+ finally:
+ sock.close()
if __name__ == "__main__":
# Port 0 means to select an arbitrary unused port
@@ -525,9 +530,9 @@
# more thread for each request
server_thread = threading.Thread(target=server.serve_forever)
# Exit the server thread when the main thread terminates
- server_thread.setDaemon(True)
+ server_thread.daemon = True
server_thread.start()
- print "Server loop running in thread:", server_thread.getName()
+ print "Server loop running in thread:", server_thread.name
client(ip, port, "Hello World 1")
client(ip, port, "Hello World 2")
@@ -535,6 +540,7 @@
server.shutdown()
+
The output of the example should look something like this::
$ python ThreadedTCPServer.py
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -31,7 +31,142 @@
Using the subprocess Module
---------------------------
-This module defines one class called :class:`Popen`:
+The recommended interface to this module is to use the following convenience
+functions for all use cases they can handle. For more advanced use cases, the
+underlying :class:`Popen` interface can be used directly.
+
+
+.. function:: call(args, *, stdin=None, stdout=None, stderr=None)
+
+ Run the command described by *args*. Wait for command to complete, then
+ return the :attr:`returncode` attribute.
+
+ The arguments shown above are merely the most common ones, described below
+ in :ref:`frequently-used-arguments`. The full function signature is the
+ same as that of the :class:`Popen` constructor - the convenience functions
+ pass all supplied arguments directly through to that interface.
+
+ Examples::
+
+ >>> subprocess.call(["ls", "-l"])
+ 0
+
+ >>> subprocess.call(["python", "-c", "import sys; sys.exit(1)"])
+ 1
+
+ .. warning::
+
+ Like :meth:`Popen.wait`, this will deadlock when using
+ ``stdout=PIPE`` and/or ``stderr=PIPE`` and the child process
+ generates enough output to a pipe such that it blocks waiting
+ for the OS pipe buffer to accept more data.
+
+
+.. function:: check_call(*callargs, **kwargs)
+
+ Run command with arguments. Wait for command to complete. If the return
+ code was zero then return, otherwise raise :exc:`CalledProcessError`. The
+ :exc:`CalledProcessError` object will have the return code in the
+ :attr:`returncode` attribute.
+
+ The arguments are the same as for :func:`call`. Examples::
+
+ >>> subprocess.check_call(["ls", "-l"])
+ 0
+
+ >>> subprocess.check_call(["python", "-c", "import sys; sys.exit(1)"])
+ Traceback (most recent call last):
+ ...
+ subprocess.CalledProcessError: Command '['python', '-c', 'import sys; sys.exit(1)']' returned non-zero exit status 1
+
+ .. versionadded:: 2.5
+
+ .. warning::
+
+ See the warning for :func:`call`.
+
+
+.. function:: check_output(*callargs, **kwargs)
+
+ Run command with arguments and return its output as a byte string.
+
+ If the return code was non-zero it raises a :exc:`CalledProcessError`. The
+ :exc:`CalledProcessError` object will have the return code in the
+ :attr:`returncode` attribute and any output in the :attr:`output`
+ attribute.
+
+ Examples::
+
+ >>> subprocess.check_output(["ls", "-l", "/dev/null"])
+ 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
+
+ >>> subprocess.check_output(["python", "-c", "import sys; sys.exit(1)"])
+ Traceback (most recent call last):
+ ...
+ subprocess.CalledProcessError: Command '['python', '-c', 'import sys; sys.exit(1)']' returned non-zero exit status 1
+
+ The arguments are the same as for :func:`call`, except that *stdout* is
+ not allowed as it is used internally. To also capture standard error in
+ the result, use ``stderr=subprocess.STDOUT``::
+
+ >>> subprocess.check_output(
+ ... ["/bin/sh", "-c", "ls non_existent_file; exit 0"],
+ ... stderr=subprocess.STDOUT)
+ 'ls: non_existent_file: No such file or directory\n'
+
+ .. versionadded:: 2.7
+
+
+.. data:: PIPE
+
+ Special value that can be used as the *stdin*, *stdout* or *stderr* argument
+ to :class:`Popen` and indicates that a pipe to the standard stream should be
+ opened.
+
+
+.. data:: STDOUT
+
+ Special value that can be used as the *stderr* argument to :class:`Popen` and
+ indicates that standard error should go into the same handle as standard
+ output.
+
+
+.. _frequently-used-arguments:
+
+Frequently Used Arguments
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To support a wide variety of use cases, the :class:`Popen` constructor (and
+the convenience functions) accept a large number of optional arguments. For
+most typical use cases, many of these arguments can be safely left at their
+default values. The arguments that are most commonly needed are:
+
+ *args* should be a string, or a sequence of program arguments. Providing
+ a sequence of arguments is generally preferred, as it allows the module to
+ take care of any required escaping and quoting of arguments (e.g. to permit
+ spaces in file names)
+
+ *stdin*, *stdout* and *stderr* specify the executed program's standard input,
+ standard output and standard error file handles, respectively. Valid values
+ are :data:`PIPE`, an existing file descriptor (a positive integer), an
+ existing file object, and ``None``. :data:`PIPE` indicates that a new pipe
+ to the child should be created. With the default settings of ``None``, no
+ redirection will occur; the child's file handles will be inherited from the
+ parent. Additionally, *stderr* can be :data:`STDOUT`, which indicates that
+ the stderr data from the child process should be captured into the same file
+ handle as for stdout.
+
+These options, along with all of the other options, are described in more
+detail in the :class:`Popen` constructor documentation.
+
+
+Popen Constuctor
+^^^^^^^^^^^^^^^^
+
+The underlying process creation and management in this module is handled by
+the :class:`Popen` class. It offers a lot of flexibility so that developers
+are able to handle the less common cases not covered by the convenience
+functions.
.. class:: Popen(args, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0)
@@ -126,14 +261,15 @@
You don't need ``shell=True`` to run a batch file, nor to run a console-based
executable.
- *stdin*, *stdout* and *stderr* specify the executed programs' standard input,
+ *stdin*, *stdout* and *stderr* specify the executed program's standard input,
standard output and standard error file handles, respectively. Valid values
are :data:`PIPE`, an existing file descriptor (a positive integer), an
existing file object, and ``None``. :data:`PIPE` indicates that a new pipe
- to the child should be created. With ``None``, no redirection will occur;
- the child's file handles will be inherited from the parent. Additionally,
- *stderr* can be :data:`STDOUT`, which indicates that the stderr data from the
- applications should be captured into the same file handle as for stdout.
+ to the child should be created. With the default settings of ``None``, no
+ redirection will occur; the child's file handles will be inherited from the
+ parent. Additionally, *stderr* can be :data:`STDOUT`, which indicates that
+ the stderr data from the child process should be captured into the same file
+ handle as for stdout.
If *preexec_fn* is set to a callable object, this object will be called in the
child process just before the child is executed. (Unix only)
@@ -184,87 +320,6 @@
:data:`CREATE_NEW_PROCESS_GROUP`. (Windows only)
-.. data:: PIPE
-
- Special value that can be used as the *stdin*, *stdout* or *stderr* argument
- to :class:`Popen` and indicates that a pipe to the standard stream should be
- opened.
-
-
-.. data:: STDOUT
-
- Special value that can be used as the *stderr* argument to :class:`Popen` and
- indicates that standard error should go into the same handle as standard
- output.
-
-
-Convenience Functions
-^^^^^^^^^^^^^^^^^^^^^
-
-This module also defines the following shortcut functions:
-
-
-.. function:: call(*popenargs, **kwargs)
-
- Run command with arguments. Wait for command to complete, then return the
- :attr:`returncode` attribute.
-
- The arguments are the same as for the :class:`Popen` constructor. Example::
-
- >>> retcode = subprocess.call(["ls", "-l"])
-
- .. warning::
-
- Like :meth:`Popen.wait`, this will deadlock when using
- ``stdout=PIPE`` and/or ``stderr=PIPE`` and the child process
- generates enough output to a pipe such that it blocks waiting
- for the OS pipe buffer to accept more data.
-
-
-.. function:: check_call(*popenargs, **kwargs)
-
- Run command with arguments. Wait for command to complete. If the exit code was
- zero then return, otherwise raise :exc:`CalledProcessError`. The
- :exc:`CalledProcessError` object will have the return code in the
- :attr:`returncode` attribute.
-
- The arguments are the same as for the :class:`Popen` constructor. Example::
-
- >>> subprocess.check_call(["ls", "-l"])
- 0
-
- .. versionadded:: 2.5
-
- .. warning::
-
- See the warning for :func:`call`.
-
-
-.. function:: check_output(*popenargs, **kwargs)
-
- Run command with arguments and return its output as a byte string.
-
- If the exit code was non-zero it raises a :exc:`CalledProcessError`. The
- :exc:`CalledProcessError` object will have the return code in the
- :attr:`returncode`
- attribute and output in the :attr:`output` attribute.
-
- The arguments are the same as for the :class:`Popen` constructor. Example::
-
- >>> subprocess.check_output(["ls", "-l", "/dev/null"])
- 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
-
- The stdout argument is not allowed as it is used internally.
- To capture standard error in the result, use ``stderr=subprocess.STDOUT``::
-
- >>> subprocess.check_output(
- ... ["/bin/sh", "-c", "ls non_existent_file; exit 0"],
- ... stderr=subprocess.STDOUT)
- 'ls: non_existent_file: No such file or directory\n'
-
- .. versionadded:: 2.7
-
-
Exceptions
^^^^^^^^^^
@@ -523,12 +578,15 @@
Replacing Older Functions with the subprocess Module
----------------------------------------------------
-In this section, "a ==> b" means that b can be used as a replacement for a.
+In this section, "a becomes b" means that b can be used as a replacement for a.
.. note::
All functions in this section fail (more or less) silently if the executed
- program cannot be found; this module raises an :exc:`OSError` exception.
+ program cannot be found; this module raises an :exc:`OSError` exception. In
+ addition, the replacements using :func:`check_output` will fail with a
+ :exc:`CalledProcessError` if the requested operation produces a non-zero
+ return code.
In the following examples, we assume that the subprocess module is imported with
"from subprocess import \*".
@@ -540,8 +598,8 @@
::
output=`mycmd myarg`
- ==>
- output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0]
+ # becomes
+ output = check_output(["mycmd", "myarg"])
Replacing shell pipeline
@@ -550,7 +608,7 @@
::
output=`dmesg | grep hda`
- ==>
+ # becomes
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
@@ -559,15 +617,22 @@
The p1.stdout.close() call after starting the p2 is important in order for p1
to receive a SIGPIPE if p2 exits before p1.
+Alternatively, for trusted input, the shell's pipeline may still be used
+directly:
+
+ output=`dmesg | grep hda`
+ # becomes
+ output=check_output("dmesg | grep hda", shell=True)
+
+
Replacing :func:`os.system`
^^^^^^^^^^^^^^^^^^^^^^^^^^^
::
sts = os.system("mycmd" + " myarg")
- ==>
- p = Popen("mycmd" + " myarg", shell=True)
- sts = os.waitpid(p.pid, 0)[1]
+ # becomes
+ sts = call("mycmd" + " myarg", shell=True)
Notes:
diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py
--- a/Lib/multiprocessing/pool.py
+++ b/Lib/multiprocessing/pool.py
@@ -294,7 +294,11 @@
@staticmethod
def _handle_workers(pool):
- while pool._worker_handler._state == RUN and pool._state == RUN:
+ thread = threading.current_thread()
+
+ # Keep maintaining workers until the cache gets drained, unless the pool
+ # is terminated.
+ while thread._state == RUN or (pool._cache and thread._state != TERMINATE):
pool._maintain_pool()
time.sleep(0.1)
# send sentinel to stop workers
diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py
--- a/Lib/test/test_multiprocessing.py
+++ b/Lib/test/test_multiprocessing.py
@@ -1168,6 +1168,20 @@
p.close()
p.join()
+ def test_pool_worker_lifetime_early_close(self):
+ # Issue #10332: closing a pool whose workers have limited lifetimes
+ # before all the tasks completed would make join() hang.
+ p = multiprocessing.Pool(3, maxtasksperchild=1)
+ results = []
+ for i in range(6):
+ results.append(p.apply_async(sqr, (i, 0.3)))
+ p.close()
+ p.join()
+ # check the results
+ for (j, res) in enumerate(results):
+ self.assertEqual(res.get(), sqr(j))
+
+
#
# Test that manager has expected number of shared objects left
#
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -69,6 +69,9 @@
Library
-------
+- Issue #10332: multiprocessing: fix a race condition when a Pool is closed
+ before all tasks have completed.
+
- Issue #1548891: The cStringIO.StringIO() constructor now encodes unicode
arguments with the system default encoding just like the write() method
does, instead of converting it to a raw buffer. This also fixes handling of
@@ -345,6 +348,14 @@
- Issue #12057: Add tests for ISO 2022 codecs (iso2022_jp, iso2022_jp_2,
iso2022_kr).
+Documentation
+-------------
+
+- Issue #13237: Reorganise subprocess documentation to emphasise convenience
+ functions and the most commonly needed arguments to Popen.
+
+- Issue #13141: Demonstrate recommended style for SocketServer examples.
+
What's New in Python 2.7.2?
===========================
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list