[Python-checkins] cpython (merge default -> default): Merge.
charles-francois.natali
python-checkins at python.org
Sat Aug 31 17:34:53 CEST 2013
http://hg.python.org/cpython/rev/78f3c32a92ce
changeset: 85472:78f3c32a92ce
parent: 85471:d37e1d846fe3
parent: 85469:1faaec66c73d
user: Charles-François Natali <cf.natali at gmail.com>
date: Sat Aug 31 17:32:30 2013 +0200
summary:
Merge.
files:
Doc/library/xml.etree.elementtree.rst | 34 ++++----
Include/pythonrun.h | 1 +
Lib/test/test_subprocess.py | 34 +++++++++
Misc/NEWS | 4 +
Python/pythonrun.c | 1 +
Python/random.c | 55 +++++++++++---
6 files changed, 99 insertions(+), 30 deletions(-)
diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst
--- a/Doc/library/xml.etree.elementtree.rst
+++ b/Doc/library/xml.etree.elementtree.rst
@@ -105,7 +105,7 @@
>>> root[0][1].text
'2008'
-Pull API for asynchronous parsing
+Pull API for non-blocking parsing
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Most parsing functions provided by this module require to read the whole
@@ -121,18 +121,18 @@
incrementally with :meth:`XMLPullParser.feed` calls. To get the parsed XML
elements, call :meth:`XMLPullParser.read_events`. Here's an example::
- >>> asyncparser = ET.XMLPullParser(['start', 'end'])
- >>> asyncparser.feed('<mytag>sometext')
- >>> list(asyncparser.read_events())
+ >>> parser = ET.XMLPullParser(['start', 'end'])
+ >>> parser.feed('<mytag>sometext')
+ >>> list(parser.read_events())
[('start', <Element 'mytag' at 0x7fa66db2be58>)]
- >>> asyncparser.feed(' more text</mytag>')
- >>> for event, elem in asyncparser.read_events():
+ >>> parser.feed(' more text</mytag>')
+ >>> for event, elem in parser.read_events():
... print(event)
... print(elem.tag, 'text=', elem.text)
...
end
-The obvious use case is applications that operate in an asynchronous fashion
+The obvious use case is applications that operate in a non-blocking fashion
where the XML data is being received from a socket or read incrementally from
some storage device. In such cases, blocking reads are unacceptable.
@@ -427,8 +427,8 @@
Note that while :func:`iterparse` builds the tree incrementally, it issues
blocking reads on *source* (or the file it names). As such, it's unsuitable
- for asynchronous applications where blocking reads can't be made. For fully
- asynchronous parsing, see :class:`XMLPullParser`.
+ for applications where blocking reads can't be made. For fully non-blocking
+ parsing, see :class:`XMLPullParser`.
.. note::
@@ -1016,14 +1016,14 @@
.. class:: XMLPullParser(events=None)
- A pull parser suitable for nonblocking (asynchronous) applications. Its
- input-side API is similar to that of :class:`XMLParser`, but instead of
- pushing calls to a callback target, :class:`XMLPullParser` collects an
- internal list of parsing events and lets the user read from it. *events* is a
- sequence of events to report back. The supported events are the strings
- ``"start"``, ``"end"``, ``"start-ns"`` and ``"end-ns"`` (the "ns" events are
- used to get detailed namespace information). If *events* is omitted, only
- ``"end"`` events are reported.
+ A pull parser suitable for non-blocking applications. Its input-side API is
+ similar to that of :class:`XMLParser`, but instead of pushing calls to a
+ callback target, :class:`XMLPullParser` collects an internal list of parsing
+ events and lets the user read from it. *events* is a sequence of events to
+ report back. The supported events are the strings ``"start"``, ``"end"``,
+ ``"start-ns"`` and ``"end-ns"`` (the "ns" events are used to get detailed
+ namespace information). If *events* is omitted, only ``"end"`` events are
+ reported.
.. method:: feed(data)
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -246,6 +246,7 @@
PyAPI_FUNC(void) _PyGC_Fini(void);
PyAPI_FUNC(void) PySlice_Fini(void);
PyAPI_FUNC(void) _PyType_Fini(void);
+PyAPI_FUNC(void) _PyRandom_Fini(void);
PyAPI_DATA(PyThreadState *) _Py_Finalizing;
#endif
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -22,6 +22,10 @@
import resource
except ImportError:
resource = None
+try:
+ import threading
+except ImportError:
+ threading = None
mswindows = (sys.platform == "win32")
@@ -1020,6 +1024,36 @@
if c.exception.errno not in (errno.ENOENT, errno.EACCES):
raise c.exception
+ @unittest.skipIf(threading is None, "threading required")
+ def test_double_close_on_error(self):
+ # Issue #18851
+ fds = []
+ def open_fds():
+ for i in range(20):
+ fds.extend(os.pipe())
+ time.sleep(0.001)
+ t = threading.Thread(target=open_fds)
+ t.start()
+ try:
+ with self.assertRaises(EnvironmentError):
+ subprocess.Popen(['nonexisting_i_hope'],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ finally:
+ t.join()
+ exc = None
+ for fd in fds:
+ # If a double close occurred, some of those fds will
+ # already have been closed by mistake, and os.close()
+ # here will raise.
+ try:
+ os.close(fd)
+ except OSError as e:
+ exc = e
+ if exc is not None:
+ raise exc
+
def test_issue8780(self):
# Ensure that stdout is inherited from the parent
# if stdout=PIPE is not used
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -51,6 +51,10 @@
Library
-------
+- Issue #18756: os.urandom() now uses a lazily-opened persistent file
+ descriptor, so as to avoid using many file descriptors when run in
+ parallel from multiple threads.
+
- Issue #18418: After fork(), reinit all threads states, not only active ones.
Patch by A. Jesse Jiryu Davis.
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -625,6 +625,7 @@
PyDict_Fini();
PySlice_Fini();
_PyGC_Fini();
+ _PyRandom_Fini();
/* Cleanup Unicode implementation */
_PyUnicode_Fini();
diff --git a/Python/random.c b/Python/random.c
--- a/Python/random.c
+++ b/Python/random.c
@@ -90,6 +90,7 @@
#if !defined(MS_WINDOWS) && !defined(__VMS)
+static int urandom_fd = -1;
/* Read size bytes from /dev/urandom into buffer.
Call Py_FatalError() on error. */
@@ -133,18 +134,30 @@
if (size <= 0)
return 0;
- Py_BEGIN_ALLOW_THREADS
- fd = _Py_open("/dev/urandom", O_RDONLY);
- Py_END_ALLOW_THREADS
- if (fd < 0)
- {
- if (errno == ENOENT || errno == ENXIO ||
- errno == ENODEV || errno == EACCES)
- PyErr_SetString(PyExc_NotImplementedError,
- "/dev/urandom (or equivalent) not found");
+ if (urandom_fd >= 0)
+ fd = urandom_fd;
+ else {
+ Py_BEGIN_ALLOW_THREADS
+ fd = _Py_open("/dev/urandom", O_RDONLY);
+ Py_END_ALLOW_THREADS
+ if (fd < 0)
+ {
+ if (errno == ENOENT || errno == ENXIO ||
+ errno == ENODEV || errno == EACCES)
+ PyErr_SetString(PyExc_NotImplementedError,
+ "/dev/urandom (or equivalent) not found");
+ else
+ PyErr_SetFromErrno(PyExc_OSError);
+ return -1;
+ }
+ if (urandom_fd >= 0) {
+ /* urandom_fd was initialized by another thread while we were
+ not holding the GIL, keep it. */
+ close(fd);
+ fd = urandom_fd;
+ }
else
- PyErr_SetFromErrno(PyExc_OSError);
- return -1;
+ urandom_fd = fd;
}
Py_BEGIN_ALLOW_THREADS
@@ -168,12 +181,20 @@
PyErr_Format(PyExc_RuntimeError,
"Failed to read %zi bytes from /dev/urandom",
size);
- close(fd);
return -1;
}
- close(fd);
return 0;
}
+
+static void
+dev_urandom_close(void)
+{
+ if (urandom_fd >= 0) {
+ close(urandom_fd);
+ urandom_fd = -1;
+ }
+}
+
#endif /* !defined(MS_WINDOWS) && !defined(__VMS) */
/* Fill buffer with pseudo-random bytes generated by a linear congruent
@@ -271,3 +292,11 @@
#endif
}
}
+
+void
+_PyRandom_Fini(void)
+{
+#if !defined(MS_WINDOWS) && !defined(__VMS)
+ dev_urandom_close();
+#endif
+}
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list