[Python-checkins] cpython (merge default -> default): Merge.
larry.hastings
python-checkins at python.org
Mon Sep 9 14:14:30 CEST 2013
http://hg.python.org/cpython/rev/1cfe82916d98
changeset: 85646:1cfe82916d98
parent: 85645:6b211a0c8042
parent: 85641:02cdd725a6eb
user: Larry Hastings <larry at hastings.org>
date: Mon Sep 09 21:12:21 2013 +0900
summary:
Merge.
files:
Doc/library/enum.rst | 5 +-
Doc/library/itertools.rst | 4 +-
Doc/library/os.rst | 2 -
Doc/library/profile.rst | 8 +-
Doc/library/test.rst | 5 +-
Doc/library/unittest.rst | 5 +-
Doc/using/cmdline.rst | 16 +-
Include/pystate.h | 26 +
Include/setobject.h | 1 -
Lib/_dummy_thread.py | 4 +
Lib/decimal.py | 2 +-
Lib/multiprocessing/connection.py | 10 +-
Lib/test/support/__init__.py | 31 +-
Lib/test/test_faulthandler.py | 30 +-
Lib/test/test_os.py | 35 +-
Lib/test/test_regrtest.py | 3 +
Lib/test/test_selectors.py | 3 +-
Lib/test/test_site.py | 26 +-
Lib/test/test_socket.py | 31 +
Lib/test/test_tcl.py | 22 +-
Lib/test/test_threading.py | 206 +++------
Lib/threading.py | 95 ++-
Lib/unittest/__init__.py | 2 +-
Misc/NEWS | 20 +-
Modules/_multiprocessing/multiprocessing.c | 6 +-
Modules/_threadmodule.c | 62 ++
Modules/faulthandler.c | 5 +-
Modules/itertoolsmodule.c | 3 +-
Objects/object.c | 1 -
Objects/setobject.c | 217 ++++-----
Python/pystate.c | 5 +
31 files changed, 528 insertions(+), 363 deletions(-)
diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst
--- a/Doc/library/enum.rst
+++ b/Doc/library/enum.rst
@@ -38,7 +38,8 @@
... blue = 3
...
-..note: Nomenclature
+.. note:: Nomenclature
+
- The class :class:`Color` is an *enumeration* (or *enum*)
- The attributes :attr:`Color.red`, :attr:`Color.green`, etc., are
*enumeration members* (or *enum members*).
@@ -474,7 +475,7 @@
4. %-style formatting: `%s` and `%r` call :class:`Enum`'s :meth:`__str__` and
:meth:`__repr__` respectively; other codes (such as `%i` or `%h` for
IntEnum) treat the enum member as its mixed-in type.
-5. :class:`str`.:meth:`__format__` (or :func:`format`) will use the mixed-in
+5. :meth:`str.__format__` (or :func:`format`) will use the mixed-in
type's :meth:`__format__`. If the :class:`Enum`'s :func:`str` or
:func:`repr` is desired use the `!s` or `!r` :class:`str` format codes.
diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -48,6 +48,7 @@
==================== ============================ ================================================= =============================================================
:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15``
:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F``
+chain.from_iterable iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F``
:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F``
:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1``
:func:`filterfalse` pred, seq elements of seq where pred(elem) is False ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8``
@@ -156,9 +157,8 @@
.. classmethod:: chain.from_iterable(iterable)
Alternate constructor for :func:`chain`. Gets chained inputs from a
- single iterable argument that is evaluated lazily. Equivalent to::
+ single iterable argument that is evaluated lazily. Roughly equivalent to::
- @classmethod
def from_iterable(iterables):
# chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
for it in iterables:
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -757,8 +757,6 @@
As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``.
- Availability: Unix.
-
.. function:: fstat(fd)
diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst
--- a/Doc/library/profile.rst
+++ b/Doc/library/profile.rst
@@ -247,11 +247,13 @@
import cProfile, pstats, io
pr = cProfile.Profile()
pr.enable()
- ... do something ...
+ # ... do something ...
pr.disable()
s = io.StringIO()
- ps = pstats.Stats(pr, stream=s)
- ps.print_results()
+ sortby = 'cumulative'
+ ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
+ ps.print_stats()
+ print(s.getvalue())
.. method:: enable()
diff --git a/Doc/library/test.rst b/Doc/library/test.rst
--- a/Doc/library/test.rst
+++ b/Doc/library/test.rst
@@ -263,12 +263,15 @@
Used when tests are executed by :mod:`test.regrtest`.
-.. function:: findfile(filename)
+.. function:: findfile(filename, subdir=None)
Return the path to the file named *filename*. If no match is found
*filename* is returned. This does not equal a failure since it could be the
path to the file.
+ Setting *subdir* indicates a relative path to use to find the file
+ rather than looking directly in the path directories.
+
.. function:: run_unittest(\*classes)
diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst
--- a/Doc/library/unittest.rst
+++ b/Doc/library/unittest.rst
@@ -1674,8 +1674,7 @@
A list containing 2-tuples of :class:`TestCase` instances and strings
holding formatted tracebacks. Each tuple represents a test where a failure
- was explicitly signalled using the :meth:`TestCase.fail\*` or
- :meth:`TestCase.assert\*` methods.
+ was explicitly signalled using the :meth:`TestCase.assert\*` methods.
.. attribute:: skipped
@@ -1772,7 +1771,7 @@
.. method:: addError(test, err)
- Called when the test case *test* raises an unexpected exception *err* is a
+ Called when the test case *test* raises an unexpected exception. *err* is a
tuple of the form returned by :func:`sys.exc_info`: ``(type, value,
traceback)``.
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -511,9 +511,9 @@
.. envvar:: PYTHONDONTWRITEBYTECODE
- If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the
- import of source modules. This is equivalent to specifying the :option:`-B`
- option.
+ If this is set to a non-empty string, Python won't try to write ``.pyc`` or
+ ``.pyo`` files on the import of source modules. This is equivalent to
+ specifying the :option:`-B` option.
.. envvar:: PYTHONHASHSEED
@@ -582,11 +582,11 @@
.. envvar:: PYTHONFAULTHANDLER
- If this environment variable is set, :func:`faulthandler.enable` is called
- at startup: install a handler for :const:`SIGSEGV`, :const:`SIGFPE`,
- :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` signals to dump the
- Python traceback. This is equivalent to :option:`-X` ``faulthandler``
- option.
+ If this environment variable is set to a non-empty string,
+ :func:`faulthandler.enable` is called at startup: install a handler for
+ :const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and
+ :const:`SIGILL` signals to dump the Python traceback. This is equivalent to
+ :option:`-X` ``faulthandler`` option.
.. versionadded:: 3.3
diff --git a/Include/pystate.h b/Include/pystate.h
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -118,6 +118,32 @@
int trash_delete_nesting;
PyObject *trash_delete_later;
+ /* Called when a thread state is deleted normally, but not when it
+ * is destroyed after fork().
+ * Pain: to prevent rare but fatal shutdown errors (issue 18808),
+ * Thread.join() must wait for the join'ed thread's tstate to be unlinked
+ * from the tstate chain. That happens at the end of a thread's life,
+ * in pystate.c.
+ * The obvious way doesn't quite work: create a lock which the tstate
+ * unlinking code releases, and have Thread.join() wait to acquire that
+ * lock. The problem is that we _are_ at the end of the thread's life:
+ * if the thread holds the last reference to the lock, decref'ing the
+ * lock will delete the lock, and that may trigger arbitrary Python code
+ * if there's a weakref, with a callback, to the lock. But by this time
+ * _PyThreadState_Current is already NULL, so only the simplest of C code
+ * can be allowed to run (in particular it must not be possible to
+ * release the GIL).
+ * So instead of holding the lock directly, the tstate holds a weakref to
+ * the lock: that's the value of on_delete_data below. Decref'ing a
+ * weakref is harmless.
+ * on_delete points to _threadmodule.c's static release_sentinel() function.
+ * After the tstate is unlinked, release_sentinel is called with the
+ * weakref-to-lock (on_delete_data) argument, and release_sentinel releases
+ * the indirectly held lock.
+ */
+ void (*on_delete)(void *);
+ void *on_delete_data;
+
/* XXX signal handlers should also be here */
} PyThreadState;
diff --git a/Include/setobject.h b/Include/setobject.h
--- a/Include/setobject.h
+++ b/Include/setobject.h
@@ -105,7 +105,6 @@
PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
PyAPI_FUNC(int) PySet_ClearFreeList(void);
-PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out);
#endif
#ifdef __cplusplus
diff --git a/Lib/_dummy_thread.py b/Lib/_dummy_thread.py
--- a/Lib/_dummy_thread.py
+++ b/Lib/_dummy_thread.py
@@ -81,6 +81,10 @@
raise error("setting thread stack size not supported")
return 0
+def _set_sentinel():
+ """Dummy implementation of _thread._set_sentinel()."""
+ return LockType()
+
class LockType(object):
"""Class implementing dummy implementation of _thread.LockType.
diff --git a/Lib/decimal.py b/Lib/decimal.py
--- a/Lib/decimal.py
+++ b/Lib/decimal.py
@@ -21,7 +21,7 @@
and IEEE standard 854-1987:
- www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html
+ http://en.wikipedia.org/wiki/IEEE_854-1987
Decimal floating point has finite precision with arbitrarily large bounds.
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -878,13 +878,21 @@
import selectors
+ # poll/select have the advantage of not requiring any extra file
+ # descriptor, contrarily to epoll/kqueue (also, they require a single
+ # syscall).
+ if hasattr(selectors, 'PollSelector'):
+ _WaitSelector = selectors.PollSelector
+ else:
+ _WaitSelector = selectors.SelectSelector
+
def wait(object_list, timeout=None):
'''
Wait till an object in object_list is ready/readable.
Returns list of those objects in object_list which are ready/readable.
'''
- with selectors.DefaultSelector() as selector:
+ with _WaitSelector() as selector:
for obj in object_list:
selector.register(obj, selectors.EVENT_READ)
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -860,24 +860,31 @@
finally:
os.umask(oldmask)
-# TEST_HOME refers to the top level directory of the "test" package
+# TEST_HOME_DIR refers to the top level directory of the "test" package
# that contains Python's regression test suite
-TEST_HOME = os.path.dirname(os.path.abspath(__file__))
+TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
-def findfile(file, here=TEST_HOME, subdir=None):
+# TEST_DATA_DIR is used as a target download location for remote resources
+TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data")
+
+def findfile(filename, subdir=None):
"""Try to find a file on sys.path or in the test directory. If it is not
found the argument passed to the function is returned (this does not
- necessarily signal failure; could still be the legitimate path)."""
- if os.path.isabs(file):
- return file
+ necessarily signal failure; could still be the legitimate path).
+
+ Setting *subdir* indicates a relative path to use to find the file
+ rather than looking directly in the path directories.
+ """
+ if os.path.isabs(filename):
+ return filename
if subdir is not None:
- file = os.path.join(subdir, file)
- path = sys.path
- path = [os.path.dirname(here)] + path
+ filename = os.path.join(subdir, filename)
+ path = [TEST_HOME_DIR] + sys.path
for dn in path:
- fn = os.path.join(dn, file)
+ fn = os.path.join(dn, filename)
if os.path.exists(fn): return fn
- return file
+ return filename
def create_empty_file(filename):
"""Create an empty file. If the file already exists, truncate it."""
@@ -914,7 +921,7 @@
filename = urllib.parse.urlparse(url)[2].split('/')[-1] # '/': it's URL!
- fn = os.path.join(os.path.dirname(__file__), "data", filename)
+ fn = os.path.join(TEST_DATA_DIR, filename)
def check_valid_file(fn):
f = open(fn, *args, **kw)
diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py
--- a/Lib/test/test_faulthandler.py
+++ b/Lib/test/test_faulthandler.py
@@ -265,17 +265,33 @@
# By default, the module should be disabled
code = "import faulthandler; print(faulthandler.is_enabled())"
args = (sys.executable, '-E', '-c', code)
- # use subprocess module directly because test.script_helper adds
- # "-X faulthandler" to the command line
- stdout = subprocess.check_output(args)
- self.assertEqual(stdout.rstrip(), b"False")
+ # don't use assert_python_ok() because it always enable faulthandler
+ output = subprocess.check_output(args)
+ self.assertEqual(output.rstrip(), b"False")
def test_sys_xoptions(self):
# Test python -X faulthandler
code = "import faulthandler; print(faulthandler.is_enabled())"
- rc, stdout, stderr = assert_python_ok("-X", "faulthandler", "-c", code)
- stdout = (stdout + stderr).strip()
- self.assertEqual(stdout, b"True")
+ args = (sys.executable, "-E", "-X", "faulthandler", "-c", code)
+ # don't use assert_python_ok() because it always enable faulthandler
+ output = subprocess.check_output(args)
+ self.assertEqual(output.rstrip(), b"True")
+
+ def test_env_var(self):
+ # empty env var
+ code = "import faulthandler; print(faulthandler.is_enabled())"
+ args = (sys.executable, "-c", code)
+ env = os.environ.copy()
+ env['PYTHONFAULTHANDLER'] = ''
+ # don't use assert_python_ok() because it always enable faulthandler
+ output = subprocess.check_output(args, env=env)
+ self.assertEqual(output.rstrip(), b"False")
+
+ # non-empty env var
+ env = os.environ.copy()
+ env['PYTHONFAULTHANDLER'] = '1'
+ output = subprocess.check_output(args, env=env)
+ self.assertEqual(output.rstrip(), b"True")
def check_dump_traceback(self, filename):
"""
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -34,6 +34,10 @@
import resource
except ImportError:
resource = None
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
from test.script_helper import assert_python_ok
@@ -2300,18 +2304,37 @@
class FDInheritanceTests(unittest.TestCase):
- def test_get_inheritable(self):
+ def test_get_set_inheritable(self):
fd = os.open(__file__, os.O_RDONLY)
self.addCleanup(os.close, fd)
- for inheritable in (False, True):
- os.set_inheritable(fd, inheritable)
- self.assertEqual(os.get_inheritable(fd), inheritable)
+ self.assertEqual(os.get_inheritable(fd), False)
- def test_set_inheritable(self):
+ os.set_inheritable(fd, True)
+ self.assertEqual(os.get_inheritable(fd), True)
+
+ @unittest.skipIf(fcntl is None, "need fcntl")
+ def test_get_inheritable_cloexec(self):
fd = os.open(__file__, os.O_RDONLY)
self.addCleanup(os.close, fd)
+ self.assertEqual(os.get_inheritable(fd), False)
+
+ # clear FD_CLOEXEC flag
+ flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+ flags &= ~fcntl.FD_CLOEXEC
+ fcntl.fcntl(fd, fcntl.F_SETFD, flags)
+
+ self.assertEqual(os.get_inheritable(fd), True)
+
+ @unittest.skipIf(fcntl is None, "need fcntl")
+ def test_set_inheritable_cloexec(self):
+ fd = os.open(__file__, os.O_RDONLY)
+ self.addCleanup(os.close, fd)
+ self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC,
+ fcntl.FD_CLOEXEC)
+
os.set_inheritable(fd, True)
- self.assertEqual(os.get_inheritable(fd), True)
+ self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC,
+ 0)
def test_open(self):
fd = os.open(__file__, os.O_RDONLY)
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -3,6 +3,7 @@
"""
import argparse
+import faulthandler
import getopt
import os.path
import unittest
@@ -25,6 +26,8 @@
regrtest._parse_args([opt])
self.assertIn('Run Python regression tests.', out.getvalue())
+ @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'),
+ "faulthandler.dump_traceback_later() required")
def test_timeout(self):
ns = regrtest._parse_args(['--timeout', '4.2'])
self.assertEqual(ns.timeout, 4.2)
diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py
--- a/Lib/test/test_selectors.py
+++ b/Lib/test/test_selectors.py
@@ -301,6 +301,7 @@
class ScalableSelectorMixIn:
+ # see issue #18963 for why it's skipped on older OS X versions
@support.requires_mac_ver(10, 5)
@unittest.skipUnless(resource, "Test needs resource module")
def test_above_fd_setsize(self):
@@ -313,7 +314,7 @@
self.addCleanup(resource.setrlimit, resource.RLIMIT_NOFILE,
(soft, hard))
NUM_FDS = hard
- except OSError:
+ except (OSError, ValueError):
NUM_FDS = soft
# guard for already allocated FDs (stdin, stdout...)
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -5,6 +5,7 @@
"""
import unittest
+import test.support
from test.support import run_unittest, TESTFN, EnvironmentVarGuard
from test.support import captured_stderr
import builtins
@@ -373,9 +374,10 @@
self.assertTrue(hasattr(builtins, "exit"))
def test_setting_copyright(self):
- # 'copyright' and 'credits' should be in builtins
+ # 'copyright', 'credits', and 'license' should be in builtins
self.assertTrue(hasattr(builtins, "copyright"))
self.assertTrue(hasattr(builtins, "credits"))
+ self.assertTrue(hasattr(builtins, "license"))
def test_setting_help(self):
# 'help' should be set in builtins
@@ -402,5 +404,27 @@
self.fail("sitecustomize not imported automatically")
+class LicenseURL(unittest.TestCase):
+ """Test accessibility of the license."""
+
+ @unittest.skipUnless(str(license).startswith('See http://'),
+ 'license is available as a file')
+ def test_license_page(self):
+ """urlopen should return the license page"""
+ pat = r'^See (http://www\.python\.org/download/releases/[^/]+/license/)$'
+ mo = re.search(pat, str(license))
+ self.assertIsNotNone(mo, msg='can\'t find appropriate url in license')
+ if mo is not None:
+ url = mo.group(1)
+ with test.support.transient_internet(url):
+ import urllib.request, urllib.error
+ try:
+ with urllib.request.urlopen(url) as data:
+ code = data.getcode()
+ except urllib.error.HTTPError as e:
+ code = e.code
+ self.assertEqual(code, 200, msg=url)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -26,6 +26,10 @@
import multiprocessing
except ImportError:
multiprocessing = False
+try:
+ import fcntl
+except ImportError:
+ fcntl = None
HOST = support.HOST
MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return
@@ -4804,6 +4808,33 @@
sock.set_inheritable(False)
self.assertEqual(sock.get_inheritable(), False)
+ @unittest.skipIf(fcntl is None, "need fcntl")
+ def test_get_inheritable_cloexec(self):
+ sock = socket.socket()
+ with sock:
+ fd = sock.fileno()
+ self.assertEqual(sock.get_inheritable(), False)
+
+ # clear FD_CLOEXEC flag
+ flags = fcntl.fcntl(fd, fcntl.F_GETFD)
+ flags &= ~fcntl.FD_CLOEXEC
+ fcntl.fcntl(fd, fcntl.F_SETFD, flags)
+
+ self.assertEqual(sock.get_inheritable(), True)
+
+ @unittest.skipIf(fcntl is None, "need fcntl")
+ def test_set_inheritable_cloexec(self):
+ sock = socket.socket()
+ with sock:
+ fd = sock.fileno()
+ self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC,
+ fcntl.FD_CLOEXEC)
+
+ sock.set_inheritable(True)
+ self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC,
+ 0)
+
+
@unittest.skipUnless(hasattr(socket, "socketpair"),
"need socket.socketpair()")
def test_socketpair(self):
diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py
--- a/Lib/test/test_tcl.py
+++ b/Lib/test/test_tcl.py
@@ -15,6 +15,14 @@
from tkinter import Tcl
from _tkinter import TclError
+tcl_version = _tkinter.TCL_VERSION.split('.')
+try:
+ for i in range(len(tcl_version)):
+ tcl_version[i] = int(tcl_version[i])
+except ValueError:
+ pass
+tcl_version = tuple(tcl_version)
+
class TkinterTest(unittest.TestCase):
@@ -200,9 +208,12 @@
(('a', 3.4), ('a', 3.4)),
((), ()),
(call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
- (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)),
- (1, '\u20ac', '\u20ac', (3.4,))),
]
+ if tcl_version >= (8, 5):
+ testcases += [
+ (call('dict', 'create', 1, '\u20ac', b'\xe2\x82\xac', (3.4,)),
+ (1, '\u20ac', '\u20ac', (3.4,))),
+ ]
for arg, res in testcases:
self.assertEqual(splitlist(arg), res, msg=arg)
self.assertRaises(TclError, splitlist, '{')
@@ -234,9 +245,12 @@
(('a', (2, 3.4)), ('a', (2, 3.4))),
((), ()),
(call('list', 1, '2', (3.4,)), (1, '2', (3.4,))),
- (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)),
- (12, '\u20ac', '\u20ac', (3.4,))),
]
+ if tcl_version >= (8, 5):
+ testcases += [
+ (call('dict', 'create', 12, '\u20ac', b'\xe2\x82\xac', (3.4,)),
+ (12, '\u20ac', '\u20ac', (3.4,))),
+ ]
for arg, res in testcases:
self.assertEqual(split(arg), res, msg=arg)
diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
--- a/Lib/test/test_threading.py
+++ b/Lib/test/test_threading.py
@@ -109,7 +109,7 @@
if verbose:
print('waiting for all tasks to complete')
for t in threads:
- t.join(NUMTASKS)
+ t.join()
self.assertTrue(not t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertFalse(t.ident is None)
@@ -539,6 +539,40 @@
self.assertEqual(err, b"")
self.assertEqual(data, "Thread-1\nTrue\nTrue\n")
+ def test_tstate_lock(self):
+ # Test an implementation detail of Thread objects.
+ started = _thread.allocate_lock()
+ finish = _thread.allocate_lock()
+ started.acquire()
+ finish.acquire()
+ def f():
+ started.release()
+ finish.acquire()
+ time.sleep(0.01)
+ # The tstate lock is None until the thread is started
+ t = threading.Thread(target=f)
+ self.assertIs(t._tstate_lock, None)
+ t.start()
+ started.acquire()
+ self.assertTrue(t.is_alive())
+ # The tstate lock can't be acquired when the thread is running
+ # (or suspended).
+ tstate_lock = t._tstate_lock
+ self.assertFalse(tstate_lock.acquire(timeout=0), False)
+ finish.release()
+ # When the thread ends, the state_lock can be successfully
+ # acquired.
+ self.assertTrue(tstate_lock.acquire(timeout=5), False)
+ # But is_alive() is still True: we hold _tstate_lock now, which
+ # prevents is_alive() from knowing the thread's end-of-life C code
+ # is done.
+ self.assertTrue(t.is_alive())
+ # Let is_alive() find out the C code is done.
+ tstate_lock.release()
+ self.assertFalse(t.is_alive())
+ # And verify the thread disposed of _tstate_lock.
+ self.assertTrue(t._tstate_lock is None)
+
class ThreadJoinOnShutdown(BaseTestCase):
@@ -613,144 +647,8 @@
"""
self._run_and_join(script)
- def assertScriptHasOutput(self, script, expected_output):
- rc, out, err = assert_python_ok("-c", script)
- data = out.decode().replace('\r', '')
- self.assertEqual(data, expected_output)
-
- @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
@unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
- def test_4_joining_across_fork_in_worker_thread(self):
- # There used to be a possible deadlock when forking from a child
- # thread. See http://bugs.python.org/issue6643.
-
- # The script takes the following steps:
- # - The main thread in the parent process starts a new thread and then
- # tries to join it.
- # - The join operation acquires the Lock inside the thread's _block
- # Condition. (See threading.py:Thread.join().)
- # - We stub out the acquire method on the condition to force it to wait
- # until the child thread forks. (See LOCK ACQUIRED HERE)
- # - The child thread forks. (See LOCK HELD and WORKER THREAD FORKS
- # HERE)
- # - The main thread of the parent process enters Condition.wait(),
- # which releases the lock on the child thread.
- # - The child process returns. Without the necessary fix, when the
- # main thread of the child process (which used to be the child thread
- # in the parent process) attempts to exit, it will try to acquire the
- # lock in the Thread._block Condition object and hang, because the
- # lock was held across the fork.
-
- script = """if 1:
- import os, time, threading
-
- finish_join = False
- start_fork = False
-
- def worker():
- # Wait until this thread's lock is acquired before forking to
- # create the deadlock.
- global finish_join
- while not start_fork:
- time.sleep(0.01)
- # LOCK HELD: Main thread holds lock across this call.
- childpid = os.fork()
- finish_join = True
- if childpid != 0:
- # Parent process just waits for child.
- os.waitpid(childpid, 0)
- # Child process should just return.
-
- w = threading.Thread(target=worker)
-
- # Stub out the private condition variable's lock acquire method.
- # This acquires the lock and then waits until the child has forked
- # before returning, which will release the lock soon after. If
- # someone else tries to fix this test case by acquiring this lock
- # before forking instead of resetting it, the test case will
- # deadlock when it shouldn't.
- condition = w._block
- orig_acquire = condition.acquire
- call_count_lock = threading.Lock()
- call_count = 0
- def my_acquire():
- global call_count
- global start_fork
- orig_acquire() # LOCK ACQUIRED HERE
- start_fork = True
- if call_count == 0:
- while not finish_join:
- time.sleep(0.01) # WORKER THREAD FORKS HERE
- with call_count_lock:
- call_count += 1
- condition.acquire = my_acquire
-
- w.start()
- w.join()
- print('end of main')
- """
- self.assertScriptHasOutput(script, "end of main\n")
-
- @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
- @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
- def test_5_clear_waiter_locks_to_avoid_crash(self):
- # Check that a spawned thread that forks doesn't segfault on certain
- # platforms, namely OS X. This used to happen if there was a waiter
- # lock in the thread's condition variable's waiters list. Even though
- # we know the lock will be held across the fork, it is not safe to
- # release locks held across forks on all platforms, so releasing the
- # waiter lock caused a segfault on OS X. Furthermore, since locks on
- # OS X are (as of this writing) implemented with a mutex + condition
- # variable instead of a semaphore, while we know that the Python-level
- # lock will be acquired, we can't know if the internal mutex will be
- # acquired at the time of the fork.
-
- script = """if True:
- import os, time, threading
-
- start_fork = False
-
- def worker():
- # Wait until the main thread has attempted to join this thread
- # before continuing.
- while not start_fork:
- time.sleep(0.01)
- childpid = os.fork()
- if childpid != 0:
- # Parent process just waits for child.
- (cpid, rc) = os.waitpid(childpid, 0)
- assert cpid == childpid
- assert rc == 0
- print('end of worker thread')
- else:
- # Child process should just return.
- pass
-
- w = threading.Thread(target=worker)
-
- # Stub out the private condition variable's _release_save method.
- # This releases the condition's lock and flips the global that
- # causes the worker to fork. At this point, the problematic waiter
- # lock has been acquired once by the waiter and has been put onto
- # the waiters list.
- condition = w._block
- orig_release_save = condition._release_save
- def my_release_save():
- global start_fork
- orig_release_save()
- # Waiter lock held here, condition lock released.
- start_fork = True
- condition._release_save = my_release_save
-
- w.start()
- w.join()
- print('end of main thread')
- """
- output = "end of worker thread\nend of main thread\n"
- self.assertScriptHasOutput(script, output)
-
- @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
- def test_6_daemon_threads(self):
+ def test_4_daemon_threads(self):
# Check that a daemon thread cannot crash the interpreter on shutdown
# by manipulating internal structures that are being disposed of in
# the main thread.
@@ -867,6 +765,38 @@
# The thread was joined properly.
self.assertEqual(os.read(r, 1), b"x")
+ def test_threads_join_2(self):
+ # Same as above, but a delay gets introduced after the thread's
+ # Python code returned but before the thread state is deleted.
+ # To achieve this, we register a thread-local object which sleeps
+ # a bit when deallocated.
+ r, w = os.pipe()
+ self.addCleanup(os.close, r)
+ self.addCleanup(os.close, w)
+ code = r"""if 1:
+ import os
+ import threading
+ import time
+
+ class Sleeper:
+ def __del__(self):
+ time.sleep(0.05)
+
+ tls = threading.local()
+
+ def f():
+ # Sleep a bit so that the thread is still running when
+ # Py_EndInterpreter is called.
+ time.sleep(0.05)
+ tls.x = Sleeper()
+ os.write(%d, b"x")
+ threading.Thread(target=f).start()
+ """ % (w,)
+ ret = _testcapi.run_in_subinterp(code)
+ self.assertEqual(ret, 0)
+ # The thread was joined properly.
+ self.assertEqual(os.read(r, 1), b"x")
+
def test_daemon_threads_fatal_error(self):
subinterp_code = r"""if 1:
import os
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -33,6 +33,7 @@
# Rename some stuff so "from threading import *" is safe
_start_new_thread = _thread.start_new_thread
_allocate_lock = _thread.allocate_lock
+_set_sentinel = _thread._set_sentinel
get_ident = _thread.get_ident
ThreadError = _thread.error
try:
@@ -516,8 +517,6 @@
_active_limbo_lock = _allocate_lock()
_active = {} # maps thread id to Thread object
_limbo = {}
-
-# For debug and leak testing
_dangling = WeakSet()
# Main class for threads
@@ -548,28 +547,33 @@
else:
self._daemonic = current_thread().daemon
self._ident = None
+ self._tstate_lock = None
self._started = Event()
- self._stopped = False
- self._block = Condition(Lock())
+ self._is_stopped = False
self._initialized = True
# sys.stderr is not stored in the class like
# sys.exc_info since it can be changed between instances
self._stderr = _sys.stderr
+ # For debugging and _after_fork()
_dangling.add(self)
- def _reset_internal_locks(self):
+ def _reset_internal_locks(self, is_alive):
# private! Called by _after_fork() to reset our internal locks as
# they may be in an invalid state leading to a deadlock or crash.
- if hasattr(self, '_block'): # DummyThread deletes _block
- self._block.__init__()
self._started._reset_internal_locks()
+ if is_alive:
+ self._set_tstate_lock()
+ else:
+ # The thread isn't alive after fork: it doesn't have a tstate
+ # anymore.
+ self._tstate_lock = None
def __repr__(self):
assert self._initialized, "Thread.__init__() was not called"
status = "initial"
if self._started.is_set():
status = "started"
- if self._stopped:
+ if self._is_stopped:
status = "stopped"
if self._daemonic:
status += " daemon"
@@ -625,9 +629,18 @@
def _set_ident(self):
self._ident = get_ident()
+ def _set_tstate_lock(self):
+ """
+ Set a lock object which will be released by the interpreter when
+ the underlying thread state (see pystate.h) gets deleted.
+ """
+ self._tstate_lock = _set_sentinel()
+ self._tstate_lock.acquire()
+
def _bootstrap_inner(self):
try:
self._set_ident()
+ self._set_tstate_lock()
self._started.set()
with _active_limbo_lock:
_active[self._ident] = self
@@ -682,7 +695,6 @@
pass
finally:
with _active_limbo_lock:
- self._stop()
try:
# We don't call self._delete() because it also
# grabs _active_limbo_lock.
@@ -691,10 +703,8 @@
pass
def _stop(self):
- self._block.acquire()
- self._stopped = True
- self._block.notify_all()
- self._block.release()
+ self._is_stopped = True
+ self._tstate_lock = None
def _delete(self):
"Remove current thread from the dict of currently running threads."
@@ -738,21 +748,24 @@
raise RuntimeError("cannot join thread before it is started")
if self is current_thread():
raise RuntimeError("cannot join current thread")
+ if timeout is None:
+ self._wait_for_tstate_lock()
+ else:
+ self._wait_for_tstate_lock(timeout=timeout)
- self._block.acquire()
- try:
- if timeout is None:
- while not self._stopped:
- self._block.wait()
- else:
- deadline = _time() + timeout
- while not self._stopped:
- delay = deadline - _time()
- if delay <= 0:
- break
- self._block.wait(delay)
- finally:
- self._block.release()
+ def _wait_for_tstate_lock(self, block=True, timeout=-1):
+ # Issue #18808: wait for the thread state to be gone.
+ # At the end of the thread's life, after all knowledge of the thread
+ # is removed from C data structures, C code releases our _tstate_lock.
+ # This method passes its arguments to _tstate_lock.aquire().
+ # If the lock is acquired, the C code is done, and self._stop() is
+ # called. That sets ._is_stopped to True, and ._tstate_lock to None.
+ lock = self._tstate_lock
+ if lock is None: # already determined that the C code is done
+ assert self._is_stopped
+ elif lock.acquire(block, timeout):
+ lock.release()
+ self._stop()
@property
def name(self):
@@ -771,7 +784,10 @@
def is_alive(self):
assert self._initialized, "Thread.__init__() not called"
- return self._started.is_set() and not self._stopped
+ if self._is_stopped or not self._started.is_set():
+ return False
+ self._wait_for_tstate_lock(False)
+ return not self._is_stopped
isAlive = is_alive
@@ -835,6 +851,7 @@
def __init__(self):
Thread.__init__(self, name="MainThread", daemon=False)
+ self._set_tstate_lock()
self._started.set()
self._set_ident()
with _active_limbo_lock:
@@ -854,11 +871,6 @@
def __init__(self):
Thread.__init__(self, name=_newname("Dummy-%d"), daemon=True)
- # Thread._block consumes an OS-level locking primitive, which
- # can never be used by a _DummyThread. Since a _DummyThread
- # instance is immortal, that's bad, so release this resource.
- del self._block
-
self._started.set()
self._set_ident()
with _active_limbo_lock:
@@ -904,6 +916,14 @@
_main_thread = _MainThread()
def _shutdown():
+ # Obscure: other threads may be waiting to join _main_thread. That's
+ # dubious, but some code does it. We can't wait for C code to release
+ # the main thread's tstate_lock - that won't happen until the interpreter
+ # is nearly dead. So we release it here. Note that just calling _stop()
+ # isn't enough: other threads may already be waiting on _tstate_lock.
+ assert _main_thread._tstate_lock is not None
+ assert _main_thread._tstate_lock.locked()
+ _main_thread._tstate_lock.release()
_main_thread._stop()
t = _pickSomeNonDaemonThread()
while t:
@@ -949,18 +969,23 @@
current = current_thread()
_main_thread = current
with _active_limbo_lock:
- for thread in _enumerate():
+ # Dangling thread instances must still have their locks reset,
+ # because someone may join() them.
+ threads = set(_enumerate())
+ threads.update(_dangling)
+ for thread in threads:
# Any lock/condition variable may be currently locked or in an
# invalid state, so we reinitialize them.
- thread._reset_internal_locks()
if thread is current:
# There is only one active thread. We reset the ident to
# its new value since it can have changed.
+ thread._reset_internal_locks(True)
ident = get_ident()
thread._ident = ident
new_active[ident] = thread
else:
# All the others are already stopped.
+ thread._reset_internal_locks(False)
thread._stop()
_limbo.clear()
diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py
--- a/Lib/unittest/__init__.py
+++ b/Lib/unittest/__init__.py
@@ -11,7 +11,7 @@
import unittest
- class IntegerArithmenticTestCase(unittest.TestCase):
+ class IntegerArithmeticTestCase(unittest.TestCase):
def testAdd(self): ## test method names begin 'test*'
self.assertEqual((1 + 2), 3)
self.assertEqual(0 + 1, 1)
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -2,9 +2,6 @@
Python News
+++++++++++
-What's New in Python 3.4.0 Alpha 3?
-===================================
-
Projected Release date: 2013-09-29
Core and Builtins
@@ -13,6 +10,17 @@
Library
-------
+- The :envvar:`PYTHONFAULTHANDLER` environment variable now only enables the
+ faulthandler module if the variable is non-empty. Same behaviour than other
+ variables like :envvar:`PYTHONDONTWRITEBYTECODE`.
+
+Tests
+-----
+
+- Issue #18952: Fix regression in support data downloads introduced when
+ test.support was converted to a package. Regression noticed by Zachary
+ Ware.
+
What's New in Python 3.4.0 Alpha 2?
===================================
@@ -68,6 +76,10 @@
Library
-------
+- Issue #18808: Thread.join() now waits for the underlying thread state to
+ be destroyed before returning. This prevents unpredictable aborts in
+ Py_EndInterpreter() when some non-daemon threads are still running.
+
- Issue #18458: Prevent crashes with newer versions of libedit. Its readline
emulation has changed from 0-based indexing to 1-based like gnu readline.
@@ -75,7 +87,7 @@
readline activation code in ``site.py``.
- Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in
- the _sre moduel.
+ the _sre module.
- Issue #18830: inspect.getclasstree() no more produces duplicated entries even
when input list contains duplicates.
diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c
--- a/Modules/_multiprocessing/multiprocessing.c
+++ b/Modules/_multiprocessing/multiprocessing.c
@@ -99,13 +99,15 @@
{
HANDLE handle;
Py_buffer buf;
- int ret;
+ int ret, length;
if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf))
return NULL;
+ length = (int)Py_MIN(buf.len, INT_MAX);
+
Py_BEGIN_ALLOW_THREADS
- ret = send((SOCKET) handle, buf.buf, buf.len, 0);
+ ret = send((SOCKET) handle, buf.buf, length, 0);
Py_END_ALLOW_THREADS
PyBuffer_Release(&buf);
diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c
--- a/Modules/_threadmodule.c
+++ b/Modules/_threadmodule.c
@@ -1172,6 +1172,66 @@
This function is meant for internal and specialized purposes only.\n\
In most applications `threading.enumerate()` should be used instead.");
+static void
+release_sentinel(void *wr)
+{
+ /* Tricky: this function is called when the current thread state
+ is being deleted. Therefore, only simple C code can safely
+ execute here. */
+ PyObject *obj = PyWeakref_GET_OBJECT(wr);
+ lockobject *lock;
+ if (obj != Py_None) {
+ assert(Py_TYPE(obj) == &Locktype);
+ lock = (lockobject *) obj;
+ if (lock->locked) {
+ PyThread_release_lock(lock->lock_lock);
+ lock->locked = 0;
+ }
+ }
+ /* Deallocating a weakref with a NULL callback only calls
+ PyObject_GC_Del(), which can't call any Python code. */
+ Py_DECREF(wr);
+}
+
+static PyObject *
+thread__set_sentinel(PyObject *self)
+{
+ PyObject *wr;
+ PyThreadState *tstate = PyThreadState_Get();
+ lockobject *lock;
+
+ if (tstate->on_delete_data != NULL) {
+ /* We must support the re-creation of the lock from a
+ fork()ed child. */
+ assert(tstate->on_delete == &release_sentinel);
+ wr = (PyObject *) tstate->on_delete_data;
+ tstate->on_delete = NULL;
+ tstate->on_delete_data = NULL;
+ Py_DECREF(wr);
+ }
+ lock = newlockobject();
+ if (lock == NULL)
+ return NULL;
+ /* The lock is owned by whoever called _set_sentinel(), but the weakref
+ hangs to the thread state. */
+ wr = PyWeakref_NewRef((PyObject *) lock, NULL);
+ if (wr == NULL) {
+ Py_DECREF(lock);
+ return NULL;
+ }
+ tstate->on_delete_data = (void *) wr;
+ tstate->on_delete = &release_sentinel;
+ return (PyObject *) lock;
+}
+
+PyDoc_STRVAR(_set_sentinel_doc,
+"_set_sentinel() -> lock\n\
+\n\
+Set a sentinel lock that will be released when the current thread\n\
+state is finalized (after it is untied from the interpreter).\n\
+\n\
+This is a private API for the threading module.");
+
static PyObject *
thread_stack_size(PyObject *self, PyObject *args)
{
@@ -1247,6 +1307,8 @@
METH_NOARGS, _count_doc},
{"stack_size", (PyCFunction)thread_stack_size,
METH_VARARGS, stack_size_doc},
+ {"_set_sentinel", (PyCFunction)thread__set_sentinel,
+ METH_NOARGS, _set_sentinel_doc},
{NULL, NULL} /* sentinel */
};
diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c
--- a/Modules/faulthandler.c
+++ b/Modules/faulthandler.c
@@ -1048,8 +1048,11 @@
{
PyObject *xoptions, *key, *module, *res;
_Py_IDENTIFIER(enable);
+ char *p;
- if (!Py_GETENV("PYTHONFAULTHANDLER")) {
+ if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) {
+ /* PYTHONFAULTHANDLER environment variable is missing
+ or an empty string */
int has_key;
xoptions = PySys_GetXOptions();
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -4,7 +4,7 @@
/* Itertools module written and maintained
by Raymond D. Hettinger <python at rcn.com>
- Copyright (c) 2003 Python Software Foundation.
+ Copyright (c) 2003-2013 Python Software Foundation.
All rights reserved.
*/
@@ -4456,6 +4456,7 @@
Iterators terminating on the shortest input sequence:\n\
accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\
chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\
+chain.from_iterable([p, q, ...]) --> p0, p1, ... plast, q0, q1, ... \n\
compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\
dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\
groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v)\n\
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1955,7 +1955,6 @@
_PyFrame_DebugMallocStats(out);
_PyList_DebugMallocStats(out);
_PyMethod_DebugMallocStats(out);
- _PySet_DebugMallocStats(out);
_PyTuple_DebugMallocStats(out);
}
diff --git a/Objects/setobject.c b/Objects/setobject.c
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -1,22 +1,36 @@
/* set object implementation
+
Written and maintained by Raymond D. Hettinger <python at rcn.com>
Derived from Lib/sets.py and Objects/dictobject.c.
Copyright (c) 2003-2013 Python Software Foundation.
All rights reserved.
+
+ The basic lookup function used by all operations.
+ This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4.
+
+ The initial probe index is computed as hash mod the table size.
+ Subsequent probe indices are computed as explained in Objects/dictobject.c.
+
+ To improve cache locality, each probe inspects a series of consecutive
+ nearby entries before moving on to probes elsewhere in memory. This leaves
+ us with a hybrid of linear probing and open addressing. The linear probing
+ reduces the cost of hash collisions because consecutive memory accesses
+ tend to be much cheaper than scattered probes. After LINEAR_PROBES steps,
+ we then use open addressing with the upper bits from the hash value. This
+ helps break-up long chains of collisions.
+
+ All arithmetic on hash should ignore overflow.
+
+ Unlike the dictionary implementation, the lookkey functions can return
+ NULL if the rich comparison returns an error.
*/
#include "Python.h"
#include "structmember.h"
#include "stringlib/eq.h"
-/* This must be >= 1 */
-#define PERTURB_SHIFT 5
-
-/* This should be >= PySet_MINSIZE - 1 */
-#define LINEAR_PROBES 9
-
/* Object used as dummy key to fill deleted entries */
static PyObject _dummy_struct;
@@ -25,46 +39,15 @@
/* Exported for the gdb plugin's benefit. */
PyObject *_PySet_Dummy = dummy;
-#define INIT_NONZERO_SET_SLOTS(so) do { \
- (so)->table = (so)->smalltable; \
- (so)->mask = PySet_MINSIZE - 1; \
- (so)->hash = -1; \
- } while(0)
-#define EMPTY_TO_MINSIZE(so) do { \
- memset((so)->smalltable, 0, sizeof((so)->smalltable)); \
- (so)->used = (so)->fill = 0; \
- INIT_NONZERO_SET_SLOTS(so); \
- } while(0)
+/* ======================================================================== */
+/* ======= Begin logic for probing the hash table ========================= */
-/* Reuse scheme to save calls to malloc, free, and memset */
-#ifndef PySet_MAXFREELIST
-#define PySet_MAXFREELIST 80
-#endif
-static PySetObject *free_list[PySet_MAXFREELIST];
-static int numfree = 0;
+/* This should be >= PySet_MINSIZE - 1 */
+#define LINEAR_PROBES 9
-
-/*
-The basic lookup function used by all operations.
-This is based on Algorithm D from Knuth Vol. 3, Sec. 6.4.
-
-The initial probe index is computed as hash mod the table size.
-Subsequent probe indices are computed as explained in Objects/dictobject.c.
-
-To improve cache locality, each probe inspects a series of consecutive
-nearby entries before moving on to probes elsewhere in memory. This leaves
-us with a hybrid of linear probing and open addressing. The linear probing
-reduces the cost of hash collisions because consecutive memory accesses
-tend to be much cheaper than scattered probes. After LINEAR_PROBES steps,
-we then use open addressing with the upper bits from the hash value. This
-helps break-up long chains of collisions.
-
-All arithmetic on hash should ignore overflow.
-
-Unlike the dictionary implementation, the lookkey functions can return
-NULL if the rich comparison returns an error.
-*/
+/* This must be >= 1 */
+#define PERTURB_SHIFT 5
static setentry *
set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash)
@@ -168,8 +151,8 @@
while (1) {
if (entry->key == key
|| (entry->hash == hash
- && entry->key != dummy
- && unicode_eq(entry->key, key)))
+ && entry->key != dummy
+ && unicode_eq(entry->key, key)))
return entry;
if (entry->key == dummy && freeslot == NULL)
freeslot = entry;
@@ -200,38 +183,6 @@
}
/*
-Internal routine to insert a new key into the table.
-Used by the public insert routine.
-Eats a reference to key.
-*/
-static int
-set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash)
-{
- setentry *entry;
-
- assert(so->lookup != NULL);
- entry = so->lookup(so, key, hash);
- if (entry == NULL)
- return -1;
- if (entry->key == NULL) {
- /* UNUSED */
- so->fill++;
- entry->key = key;
- entry->hash = hash;
- so->used++;
- } else if (entry->key == dummy) {
- /* DUMMY */
- entry->key = key;
- entry->hash = hash;
- so->used++;
- } else {
- /* ACTIVE */
- Py_DECREF(key);
- }
- return 0;
-}
-
-/*
Internal routine used by set_table_resize() to insert an item which is
known to be absent from the set. This routine also assumes that
the set contains no deleted entries. Besides the performance benefit,
@@ -268,6 +219,42 @@
so->used++;
}
+/* ======== End logic for probing the hash table ========================== */
+/* ======================================================================== */
+
+
+/*
+Internal routine to insert a new key into the table.
+Used by the public insert routine.
+Eats a reference to key.
+*/
+static int
+set_insert_key(PySetObject *so, PyObject *key, Py_hash_t hash)
+{
+ setentry *entry;
+
+ assert(so->lookup != NULL);
+ entry = so->lookup(so, key, hash);
+ if (entry == NULL)
+ return -1;
+ if (entry->key == NULL) {
+ /* UNUSED */
+ so->fill++;
+ entry->key = key;
+ entry->hash = hash;
+ so->used++;
+ } else if (entry->key == dummy) {
+ /* DUMMY */
+ entry->key = key;
+ entry->hash = hash;
+ so->used++;
+ } else {
+ /* ACTIVE */
+ Py_DECREF(key);
+ }
+ return 0;
+}
+
/*
Restructure the table by allocating a new table and reinserting all
keys again. When entries have been deleted, the new table may
@@ -441,6 +428,17 @@
return DISCARD_FOUND;
}
+static void
+set_empty_to_minsize(PySetObject *so)
+{
+ memset(so->smalltable, 0, sizeof(so->smalltable));
+ so->fill = 0;
+ so->used = 0;
+ so->mask = PySet_MINSIZE - 1;
+ so->table = so->smalltable;
+ so->hash = -1;
+}
+
static int
set_clear_internal(PySetObject *so)
{
@@ -448,14 +446,13 @@
int table_is_malloced;
Py_ssize_t fill;
setentry small_copy[PySet_MINSIZE];
+
#ifdef Py_DEBUG
- Py_ssize_t i, n;
- assert (PyAnySet_Check(so));
-
- n = so->mask + 1;
- i = 0;
+ Py_ssize_t i = 0;
+ Py_ssize_t n = so->mask + 1;
#endif
+ assert (PyAnySet_Check(so));
table = so->table;
assert(table != NULL);
table_is_malloced = table != so->smalltable;
@@ -468,7 +465,7 @@
*/
fill = so->fill;
if (table_is_malloced)
- EMPTY_TO_MINSIZE(so);
+ set_empty_to_minsize(so);
else if (fill > 0) {
/* It's a small table with something that needs to be cleared.
@@ -477,7 +474,7 @@
*/
memcpy(small_copy, table, sizeof(small_copy));
table = small_copy;
- EMPTY_TO_MINSIZE(so);
+ set_empty_to_minsize(so);
}
/* else it's a small table that's already empty */
@@ -560,10 +557,7 @@
}
if (so->table != so->smalltable)
PyMem_DEL(so->table);
- if (numfree < PySet_MAXFREELIST && PyAnySet_CheckExact(so))
- free_list[numfree++] = so;
- else
- Py_TYPE(so)->tp_free(so);
+ Py_TYPE(so)->tp_free(so);
Py_TRASHCAN_SAFE_END(so)
}
@@ -1018,24 +1012,16 @@
PySetObject *so = NULL;
/* create PySetObject structure */
- if (numfree &&
- (type == &PySet_Type || type == &PyFrozenSet_Type)) {
- so = free_list[--numfree];
- assert (so != NULL && PyAnySet_CheckExact(so));
- Py_TYPE(so) = type;
- _Py_NewReference((PyObject *)so);
- EMPTY_TO_MINSIZE(so);
- PyObject_GC_Track(so);
- } else {
- so = (PySetObject *)type->tp_alloc(type, 0);
- if (so == NULL)
- return NULL;
- /* tp_alloc has already zeroed the structure */
- assert(so->table == NULL && so->fill == 0 && so->used == 0);
- INIT_NONZERO_SET_SLOTS(so);
- }
+ so = (PySetObject *)type->tp_alloc(type, 0);
+ if (so == NULL)
+ return NULL;
+ so->fill = 0;
+ so->used = 0;
+ so->mask = PySet_MINSIZE - 1;
+ so->table = so->smalltable;
so->lookup = set_lookkey_unicode;
+ so->hash = -1;
so->weakreflist = NULL;
if (iterable != NULL) {
@@ -1098,34 +1084,15 @@
int
PySet_ClearFreeList(void)
{
- int freelist_size = numfree;
- PySetObject *so;
-
- while (numfree) {
- numfree--;
- so = free_list[numfree];
- PyObject_GC_Del(so);
- }
- return freelist_size;
+ return 0;
}
void
PySet_Fini(void)
{
- PySet_ClearFreeList();
Py_CLEAR(emptyfrozenset);
}
-/* Print summary info about the state of the optimized allocator */
-void
-_PySet_DebugMallocStats(FILE *out)
-{
- _PyDebugAllocatorStats(out,
- "free PySetObject",
- numfree, sizeof(PySetObject));
-}
-
-
static PyObject *
set_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
@@ -2398,7 +2365,7 @@
Py_ssize_t count;
char *s;
Py_ssize_t i;
- PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x;
+ PyObject *elem=NULL, *dup=NULL, *t, *f, *dup2, *x=NULL;
PyObject *ob = (PyObject *)so;
Py_hash_t hash;
PyObject *str;
diff --git a/Python/pystate.c b/Python/pystate.c
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -208,6 +208,8 @@
tstate->trash_delete_nesting = 0;
tstate->trash_delete_later = NULL;
+ tstate->on_delete = NULL;
+ tstate->on_delete_data = NULL;
if (init)
_PyThreadState_Init(tstate);
@@ -390,6 +392,9 @@
if (tstate->next)
tstate->next->prev = tstate->prev;
HEAD_UNLOCK();
+ if (tstate->on_delete != NULL) {
+ tstate->on_delete(tstate->on_delete_data);
+ }
PyMem_RawFree(tstate);
}
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list