[Python-checkins] bpo-32043: New "developer mode": "-X dev" option (#4413)
Victor Stinner
webhook-mailer at python.org
Thu Nov 16 06:20:33 EST 2017
https://github.com/python/cpython/commit/ccb0442a338066bf40fe417455e5a374e5238afb
commit: ccb0442a338066bf40fe417455e5a374e5238afb
branch: master
author: Victor Stinner <victor.stinner at gmail.com>
committer: GitHub <noreply at github.com>
date: 2017-11-16T03:20:31-08:00
summary:
bpo-32043: New "developer mode": "-X dev" option (#4413)
Add a new "developer mode": new "-X dev" command line option to
enable debug checks at runtime.
Changes:
* Add unit tests for -X dev
* test_cmd_line: replace test.support with support.
* Fix _PyRuntimeState_Fini(): Use the same memory allocator
than _PyRuntimeState_Init().
* Fix _PyMem_GetDefaultRawAllocator()
files:
A Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst
M Doc/using/cmdline.rst
M Doc/whatsnew/3.7.rst
M Lib/test/test_cmd_line.py
M Modules/main.c
M Objects/obmalloc.c
M Python/pystate.c
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index 9100494fa1a..01869d117f7 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -413,6 +413,17 @@ Miscellaneous options
nested imports). Note that its output may be broken in multi-threaded
application. Typical usage is ``python3 -X importtime -c 'import
asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`.
+ * ``-X dev`` enables the "developer mode": enable debug checks at runtime.
+ In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3
+ -W default -X faulthandler ...``, except that the :envvar:`PYTHONMALLOC`
+ environment variable is not set in practice. Developer mode:
+
+ * Add ``default`` warnings option. For example, display
+ :exc:`DeprecationWarning` and :exc:`ResourceWarning` warnings.
+ * Install debug hooks on memory allocators as if :envvar:`PYTHONMALLOC`
+ is set to ``debug``.
+ * Enable the :mod:`faulthandler` module to dump the Python traceback
+ on a crash.
It also allows passing arbitrary values and retrieving them through the
:data:`sys._xoptions` dictionary.
@@ -430,7 +441,8 @@ Miscellaneous options
The ``-X showalloccount`` option.
.. versionadded:: 3.7
- The ``-X importtime`` and :envvar:`PYTHONPROFILEIMPORTTIME` options.
+ The ``-X importtime``, ``-X dev`` and :envvar:`PYTHONPROFILEIMPORTTIME`
+ options.
Options you shouldn't use
diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
index 9d63540e630..bb75939321d 100644
--- a/Doc/whatsnew/3.7.rst
+++ b/Doc/whatsnew/3.7.rst
@@ -185,6 +185,19 @@ resolution on Linux and Windows.
PEP written and implemented by Victor Stinner
+New Developer Mode: -X dev
+--------------------------
+
+Add a new "developer mode": ``-X dev`` command line option to enable debug
+checks at runtime.
+
+In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W
+default -X faulthandler ...``, except that the PYTHONMALLOC environment
+variable is not set in practice.
+
+See :option:`-X` ``dev`` for the details.
+
+
Other Language Changes
======================
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 1b584ebb7c5..3dbe75f0a09 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -2,14 +2,15 @@
# Most tests are executed with environment variables ignored
# See test_cmd_line_script.py for testing of script execution
-import test.support, unittest
import os
-import sys
import subprocess
+import sys
import tempfile
-from test.support import script_helper, is_android
+import unittest
+from test import support
from test.support.script_helper import (
- spawn_python, kill_python, assert_python_ok, assert_python_failure
+ spawn_python, kill_python, assert_python_ok, assert_python_failure,
+ interpreter_requires_environment
)
# XXX (ncoghlan): Move to script_helper and make consistent with run_python
@@ -132,11 +133,11 @@ def test_run_code(self):
# All good if execution is successful
assert_python_ok('-c', 'pass')
- @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII')
+ @unittest.skipUnless(support.FS_NONASCII, 'need support.FS_NONASCII')
def test_non_ascii(self):
# Test handling of non-ascii data
command = ("assert(ord(%r) == %s)"
- % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII)))
+ % (support.FS_NONASCII, ord(support.FS_NONASCII)))
assert_python_ok('-c', command)
# On Windows, pass bytes to subprocess doesn't test how Python decodes the
@@ -179,7 +180,7 @@ def test_undecodable_code(self):
raise AssertionError("%a doesn't start with %a" % (stdout, pattern))
@unittest.skipUnless((sys.platform == 'darwin' or
- is_android), 'test specific to Mac OS X and Android')
+ support.is_android), 'test specific to Mac OS X and Android')
def test_osx_android_utf8(self):
def check_output(text):
decoded = text.decode('utf-8', 'surrogateescape')
@@ -385,7 +386,7 @@ def preexec():
stderr=subprocess.PIPE,
preexec_fn=preexec)
out, err = p.communicate()
- self.assertEqual(test.support.strip_python_stderr(err), b'')
+ self.assertEqual(support.strip_python_stderr(err), b'')
self.assertEqual(p.returncode, 42)
def test_no_stdin(self):
@@ -433,8 +434,8 @@ def test_del___main__(self):
# Issue #15001: PyRun_SimpleFileExFlags() did crash because it kept a
# borrowed reference to the dict of __main__ module and later modify
# the dict whereas the module was destroyed
- filename = test.support.TESTFN
- self.addCleanup(test.support.unlink, filename)
+ filename = support.TESTFN
+ self.addCleanup(support.unlink, filename)
with open(filename, "w") as script:
print("import sys", file=script)
print("del sys.modules['__main__']", file=script)
@@ -458,7 +459,7 @@ def test_unknown_options(self):
self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1)
self.assertEqual(b'', out)
- @unittest.skipIf(script_helper.interpreter_requires_environment(),
+ @unittest.skipIf(interpreter_requires_environment(),
'Cannot run -I tests when PYTHON env vars are required.')
def test_isolatedmode(self):
self.verify_valid_flag('-I')
@@ -469,7 +470,7 @@ def test_isolatedmode(self):
# dummyvar to prevent extraneous -E
dummyvar="")
self.assertEqual(out.strip(), b'1 1 1')
- with test.support.temp_cwd() as tmpdir:
+ with support.temp_cwd() as tmpdir:
fake = os.path.join(tmpdir, "uuid.py")
main = os.path.join(tmpdir, "main.py")
with open(fake, "w") as f:
@@ -506,6 +507,46 @@ def test_sys_flags_set(self):
with self.subTest(envar_value=value):
assert_python_ok('-c', code, **env_vars)
+ def run_xdev(self, code, check_exitcode=True):
+ env = dict(os.environ)
+ env.pop('PYTHONWARNINGS', None)
+ # Force malloc() to disable the debug hooks which are enabled
+ # by default for Python compiled in debug mode
+ env['PYTHONMALLOC'] = 'malloc'
+
+ args = (sys.executable, '-X', 'dev', '-c', code)
+ proc = subprocess.run(args,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=True,
+ env=env)
+ if check_exitcode:
+ self.assertEqual(proc.returncode, 0, proc)
+ return proc.stdout.rstrip()
+
+ def test_xdev(self):
+ out = self.run_xdev("import sys; print(sys.warnoptions)")
+ self.assertEqual(out, "['default']")
+
+ try:
+ import _testcapi
+ except ImportError:
+ pass
+ else:
+ code = "import _testcapi; _testcapi.pymem_api_misuse()"
+ with support.SuppressCrashReport():
+ out = self.run_xdev(code, check_exitcode=False)
+ self.assertIn("Debug memory block at address p=", out)
+
+ try:
+ import faulthandler
+ except ImportError:
+ pass
+ else:
+ code = "import faulthandler; print(faulthandler.is_enabled())"
+ out = self.run_xdev(code)
+ self.assertEqual(out, "True")
+
class IgnoreEnvironmentTest(unittest.TestCase):
def run_ignoring_vars(self, predicate, **env_vars):
@@ -541,8 +582,8 @@ def test_sys_flags_not_set(self):
def test_main():
- test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest)
- test.support.reap_children()
+ support.run_unittest(CmdLineTest, IgnoreEnvironmentTest)
+ support.reap_children()
if __name__ == "__main__":
test_main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst
new file mode 100644
index 00000000000..21d59e0d61e
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst
@@ -0,0 +1,2 @@
+Add a new "developer mode": new "-X dev" command line option to enable debug
+checks at runtime.
diff --git a/Modules/main.c b/Modules/main.c
index 95076c642d3..203abf414cc 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -1396,6 +1396,16 @@ pymain_parse_envvars(_PyMain *pymain)
if (pymain_init_tracemalloc(pymain) < 0) {
return -1;
}
+ if (pymain_get_xoption(pymain, L"dev")) {
+ /* "python3 -X dev ..." behaves
+ as "PYTHONMALLOC=debug python3 -Wd -X faulthandler ..." */
+ core_config->allocator = "debug";
+ if (pymain_optlist_append(pymain, &pymain->cmdline.warning_options,
+ L"default") < 0) {
+ return -1;
+ }
+ core_config->faulthandler = 1;
+ }
return 0;
}
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 699cce94d2c..7c6973ec035 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -190,8 +190,14 @@ static struct {
void
_PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p)
{
- PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS};
- *alloc_p = alloc;
+ PyMemAllocatorEx pymem_raw = {
+#ifdef Py_DEBUG
+ &_PyMem_Debug.raw, PYRAWDBG_FUNCS
+#else
+ NULL, PYRAW_FUNCS
+#endif
+ };
+ *alloc_p = pymem_raw;
}
int
@@ -274,13 +280,6 @@ _PyObject_Initialize(struct _pyobj_runtime_state *state)
void
_PyMem_Initialize(struct _pymem_runtime_state *state)
{
- PyMemAllocatorEx pymem_raw = {
-#ifdef Py_DEBUG
- &_PyMem_Debug.raw, PYRAWDBG_FUNCS
-#else
- NULL, PYRAW_FUNCS
-#endif
- };
PyMemAllocatorEx pymem = {
#ifdef Py_DEBUG
&_PyMem_Debug.mem, PYDBG_FUNCS
@@ -296,7 +295,7 @@ _PyMem_Initialize(struct _pymem_runtime_state *state)
#endif
};
- state->allocators.raw = pymem_raw;
+ _PyMem_GetDefaultRawAllocator(&state->allocators.raw);
state->allocators.mem = pymem;
state->allocators.obj = pyobject;
diff --git a/Python/pystate.c b/Python/pystate.c
index 4544de9986f..807ac4eb9d1 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -64,10 +64,18 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
void
_PyRuntimeState_Fini(_PyRuntimeState *runtime)
{
+ /* Use the same memory allocator than _PyRuntimeState_Init() */
+ PyMemAllocatorEx old_alloc, raw_alloc;
+ PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+ _PyMem_GetDefaultRawAllocator(&raw_alloc);
+ PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &raw_alloc);
+
if (runtime->interpreters.mutex != NULL) {
PyThread_free_lock(runtime->interpreters.mutex);
runtime->interpreters.mutex = NULL;
}
+
+ PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
}
#define HEAD_LOCK() PyThread_acquire_lock(_PyRuntime.interpreters.mutex, \
More information about the Python-checkins
mailing list