[Python-checkins] gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551)
erlend-aasland
webhook-mailer at python.org
Sat Dec 17 03:53:42 EST 2022
https://github.com/python/cpython/commit/2b38a9aa747785f6d540e95e6846d6510de6b306
commit: 2b38a9aa747785f6d540e95e6846d6510de6b306
branch: main
author: Erlend E. Aasland <erlend.aasland at protonmail.com>
committer: erlend-aasland <erlend.aasland at protonmail.com>
date: 2022-12-17T09:53:36+01:00
summary:
gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551)
files:
A Lib/test/test_capi/test_mem.py
M Lib/test/test_capi/test_misc.py
M Modules/_testcapi/mem.c
M Modules/_testcapimodule.c
diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py
new file mode 100644
index 000000000000..a9ff410cb93a
--- /dev/null
+++ b/Lib/test/test_capi/test_mem.py
@@ -0,0 +1,169 @@
+import re
+import textwrap
+import unittest
+
+
+from test import support
+from test.support import import_helper, requires_subprocess
+from test.support.script_helper import assert_python_failure, assert_python_ok
+
+
+# Skip this test if the _testcapi module isn't available.
+_testcapi = import_helper.import_module('_testcapi')
+
+ at requires_subprocess()
+class PyMemDebugTests(unittest.TestCase):
+ PYTHONMALLOC = 'debug'
+ # '0x04c06e0' or '04C06E0'
+ PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+'
+
+ def check(self, code):
+ with support.SuppressCrashReport():
+ out = assert_python_failure(
+ '-c', code,
+ PYTHONMALLOC=self.PYTHONMALLOC,
+ # FreeBSD: instruct jemalloc to not fill freed() memory
+ # with junk byte 0x5a, see JEMALLOC(3)
+ MALLOC_CONF="junk:false",
+ )
+ stderr = out.err
+ return stderr.decode('ascii', 'replace')
+
+ def test_buffer_overflow(self):
+ out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()')
+ regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
+ r" 16 bytes originally requested\n"
+ r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
+ r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
+ r" at tail\+0: 0x78 \*\*\* OUCH\n"
+ r" at tail\+1: 0xfd\n"
+ r" at tail\+2: 0xfd\n"
+ r" .*\n"
+ r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
+ r" Data at p: cd cd cd .*\n"
+ r"\n"
+ r"Enable tracemalloc to get the memory block allocation traceback\n"
+ r"\n"
+ r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte")
+ regex = regex.format(ptr=self.PTR_REGEX)
+ regex = re.compile(regex, flags=re.DOTALL)
+ self.assertRegex(out, regex)
+
+ def test_api_misuse(self):
+ out = self.check('import _testcapi; _testcapi.pymem_api_misuse()')
+ regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
+ r" 16 bytes originally requested\n"
+ r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
+ r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
+ r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
+ r" Data at p: cd cd cd .*\n"
+ r"\n"
+ r"Enable tracemalloc to get the memory block allocation traceback\n"
+ r"\n"
+ r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n")
+ regex = regex.format(ptr=self.PTR_REGEX)
+ self.assertRegex(out, regex)
+
+ def check_malloc_without_gil(self, code):
+ out = self.check(code)
+ expected = ('Fatal Python error: _PyMem_DebugMalloc: '
+ 'Python memory allocator called without holding the GIL')
+ self.assertIn(expected, out)
+
+ def test_pymem_malloc_without_gil(self):
+ # Debug hooks must raise an error if PyMem_Malloc() is called
+ # without holding the GIL
+ code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
+ self.check_malloc_without_gil(code)
+
+ def test_pyobject_malloc_without_gil(self):
+ # Debug hooks must raise an error if PyObject_Malloc() is called
+ # without holding the GIL
+ code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
+ self.check_malloc_without_gil(code)
+
+ def check_pyobject_is_freed(self, func_name):
+ code = textwrap.dedent(f'''
+ import gc, os, sys, _testcapi
+ # Disable the GC to avoid crash on GC collection
+ gc.disable()
+ try:
+ _testcapi.{func_name}()
+ # Exit immediately to avoid a crash while deallocating
+ # the invalid object
+ os._exit(0)
+ except _testcapi.error:
+ os._exit(1)
+ ''')
+ assert_python_ok(
+ '-c', code,
+ PYTHONMALLOC=self.PYTHONMALLOC,
+ MALLOC_CONF="junk:false",
+ )
+
+ def test_pyobject_null_is_freed(self):
+ self.check_pyobject_is_freed('check_pyobject_null_is_freed')
+
+ def test_pyobject_uninitialized_is_freed(self):
+ self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
+
+ def test_pyobject_forbidden_bytes_is_freed(self):
+ self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
+
+ def test_pyobject_freed_is_freed(self):
+ self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
+
+ def test_set_nomemory(self):
+ code = """if 1:
+ import _testcapi
+
+ class C(): pass
+
+ # The first loop tests both functions and that remove_mem_hooks()
+ # can be called twice in a row. The second loop checks a call to
+ # set_nomemory() after a call to remove_mem_hooks(). The third
+ # loop checks the start and stop arguments of set_nomemory().
+ for outer_cnt in range(1, 4):
+ start = 10 * outer_cnt
+ for j in range(100):
+ if j == 0:
+ if outer_cnt != 3:
+ _testcapi.set_nomemory(start)
+ else:
+ _testcapi.set_nomemory(start, start + 1)
+ try:
+ C()
+ except MemoryError as e:
+ if outer_cnt != 3:
+ _testcapi.remove_mem_hooks()
+ print('MemoryError', outer_cnt, j)
+ _testcapi.remove_mem_hooks()
+ break
+ """
+ rc, out, err = assert_python_ok('-c', code)
+ lines = out.splitlines()
+ for i, line in enumerate(lines, 1):
+ self.assertIn(b'MemoryError', out)
+ *_, count = line.split(b' ')
+ count = int(count)
+ self.assertLessEqual(count, i*5)
+ self.assertGreaterEqual(count, i*5-2)
+
+
+class PyMemMallocDebugTests(PyMemDebugTests):
+ PYTHONMALLOC = 'malloc_debug'
+
+
+ at unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
+class PyMemPymallocDebugTests(PyMemDebugTests):
+ PYTHONMALLOC = 'pymalloc_debug'
+
+
+ at unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
+class PyMemDefaultTests(PyMemDebugTests):
+ # test default allocator of Python compiled in debug mode
+ PYTHONMALLOC = ''
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index 06a51aa3cc21..dace37c362e5 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -323,42 +323,6 @@ def test_getitem_with_error(self):
def test_buildvalue_N(self):
_testcapi.test_buildvalue_N()
- def test_set_nomemory(self):
- code = """if 1:
- import _testcapi
-
- class C(): pass
-
- # The first loop tests both functions and that remove_mem_hooks()
- # can be called twice in a row. The second loop checks a call to
- # set_nomemory() after a call to remove_mem_hooks(). The third
- # loop checks the start and stop arguments of set_nomemory().
- for outer_cnt in range(1, 4):
- start = 10 * outer_cnt
- for j in range(100):
- if j == 0:
- if outer_cnt != 3:
- _testcapi.set_nomemory(start)
- else:
- _testcapi.set_nomemory(start, start + 1)
- try:
- C()
- except MemoryError as e:
- if outer_cnt != 3:
- _testcapi.remove_mem_hooks()
- print('MemoryError', outer_cnt, j)
- _testcapi.remove_mem_hooks()
- break
- """
- rc, out, err = assert_python_ok('-c', code)
- lines = out.splitlines()
- for i, line in enumerate(lines, 1):
- self.assertIn(b'MemoryError', out)
- *_, count = line.split(b' ')
- count = int(count)
- self.assertLessEqual(count, i*5)
- self.assertGreaterEqual(count, i*5-2)
-
def test_mapping_keys_values_items(self):
class Mapping1(dict):
def keys(self):
@@ -1470,124 +1434,6 @@ class Test_testinternalcapi(unittest.TestCase):
if name.startswith('test_'))
- at support.requires_subprocess()
-class PyMemDebugTests(unittest.TestCase):
- PYTHONMALLOC = 'debug'
- # '0x04c06e0' or '04C06E0'
- PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+'
-
- def check(self, code):
- with support.SuppressCrashReport():
- out = assert_python_failure(
- '-c', code,
- PYTHONMALLOC=self.PYTHONMALLOC,
- # FreeBSD: instruct jemalloc to not fill freed() memory
- # with junk byte 0x5a, see JEMALLOC(3)
- MALLOC_CONF="junk:false",
- )
- stderr = out.err
- return stderr.decode('ascii', 'replace')
-
- def test_buffer_overflow(self):
- out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()')
- regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
- r" 16 bytes originally requested\n"
- r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
- r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
- r" at tail\+0: 0x78 \*\*\* OUCH\n"
- r" at tail\+1: 0xfd\n"
- r" at tail\+2: 0xfd\n"
- r" .*\n"
- r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
- r" Data at p: cd cd cd .*\n"
- r"\n"
- r"Enable tracemalloc to get the memory block allocation traceback\n"
- r"\n"
- r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte")
- regex = regex.format(ptr=self.PTR_REGEX)
- regex = re.compile(regex, flags=re.DOTALL)
- self.assertRegex(out, regex)
-
- def test_api_misuse(self):
- out = self.check('import _testcapi; _testcapi.pymem_api_misuse()')
- regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
- r" 16 bytes originally requested\n"
- r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
- r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
- r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
- r" Data at p: cd cd cd .*\n"
- r"\n"
- r"Enable tracemalloc to get the memory block allocation traceback\n"
- r"\n"
- r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n")
- regex = regex.format(ptr=self.PTR_REGEX)
- self.assertRegex(out, regex)
-
- def check_malloc_without_gil(self, code):
- out = self.check(code)
- expected = ('Fatal Python error: _PyMem_DebugMalloc: '
- 'Python memory allocator called without holding the GIL')
- self.assertIn(expected, out)
-
- def test_pymem_malloc_without_gil(self):
- # Debug hooks must raise an error if PyMem_Malloc() is called
- # without holding the GIL
- code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
- self.check_malloc_without_gil(code)
-
- def test_pyobject_malloc_without_gil(self):
- # Debug hooks must raise an error if PyObject_Malloc() is called
- # without holding the GIL
- code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
- self.check_malloc_without_gil(code)
-
- def check_pyobject_is_freed(self, func_name):
- code = textwrap.dedent(f'''
- import gc, os, sys, _testcapi
- # Disable the GC to avoid crash on GC collection
- gc.disable()
- try:
- _testcapi.{func_name}()
- # Exit immediately to avoid a crash while deallocating
- # the invalid object
- os._exit(0)
- except _testcapi.error:
- os._exit(1)
- ''')
- assert_python_ok(
- '-c', code,
- PYTHONMALLOC=self.PYTHONMALLOC,
- MALLOC_CONF="junk:false",
- )
-
- def test_pyobject_null_is_freed(self):
- self.check_pyobject_is_freed('check_pyobject_null_is_freed')
-
- def test_pyobject_uninitialized_is_freed(self):
- self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
-
- def test_pyobject_forbidden_bytes_is_freed(self):
- self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
-
- def test_pyobject_freed_is_freed(self):
- self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
-
-
-class PyMemMallocDebugTests(PyMemDebugTests):
- PYTHONMALLOC = 'malloc_debug'
-
-
- at unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
-class PyMemPymallocDebugTests(PyMemDebugTests):
- PYTHONMALLOC = 'pymalloc_debug'
-
-
- at unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
-class PyMemDefaultTests(PyMemDebugTests):
- # test default allocator of Python compiled in debug mode
- PYTHONMALLOC = ''
-
-
@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
class Test_ModuleStateAccess(unittest.TestCase):
"""Test access to module start (PEP 573)"""
diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c
index ef723bf06585..ae3f7a4372dc 100644
--- a/Modules/_testcapi/mem.c
+++ b/Modules/_testcapi/mem.c
@@ -596,6 +596,82 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
#endif
}
+// Tracemalloc tests
+static PyObject *
+tracemalloc_track(PyObject *self, PyObject *args)
+{
+ unsigned int domain;
+ PyObject *ptr_obj;
+ Py_ssize_t size;
+ int release_gil = 0;
+
+ if (!PyArg_ParseTuple(args, "IOn|i",
+ &domain, &ptr_obj, &size, &release_gil))
+ {
+ return NULL;
+ }
+ void *ptr = PyLong_AsVoidPtr(ptr_obj);
+ if (PyErr_Occurred()) {
+ return NULL;
+ }
+
+ int res;
+ if (release_gil) {
+ Py_BEGIN_ALLOW_THREADS
+ res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
+ Py_END_ALLOW_THREADS
+ }
+ else {
+ res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
+ }
+ if (res < 0) {
+ PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error");
+ return NULL;
+ }
+
+ Py_RETURN_NONE;
+}
+
+static PyObject *
+tracemalloc_untrack(PyObject *self, PyObject *args)
+{
+ unsigned int domain;
+ PyObject *ptr_obj;
+
+ if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) {
+ return NULL;
+ }
+ void *ptr = PyLong_AsVoidPtr(ptr_obj);
+ if (PyErr_Occurred()) {
+ return NULL;
+ }
+
+ int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr);
+ if (res < 0) {
+ PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error");
+ return NULL;
+ }
+
+ Py_RETURN_NONE;
+}
+
+static PyObject *
+tracemalloc_get_traceback(PyObject *self, PyObject *args)
+{
+ unsigned int domain;
+ PyObject *ptr_obj;
+
+ if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) {
+ return NULL;
+ }
+ void *ptr = PyLong_AsVoidPtr(ptr_obj);
+ if (PyErr_Occurred()) {
+ return NULL;
+ }
+
+ return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr);
+}
+
static PyMethodDef test_methods[] = {
{"check_pyobject_forbidden_bytes_is_freed",
check_pyobject_forbidden_bytes_is_freed, METH_NOARGS},
@@ -617,6 +693,11 @@ static PyMethodDef test_methods[] = {
{"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS},
{"test_pyobject_new", test_pyobject_new, METH_NOARGS},
{"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS},
+
+ // Tracemalloc tests
+ {"tracemalloc_track", tracemalloc_track, METH_VARARGS},
+ {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
+ {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
{NULL},
};
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 83eef73a875d..35c895d9ceb2 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -2065,78 +2065,6 @@ getitem_with_error(PyObject *self, PyObject *args)
return PyObject_GetItem(map, key);
}
-static PyObject *
-tracemalloc_track(PyObject *self, PyObject *args)
-{
- unsigned int domain;
- PyObject *ptr_obj;
- void *ptr;
- Py_ssize_t size;
- int release_gil = 0;
- int res;
-
- if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil))
- return NULL;
- ptr = PyLong_AsVoidPtr(ptr_obj);
- if (PyErr_Occurred())
- return NULL;
-
- if (release_gil) {
- Py_BEGIN_ALLOW_THREADS
- res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
- Py_END_ALLOW_THREADS
- }
- else {
- res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
- }
-
- if (res < 0) {
- PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error");
- return NULL;
- }
-
- Py_RETURN_NONE;
-}
-
-static PyObject *
-tracemalloc_untrack(PyObject *self, PyObject *args)
-{
- unsigned int domain;
- PyObject *ptr_obj;
- void *ptr;
- int res;
-
- if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
- return NULL;
- ptr = PyLong_AsVoidPtr(ptr_obj);
- if (PyErr_Occurred())
- return NULL;
-
- res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr);
- if (res < 0) {
- PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error");
- return NULL;
- }
-
- Py_RETURN_NONE;
-}
-
-static PyObject *
-tracemalloc_get_traceback(PyObject *self, PyObject *args)
-{
- unsigned int domain;
- PyObject *ptr_obj;
- void *ptr;
-
- if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
- return NULL;
- ptr = PyLong_AsVoidPtr(ptr_obj);
- if (PyErr_Occurred())
- return NULL;
-
- return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr);
-}
-
static PyObject *
dict_get_version(PyObject *self, PyObject *args)
{
@@ -3301,9 +3229,6 @@ static PyMethodDef TestMethods[] = {
{"return_result_with_error", return_result_with_error, METH_NOARGS},
{"getitem_with_error", getitem_with_error, METH_VARARGS},
{"Py_CompileString", pycompilestring, METH_O},
- {"tracemalloc_track", tracemalloc_track, METH_VARARGS},
- {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
- {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
{"dict_get_version", dict_get_version, METH_VARARGS},
{"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
{"stack_pointer", stack_pointer, METH_NOARGS},
More information about the Python-checkins
mailing list