[Python-checkins] bpo-36611: Disable serialno field of debug memory allocators (#12796)
Victor Stinner
webhook-mailer at python.org
Fri Apr 12 15:54:10 EDT 2019
https://github.com/python/cpython/commit/e8f9acf03484c6c3f163f04a76321419369c28aa
commit: e8f9acf03484c6c3f163f04a76321419369c28aa
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2019-04-12T21:54:06+02:00
summary:
bpo-36611: Disable serialno field of debug memory allocators (#12796)
Omit serialno field from debug hooks on Python memory allocators to
reduce the memory footprint by 5%.
Enable tracemalloc to get the traceback where a memory block has been
allocated when a fatal memory error is logged to decide where to put
a breakpoint.
Compile Python with PYMEM_DEBUG_SERIALNO defined to get back the
field.
files:
A Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst
M Lib/test/test_capi.py
M Objects/obmalloc.c
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 33c98ac28bc5..31dab6a423e2 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -483,7 +483,7 @@ def test_buffer_overflow(self):
r" at tail\+1: 0xfd\n"
r" at tail\+2: 0xfd\n"
r" .*\n"
- r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
+ r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
@@ -499,7 +499,7 @@ def test_api_misuse(self):
r" 16 bytes originally requested\n"
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
- r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
+ r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst
new file mode 100644
index 000000000000..f55a9efc5d38
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst
@@ -0,0 +1,5 @@
+Debug memory allocators: disable serialno field by default from debug hooks on
+Python memory allocators to reduce the memory footprint by 5%. Enable
+:mod:`tracemalloc` to get the traceback where a memory block has been allocated
+when a fatal memory error is logged to decide where to put a breakpoint.
+Compile Python with ``PYMEM_DEBUG_SERIALNO`` defined to get back the field.
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index be43c7a1c2b8..3ee143549d21 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1926,6 +1926,10 @@ _Py_GetAllocatedBlocks(void)
#define DEADBYTE 0xDD /* dead (newly freed) memory */
#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */
+/* Uncomment this define to add the "serialno" field */
+/* #define PYMEM_DEBUG_SERIALNO */
+
+#ifdef PYMEM_DEBUG_SERIALNO
static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
/* serialno is always incremented via calling this routine. The point is
@@ -1936,9 +1940,16 @@ bumpserialno(void)
{
++serialno;
}
+#endif
#define SST SIZEOF_SIZE_T
+#ifdef PYMEM_DEBUG_SERIALNO
+# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST
+#else
+# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST
+#endif
+
/* Read sizeof(size_t) bytes at p as a big-endian size_t. */
static size_t
read_size_t(const void *p)
@@ -1967,7 +1978,7 @@ write_size_t(void *p, size_t n)
}
}
-/* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and
+/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and
fills them with useful stuff, here calling the underlying malloc's result p:
p[0: S]
@@ -1991,6 +2002,9 @@ p[2*S+n+S: 2*S+n+2*S]
If "bad memory" is detected later, the serial number gives an
excellent way to set a breakpoint on the next run, to capture the
instant at which this block was passed out.
+
+If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks
+for 3 * S extra bytes, and omits the last serialno field.
*/
static void *
@@ -2000,21 +2014,24 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
uint8_t *p; /* base address of malloc'ed pad block */
uint8_t *data; /* p + 2*SST == pointer to data bytes */
uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
- size_t total; /* 2 * SST + nbytes + 2 * SST */
+ size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */
- if (nbytes > (size_t)PY_SSIZE_T_MAX - 4 * SST) {
+ if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
/* integer overflow: can't represent total as a Py_ssize_t */
return NULL;
}
- total = nbytes + 4 * SST;
+ total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
/* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN]
- * ^--- p ^--- data ^--- tail
+ ^--- p ^--- data ^--- tail
S: nbytes stored as size_t
I: API identifier (1 byte)
F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after)
C: Clean bytes used later to store actual data
- N: Serial number stored as size_t */
+ N: Serial number stored as size_t
+
+ If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field
+ is omitted. */
if (use_calloc) {
p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total);
@@ -2027,7 +2044,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
}
data = p + 2*SST;
+#ifdef PYMEM_DEBUG_SERIALNO
bumpserialno();
+#endif
/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes);
@@ -2041,7 +2060,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
/* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
+#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, serialno);
+#endif
return data;
}
@@ -2081,7 +2102,7 @@ _PyMem_DebugRawFree(void *ctx, void *p)
_PyMem_DebugCheckAddress(api->api_id, p);
nbytes = read_size_t(q);
- nbytes += 4 * SST;
+ nbytes += PYMEM_DEBUG_EXTRA_BYTES;
memset(q, DEADBYTE, nbytes);
api->alloc.free(api->alloc.ctx, q);
}
@@ -2101,7 +2122,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
size_t total; /* 2 * SST + nbytes + 2 * SST */
size_t original_nbytes;
- size_t block_serialno;
#define ERASED_SIZE 64
uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */
@@ -2110,47 +2130,57 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
data = (uint8_t *)p;
head = data - 2*SST;
original_nbytes = read_size_t(head);
- if (nbytes > (size_t)PY_SSIZE_T_MAX - 4*SST) {
+ if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
/* integer overflow: can't represent total as a Py_ssize_t */
return NULL;
}
- total = nbytes + 4*SST;
+ total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
tail = data + original_nbytes;
- block_serialno = read_size_t(tail + SST);
+#ifdef PYMEM_DEBUG_SERIALNO
+ size_t block_serialno = read_size_t(tail + SST);
+#endif
/* Mark the header, the trailer, ERASED_SIZE bytes at the begin and
ERASED_SIZE bytes at the end as dead and save the copy of erased bytes.
*/
if (original_nbytes <= sizeof(save)) {
memcpy(save, data, original_nbytes);
- memset(data - 2*SST, DEADBYTE, original_nbytes + 4*SST);
+ memset(data - 2 * SST, DEADBYTE,
+ original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
}
else {
memcpy(save, data, ERASED_SIZE);
- memset(head, DEADBYTE, ERASED_SIZE + 2*SST);
+ memset(head, DEADBYTE, ERASED_SIZE + 2 * SST);
memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
- memset(tail - ERASED_SIZE, DEADBYTE, ERASED_SIZE + 2*SST);
+ memset(tail - ERASED_SIZE, DEADBYTE,
+ ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
}
/* Resize and add decorations. */
r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total);
if (r == NULL) {
+ /* if realloc() failed: rewrite header and footer which have
+ just been erased */
nbytes = original_nbytes;
}
else {
head = r;
+#ifdef PYMEM_DEBUG_SERIALNO
bumpserialno();
block_serialno = serialno;
+#endif
}
+ data = head + 2*SST;
write_size_t(head, nbytes);
head[SST] = (uint8_t)api->api_id;
memset(head + SST + 1, FORBIDDENBYTE, SST-1);
- data = head + 2*SST;
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
+#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, block_serialno);
+#endif
/* Restore saved bytes. */
if (original_nbytes <= sizeof(save)) {
@@ -2170,7 +2200,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
}
if (nbytes > original_nbytes) {
- /* growing: mark new extra memory clean */
+ /* growing: mark new extra memory clean */
memset(data + original_nbytes, CLEANBYTE, nbytes - original_nbytes);
}
@@ -2278,7 +2308,7 @@ _PyObject_DebugDumpAddress(const void *p)
{
const uint8_t *q = (const uint8_t *)p;
const uint8_t *tail;
- size_t nbytes, serial;
+ size_t nbytes;
int i;
int ok;
char id;
@@ -2347,9 +2377,11 @@ _PyObject_DebugDumpAddress(const void *p)
}
}
- serial = read_size_t(tail + SST);
+#ifdef PYMEM_DEBUG_SERIALNO
+ size_t serial = read_size_t(tail + SST);
fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T
"u to debug malloc/realloc.\n", serial);
+#endif
if (nbytes > 0) {
i = 0;
@@ -2575,8 +2607,11 @@ _PyObject_DebugMallocStats(FILE *out)
quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
}
fputc('\n', out);
- if (_PyMem_DebugEnabled())
+#ifdef PYMEM_DEBUG_SERIALNO
+ if (_PyMem_DebugEnabled()) {
(void)printone(out, "# times object malloc called", serialno);
+ }
+#endif
(void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
(void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
(void)printone(out, "# arenas highwater mark", narenas_highwater);
More information about the Python-checkins
mailing list