[Python-checkins] bpo-46072: Add some object layout and allocation stats (GH-31051)
markshannon
webhook-mailer at python.org
Tue Feb 1 10:05:40 EST 2022
https://github.com/python/cpython/commit/48be46ec1f3f8010570165daa1da4bf9961f3a83
commit: 48be46ec1f3f8010570165daa1da4bf9961f3a83
branch: main
author: Mark Shannon <mark at hotpy.org>
committer: markshannon <mark at hotpy.org>
date: 2022-02-01T15:05:18Z
summary:
bpo-46072: Add some object layout and allocation stats (GH-31051)
files:
M Include/internal/pycore_code.h
M Objects/dictobject.c
M Objects/obmalloc.c
M Python/specialize.c
M Tools/scripts/summarize_stats.py
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 68b536f75ca5e..45c7752112b46 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -305,9 +305,19 @@ typedef struct _call_stats {
uint64_t pyeval_calls;
} CallStats;
+typedef struct _object_stats {
+ uint64_t allocations;
+ uint64_t frees;
+ uint64_t new_values;
+ uint64_t dict_materialized_on_request;
+ uint64_t dict_materialized_new_key;
+ uint64_t dict_materialized_too_big;
+} ObjectStats;
+
typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
+ ObjectStats object_stats;
} PyStats;
extern PyStats _py_stats;
@@ -316,6 +326,7 @@ extern PyStats _py_stats;
#define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
#define OPCODE_EXE_INC(opname) _py_stats.opcode_stats[opname].execution_count++
#define CALL_STAT_INC(name) _py_stats.call_stats.name++
+#define OBJECT_STAT_INC(name) _py_stats.object_stats.name++
void _Py_PrintSpecializationStats(int to_file);
@@ -326,6 +337,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
#define STAT_DEC(opname, name) ((void)0)
#define OPCODE_EXE_INC(opname) ((void)0)
#define CALL_STAT_INC(name) ((void)0)
+#define OBJECT_STAT_INC(name) ((void)0)
#endif
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index 39be189e12000..0ad0f0b59c87e 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -114,6 +114,7 @@ As a consequence of this, split keys have a maximum size of 16.
#include "Python.h"
#include "pycore_bitutils.h" // _Py_bit_length
#include "pycore_call.h" // _PyObject_CallNoArgs()
+#include "pycore_code.h" // stats
#include "pycore_dict.h" // PyDictKeysObject
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
#include "pycore_object.h" // _PyObject_GC_TRACK()
@@ -4990,6 +4991,7 @@ _PyObject_InitializeDict(PyObject *obj)
return 0;
}
if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+ OBJECT_STAT_INC(new_values);
return init_inline_values(obj, tp);
}
PyObject *dict;
@@ -5033,6 +5035,7 @@ _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
{
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
+ OBJECT_STAT_INC(dict_materialized_on_request);
return make_dict_from_instance_attributes(keys, values);
}
@@ -5051,6 +5054,14 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
PyErr_SetObject(PyExc_AttributeError, name);
return -1;
}
+#ifdef Py_STATS
+ if (shared_keys_usable_size(keys) > 14) {
+ OBJECT_STAT_INC(dict_materialized_too_big);
+ }
+ else {
+ OBJECT_STAT_INC(dict_materialized_new_key);
+ }
+#endif
PyObject *dict = make_dict_from_instance_attributes(keys, values);
if (dict == NULL) {
return -1;
@@ -5183,6 +5194,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
PyObject **dictptr = _PyObject_ManagedDictPointer(obj);
if (*values_ptr) {
assert(*dictptr == NULL);
+ OBJECT_STAT_INC(dict_materialized_on_request);
*dictptr = dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), *values_ptr);
if (dict != NULL) {
*values_ptr = NULL;
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index e3df7e8cc410e..bad4dc0963921 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1,5 +1,6 @@
#include "Python.h"
#include "pycore_pymem.h" // _PyTraceMalloc_Config
+#include "pycore_code.h" // stats
#include <stdbool.h>
#include <stdlib.h> // malloc()
@@ -695,6 +696,7 @@ PyObject_Malloc(size_t size)
/* see PyMem_RawMalloc() */
if (size > (size_t)PY_SSIZE_T_MAX)
return NULL;
+ OBJECT_STAT_INC(allocations);
return _PyObject.malloc(_PyObject.ctx, size);
}
@@ -704,6 +706,7 @@ PyObject_Calloc(size_t nelem, size_t elsize)
/* see PyMem_RawMalloc() */
if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize)
return NULL;
+ OBJECT_STAT_INC(allocations);
return _PyObject.calloc(_PyObject.ctx, nelem, elsize);
}
@@ -719,6 +722,7 @@ PyObject_Realloc(void *ptr, size_t new_size)
void
PyObject_Free(void *ptr)
{
+ OBJECT_STAT_INC(frees);
_PyObject.free(_PyObject.ctx, ptr);
}
diff --git a/Python/specialize.c b/Python/specialize.c
index aec94d9e60be4..5771a41dcfd2c 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -171,10 +171,22 @@ print_call_stats(FILE *out, CallStats *stats)
fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
}
+static void
+print_object_stats(FILE *out, ObjectStats *stats)
+{
+ fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);
+ fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
+ fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+ fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
+ fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
+ fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
+}
+
static void
print_stats(FILE *out, PyStats *stats) {
print_spec_stats(out, stats->opcode_stats);
print_call_stats(out, &stats->call_stats);
+ print_object_stats(out, &stats->object_stats);
}
void
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index 34cbad5fa7ea2..2761dcb50985b 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -105,7 +105,17 @@ def main():
total += value
for key, value in stats.items():
if "Calls to" in key:
- print(f"{key}: {value} {100*value/total:0.1f}%")
+ print(f" {key}: {value} {100*value/total:0.1f}%")
+ print("Object stats:")
+ total = stats.get("Object new values")
+ for key, value in stats.items():
+ if key.startswith("Object"):
+ if "materialize" in key:
+ print(f" {key}: {value} {100*value/total:0.1f}%")
+ else:
+ print(f" {key}: {value}")
+ total = 0
+
if __name__ == "__main__":
main()
More information about the Python-checkins
mailing list