[Python-checkins] bpo-45367: Specialize BINARY_MULTIPLY (GH-28727)
markshannon
webhook-mailer at python.org
Thu Oct 14 10:56:39 EDT 2021
https://github.com/python/cpython/commit/3b3d30e8f78271a488965c9cd11136e1aa890757
commit: 3b3d30e8f78271a488965c9cd11136e1aa890757
branch: main
author: Dennis Sweeney <36520290+sweeneyde at users.noreply.github.com>
committer: markshannon <mark at hotpy.org>
date: 2021-10-14T15:56:33+01:00
summary:
bpo-45367: Specialize BINARY_MULTIPLY (GH-28727)
files:
A Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst
M Include/internal/pycore_code.h
M Include/internal/pycore_long.h
M Include/opcode.h
M Lib/opcode.py
M Objects/longobject.c
M Python/ceval.c
M Python/opcode_targets.h
M Python/specialize.c
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 0b127ed28993b..a91209b39f654 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -307,7 +307,8 @@ int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *nam
int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
-int _Py_Specialize_BinaryAdd(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr);
+int _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
+int _Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr);
#define PRINT_SPECIALIZATION_STATS 0
#define PRINT_SPECIALIZATION_STATS_DETAILED 0
diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h
index 7336c317c3f2c..8edc90218547f 100644
--- a/Include/internal/pycore_long.h
+++ b/Include/internal/pycore_long.h
@@ -35,6 +35,7 @@ static inline PyObject* _PyLong_GetOne(void)
{ return __PyLong_GetSmallInt_internal(1); }
PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right);
+PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right);
#ifdef __cplusplus
}
diff --git a/Include/opcode.h b/Include/opcode.h
index 15f722630dc5d..fabb8d123bb9c 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -141,33 +141,36 @@ extern "C" {
#define BINARY_ADD_FLOAT 13
#define BINARY_ADD_UNICODE 14
#define BINARY_ADD_UNICODE_INPLACE_FAST 18
-#define BINARY_SUBSCR_ADAPTIVE 21
-#define BINARY_SUBSCR_LIST_INT 36
-#define BINARY_SUBSCR_TUPLE_INT 38
-#define BINARY_SUBSCR_DICT 39
-#define JUMP_ABSOLUTE_QUICK 40
-#define LOAD_ATTR_ADAPTIVE 41
-#define LOAD_ATTR_INSTANCE_VALUE 42
-#define LOAD_ATTR_WITH_HINT 43
-#define LOAD_ATTR_SLOT 44
-#define LOAD_ATTR_MODULE 45
-#define LOAD_GLOBAL_ADAPTIVE 46
-#define LOAD_GLOBAL_MODULE 47
-#define LOAD_GLOBAL_BUILTIN 48
-#define LOAD_METHOD_ADAPTIVE 58
-#define LOAD_METHOD_CACHED 80
-#define LOAD_METHOD_CLASS 81
-#define LOAD_METHOD_MODULE 87
-#define LOAD_METHOD_NO_DICT 88
-#define STORE_ATTR_ADAPTIVE 120
-#define STORE_ATTR_INSTANCE_VALUE 122
-#define STORE_ATTR_SLOT 123
-#define STORE_ATTR_WITH_HINT 127
-#define LOAD_FAST__LOAD_FAST 128
-#define STORE_FAST__LOAD_FAST 134
-#define LOAD_FAST__LOAD_CONST 140
-#define LOAD_CONST__LOAD_FAST 143
-#define STORE_FAST__STORE_FAST 149
+#define BINARY_MULTIPLY_ADAPTIVE 21
+#define BINARY_MULTIPLY_INT 36
+#define BINARY_MULTIPLY_FLOAT 38
+#define BINARY_SUBSCR_ADAPTIVE 39
+#define BINARY_SUBSCR_LIST_INT 40
+#define BINARY_SUBSCR_TUPLE_INT 41
+#define BINARY_SUBSCR_DICT 42
+#define JUMP_ABSOLUTE_QUICK 43
+#define LOAD_ATTR_ADAPTIVE 44
+#define LOAD_ATTR_INSTANCE_VALUE 45
+#define LOAD_ATTR_WITH_HINT 46
+#define LOAD_ATTR_SLOT 47
+#define LOAD_ATTR_MODULE 48
+#define LOAD_GLOBAL_ADAPTIVE 58
+#define LOAD_GLOBAL_MODULE 80
+#define LOAD_GLOBAL_BUILTIN 81
+#define LOAD_METHOD_ADAPTIVE 87
+#define LOAD_METHOD_CACHED 88
+#define LOAD_METHOD_CLASS 120
+#define LOAD_METHOD_MODULE 122
+#define LOAD_METHOD_NO_DICT 123
+#define STORE_ATTR_ADAPTIVE 127
+#define STORE_ATTR_INSTANCE_VALUE 128
+#define STORE_ATTR_SLOT 134
+#define STORE_ATTR_WITH_HINT 140
+#define LOAD_FAST__LOAD_FAST 143
+#define STORE_FAST__LOAD_FAST 149
+#define LOAD_FAST__LOAD_CONST 150
+#define LOAD_CONST__LOAD_FAST 151
+#define STORE_FAST__STORE_FAST 153
#define DO_TRACING 255
#ifdef NEED_OPCODE_JUMP_TABLES
static uint32_t _PyOpcode_RelativeJump[8] = {
diff --git a/Lib/opcode.py b/Lib/opcode.py
index efd6aefccc571..9d9f35855e25b 100644
--- a/Lib/opcode.py
+++ b/Lib/opcode.py
@@ -225,6 +225,9 @@ def jabs_op(name, op):
"BINARY_ADD_FLOAT",
"BINARY_ADD_UNICODE",
"BINARY_ADD_UNICODE_INPLACE_FAST",
+ "BINARY_MULTIPLY_ADAPTIVE",
+ "BINARY_MULTIPLY_INT",
+ "BINARY_MULTIPLY_FLOAT",
"BINARY_SUBSCR_ADAPTIVE",
"BINARY_SUBSCR_LIST_INT",
"BINARY_SUBSCR_TUPLE_INT",
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst b/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst
new file mode 100644
index 0000000000000..aae518da1e86f
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-10-05-03-49-07.bpo-45367._astoU.rst
@@ -0,0 +1 @@
+Specialized the ``BINARY_MULTIPLY`` opcode to ``BINARY_MULTIPLY_INT`` and ``BINARY_MULTIPLY_FLOAT`` using the PEP 659 machinery.
\ No newline at end of file
diff --git a/Objects/longobject.c b/Objects/longobject.c
index 111ef4566cda8..66e164974a92c 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -3594,13 +3594,11 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b)
return NULL;
}
-static PyObject *
-long_mul(PyLongObject *a, PyLongObject *b)
+PyObject *
+_PyLong_Multiply(PyLongObject *a, PyLongObject *b)
{
PyLongObject *z;
- CHECK_BINOP(a, b);
-
/* fast path for single-digit multiplication */
if (IS_MEDIUM_VALUE(a) && IS_MEDIUM_VALUE(b)) {
stwodigits v = medium_value(a) * medium_value(b);
@@ -3617,6 +3615,13 @@ long_mul(PyLongObject *a, PyLongObject *b)
return (PyObject *)z;
}
+static PyObject *
+long_mul(PyLongObject *a, PyLongObject *b)
+{
+ CHECK_BINOP(a, b);
+ return _PyLong_Multiply(a, b);
+}
+
/* Fast modulo division for single-digit longs. */
static PyObject *
fast_mod(PyLongObject *a, PyLongObject *b)
diff --git a/Python/ceval.c b/Python/ceval.c
index aef83b9c355fc..de71ae5da0f82 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1933,14 +1933,73 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
}
TARGET(BINARY_MULTIPLY) {
+ PREDICTED(BINARY_MULTIPLY);
+ STAT_INC(BINARY_MULTIPLY, unquickened);
PyObject *right = POP();
PyObject *left = TOP();
PyObject *res = PyNumber_Multiply(left, right);
Py_DECREF(left);
Py_DECREF(right);
SET_TOP(res);
- if (res == NULL)
+ if (res == NULL) {
goto error;
+ }
+ DISPATCH();
+ }
+
+ TARGET(BINARY_MULTIPLY_ADAPTIVE) {
+ if (oparg == 0) {
+ PyObject *left = SECOND();
+ PyObject *right = TOP();
+ next_instr--;
+ if (_Py_Specialize_BinaryMultiply(left, right, next_instr) < 0) {
+ goto error;
+ }
+ DISPATCH();
+ }
+ else {
+ STAT_INC(BINARY_MULTIPLY, deferred);
+ UPDATE_PREV_INSTR_OPARG(next_instr, oparg - 1);
+ STAT_DEC(BINARY_MULTIPLY, unquickened);
+ JUMP_TO_INSTRUCTION(BINARY_MULTIPLY);
+ }
+ }
+
+ TARGET(BINARY_MULTIPLY_INT) {
+ PyObject *left = SECOND();
+ PyObject *right = TOP();
+ DEOPT_IF(!PyLong_CheckExact(left), BINARY_MULTIPLY);
+ DEOPT_IF(!PyLong_CheckExact(right), BINARY_MULTIPLY);
+ STAT_INC(BINARY_MULTIPLY, hit);
+ record_hit_inline(next_instr, oparg);
+ PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
+ SET_SECOND(prod);
+ Py_DECREF(right);
+ Py_DECREF(left);
+ STACK_SHRINK(1);
+ if (prod == NULL) {
+ goto error;
+ }
+ DISPATCH();
+ }
+
+ TARGET(BINARY_MULTIPLY_FLOAT) {
+ PyObject *left = SECOND();
+ PyObject *right = TOP();
+ DEOPT_IF(!PyFloat_CheckExact(left), BINARY_MULTIPLY);
+ DEOPT_IF(!PyFloat_CheckExact(right), BINARY_MULTIPLY);
+ STAT_INC(BINARY_MULTIPLY, hit);
+ record_hit_inline(next_instr, oparg);
+ double dprod = ((PyFloatObject *)left)->ob_fval *
+ ((PyFloatObject *)right)->ob_fval;
+ PyObject *prod = PyFloat_FromDouble(dprod);
+ SET_SECOND(prod);
+ Py_DECREF(right);
+ Py_DECREF(left);
+ STACK_SHRINK(1);
+ if (prod == NULL) {
+ goto error;
+ }
DISPATCH();
}
@@ -4954,6 +5013,7 @@ MISS_WITH_CACHE(LOAD_GLOBAL)
MISS_WITH_CACHE(LOAD_METHOD)
MISS_WITH_OPARG_COUNTER(BINARY_SUBSCR)
MISS_WITH_OPARG_COUNTER(BINARY_ADD)
+MISS_WITH_OPARG_COUNTER(BINARY_MULTIPLY)
binary_subscr_dict_error:
{
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 773f9254cf624..7d701e81a9a48 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -20,7 +20,7 @@ static void *opcode_targets[256] = {
&&TARGET_BINARY_ADD_UNICODE_INPLACE_FAST,
&&TARGET_BINARY_POWER,
&&TARGET_BINARY_MULTIPLY,
- &&TARGET_BINARY_SUBSCR_ADAPTIVE,
+ &&TARGET_BINARY_MULTIPLY_ADAPTIVE,
&&TARGET_BINARY_MODULO,
&&TARGET_BINARY_ADD,
&&TARGET_BINARY_SUBTRACT,
@@ -35,8 +35,11 @@ static void *opcode_targets[256] = {
&&TARGET_MATCH_KEYS,
&&TARGET_COPY_DICT_WITHOUT_KEYS,
&&TARGET_PUSH_EXC_INFO,
- &&TARGET_BINARY_SUBSCR_LIST_INT,
+ &&TARGET_BINARY_MULTIPLY_INT,
&&TARGET_POP_EXCEPT_AND_RERAISE,
+ &&TARGET_BINARY_MULTIPLY_FLOAT,
+ &&TARGET_BINARY_SUBSCR_ADAPTIVE,
+ &&TARGET_BINARY_SUBSCR_LIST_INT,
&&TARGET_BINARY_SUBSCR_TUPLE_INT,
&&TARGET_BINARY_SUBSCR_DICT,
&&TARGET_JUMP_ABSOLUTE_QUICK,
@@ -45,9 +48,6 @@ static void *opcode_targets[256] = {
&&TARGET_LOAD_ATTR_WITH_HINT,
&&TARGET_LOAD_ATTR_SLOT,
&&TARGET_LOAD_ATTR_MODULE,
- &&TARGET_LOAD_GLOBAL_ADAPTIVE,
- &&TARGET_LOAD_GLOBAL_MODULE,
- &&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_WITH_EXCEPT_START,
&&TARGET_GET_AITER,
&&TARGET_GET_ANEXT,
@@ -57,7 +57,7 @@ static void *opcode_targets[256] = {
&&TARGET_INPLACE_ADD,
&&TARGET_INPLACE_SUBTRACT,
&&TARGET_INPLACE_MULTIPLY,
- &&TARGET_LOAD_METHOD_ADAPTIVE,
+ &&TARGET_LOAD_GLOBAL_ADAPTIVE,
&&TARGET_INPLACE_MODULO,
&&TARGET_STORE_SUBSCR,
&&TARGET_DELETE_SUBSCR,
@@ -79,15 +79,15 @@ static void *opcode_targets[256] = {
&&TARGET_INPLACE_AND,
&&TARGET_INPLACE_XOR,
&&TARGET_INPLACE_OR,
- &&TARGET_LOAD_METHOD_CACHED,
- &&TARGET_LOAD_METHOD_CLASS,
+ &&TARGET_LOAD_GLOBAL_MODULE,
+ &&TARGET_LOAD_GLOBAL_BUILTIN,
&&TARGET_LIST_TO_TUPLE,
&&TARGET_RETURN_VALUE,
&&TARGET_IMPORT_STAR,
&&TARGET_SETUP_ANNOTATIONS,
&&TARGET_YIELD_VALUE,
- &&TARGET_LOAD_METHOD_MODULE,
- &&TARGET_LOAD_METHOD_NO_DICT,
+ &&TARGET_LOAD_METHOD_ADAPTIVE,
+ &&TARGET_LOAD_METHOD_CACHED,
&&TARGET_POP_EXCEPT,
&&TARGET_STORE_NAME,
&&TARGET_DELETE_NAME,
@@ -119,40 +119,40 @@ static void *opcode_targets[256] = {
&&TARGET_IS_OP,
&&TARGET_CONTAINS_OP,
&&TARGET_RERAISE,
- &&TARGET_STORE_ATTR_ADAPTIVE,
+ &&TARGET_LOAD_METHOD_CLASS,
&&TARGET_JUMP_IF_NOT_EXC_MATCH,
- &&TARGET_STORE_ATTR_INSTANCE_VALUE,
- &&TARGET_STORE_ATTR_SLOT,
+ &&TARGET_LOAD_METHOD_MODULE,
+ &&TARGET_LOAD_METHOD_NO_DICT,
&&TARGET_LOAD_FAST,
&&TARGET_STORE_FAST,
&&TARGET_DELETE_FAST,
- &&TARGET_STORE_ATTR_WITH_HINT,
- &&TARGET_LOAD_FAST__LOAD_FAST,
+ &&TARGET_STORE_ATTR_ADAPTIVE,
+ &&TARGET_STORE_ATTR_INSTANCE_VALUE,
&&TARGET_GEN_START,
&&TARGET_RAISE_VARARGS,
&&TARGET_CALL_FUNCTION,
&&TARGET_MAKE_FUNCTION,
&&TARGET_BUILD_SLICE,
- &&TARGET_STORE_FAST__LOAD_FAST,
+ &&TARGET_STORE_ATTR_SLOT,
&&TARGET_MAKE_CELL,
&&TARGET_LOAD_CLOSURE,
&&TARGET_LOAD_DEREF,
&&TARGET_STORE_DEREF,
&&TARGET_DELETE_DEREF,
- &&TARGET_LOAD_FAST__LOAD_CONST,
+ &&TARGET_STORE_ATTR_WITH_HINT,
&&TARGET_CALL_FUNCTION_KW,
&&TARGET_CALL_FUNCTION_EX,
- &&TARGET_LOAD_CONST__LOAD_FAST,
+ &&TARGET_LOAD_FAST__LOAD_FAST,
&&TARGET_EXTENDED_ARG,
&&TARGET_LIST_APPEND,
&&TARGET_SET_ADD,
&&TARGET_MAP_ADD,
&&TARGET_LOAD_CLASSDEREF,
- &&TARGET_STORE_FAST__STORE_FAST,
- &&_unknown_opcode,
- &&_unknown_opcode,
+ &&TARGET_STORE_FAST__LOAD_FAST,
+ &&TARGET_LOAD_FAST__LOAD_CONST,
+ &&TARGET_LOAD_CONST__LOAD_FAST,
&&TARGET_MATCH_CLASS,
- &&_unknown_opcode,
+ &&TARGET_STORE_FAST__STORE_FAST,
&&_unknown_opcode,
&&TARGET_FORMAT_VALUE,
&&TARGET_BUILD_CONST_KEY_MAP,
diff --git a/Python/specialize.c b/Python/specialize.c
index 6efee7643a455..529eabf6bc3ab 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -124,6 +124,7 @@ _Py_GetSpecializationStats(void) {
err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
err += add_stat_dict(stats, LOAD_METHOD, "load_method");
err += add_stat_dict(stats, BINARY_ADD, "binary_add");
+ err += add_stat_dict(stats, BINARY_MULTIPLY, "binary_multiply");
err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
err += add_stat_dict(stats, STORE_ATTR, "store_attr");
if (err < 0) {
@@ -180,6 +181,7 @@ _Py_PrintSpecializationStats(void)
print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global");
print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method");
print_stats(out, &_specialization_stats[BINARY_ADD], "binary_add");
+ print_stats(out, &_specialization_stats[BINARY_MULTIPLY], "binary_multiply");
print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr");
print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr");
if (out != stderr) {
@@ -230,6 +232,7 @@ static uint8_t adaptive_opcodes[256] = {
[LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
[LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
[BINARY_ADD] = BINARY_ADD_ADAPTIVE,
+ [BINARY_MULTIPLY] = BINARY_MULTIPLY_ADAPTIVE,
[BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
[STORE_ATTR] = STORE_ATTR_ADAPTIVE,
};
@@ -240,6 +243,7 @@ static uint8_t cache_requirements[256] = {
[LOAD_GLOBAL] = 2, /* _PyAdaptiveEntry and _PyLoadGlobalCache */
[LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
[BINARY_ADD] = 0,
+ [BINARY_MULTIPLY] = 0,
[BINARY_SUBSCR] = 0,
[STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
};
@@ -1188,3 +1192,32 @@ _Py_Specialize_BinaryAdd(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
assert(!PyErr_Occurred());
return 0;
}
+
+int
+_Py_Specialize_BinaryMultiply(PyObject *left, PyObject *right, _Py_CODEUNIT *instr)
+{
+ if (!Py_IS_TYPE(left, Py_TYPE(right))) {
+ SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_DIFFERENT_TYPES);
+ goto fail;
+ }
+ if (PyLong_CheckExact(left)) {
+ *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_INT, saturating_start());
+ goto success;
+ }
+ else if (PyFloat_CheckExact(left)) {
+ *instr = _Py_MAKECODEUNIT(BINARY_MULTIPLY_FLOAT, saturating_start());
+ goto success;
+ }
+ else {
+ SPECIALIZATION_FAIL(BINARY_MULTIPLY, SPEC_FAIL_OTHER);
+ }
+fail:
+ STAT_INC(BINARY_MULTIPLY, specialization_failure);
+ assert(!PyErr_Occurred());
+ *instr = _Py_MAKECODEUNIT(_Py_OPCODE(*instr), ADAPTIVE_CACHE_BACKOFF);
+ return 0;
+success:
+ STAT_INC(BINARY_MULTIPLY, specialization_success);
+ assert(!PyErr_Occurred());
+ return 0;
+}
More information about the Python-checkins
mailing list