[Python-checkins] gh-104584: Allow unspecialized instructions in superblocks (#106497)
gvanrossum
webhook-mailer at python.org
Fri Jul 7 14:03:31 EDT 2023
https://github.com/python/cpython/commit/b3648f036e502db7e7da951ec4eb1f205cb3d74e
commit: b3648f036e502db7e7da951ec4eb1f205cb3d74e
branch: main
author: Guido van Rossum <guido at python.org>
committer: gvanrossum <gvanrossum at gmail.com>
date: 2023-07-07T18:03:27Z
summary:
gh-104584: Allow unspecialized instructions in superblocks (#106497)
This adds several of unspecialized opcodes to superblocks:
TO_BOOL, BINARY_SUBSCR, STORE_SUBSCR,
UNPACK_SEQUENCE, LOAD_GLOBAL, LOAD_ATTR,
COMPARE_OP, BINARY_OP.
While we may not want that eventually, for now this helps finding bugs.
There is a rudimentary test checking for UNPACK_SEQUENCE.
Once we're ready to undo this, that would be simple:
just replace the call to variable_used_unspecialized
with a call to variable_used (as shown in a comment).
Or add individual opcdes to FORBIDDEN_NAMES_IN_UOPS.
files:
M Lib/test/test_capi/test_misc.py
M Python/executor_cases.c.h
M Python/opcode_metadata.h
M Tools/cases_generator/generate_cases.py
diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py
index 181e6b8077f9f..5f39f23401c3f 100644
--- a/Lib/test/test_capi/test_misc.py
+++ b/Lib/test/test_capi/test_misc.py
@@ -2499,6 +2499,34 @@ def many_vars():
ex = get_first_executor(many_vars.__code__)
self.assertIn(("LOAD_FAST", 259), list(ex))
+ def test_unspecialized_unpack(self):
+ # An example of an unspecialized opcode
+ def testfunc(x):
+ i = 0
+ while i < x:
+ i += 1
+ a, b = {1: 2, 3: 3}
+ assert a == 1 and b == 3
+ i = 0
+ while i < x:
+ i += 1
+
+ opt = _testinternalcapi.get_uop_optimizer()
+
+ with temporary_optimizer(opt):
+ testfunc(10)
+
+ ex = None
+ for offset in range(0, len(testfunc.__code__.co_code), 2):
+ try:
+ ex = _testinternalcapi.get_executor(testfunc.__code__, offset)
+ break
+ except ValueError:
+ pass
+ self.assertIsNotNone(ex)
+ uops = {opname for opname, _ in ex}
+ self.assertIn("UNPACK_SEQUENCE", uops)
+
if __name__ == "__main__":
unittest.main()
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index 29ebb0b3e8b2d..32efeb099d9b3 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -118,12 +118,38 @@
break;
}
+ case TO_BOOL: {
+ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size");
+ PyObject *value = stack_pointer[-1];
+ PyObject *res;
+ #line 296 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyToBoolCache *cache = (_PyToBoolCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ next_instr--;
+ _Py_Specialize_ToBool(value, next_instr);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(TO_BOOL, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ int err = PyObject_IsTrue(value);
+ #line 138 "Python/executor_cases.c.h"
+ Py_DECREF(value);
+ #line 308 "Python/bytecodes.c"
+ if (err < 0) goto pop_1_error;
+ res = err ? Py_True : Py_False;
+ #line 143 "Python/executor_cases.c.h"
+ stack_pointer[-1] = res;
+ break;
+ }
+
case TO_BOOL_BOOL: {
PyObject *value = stack_pointer[-1];
#line 313 "Python/bytecodes.c"
DEOPT_IF(!PyBool_Check(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
- #line 127 "Python/executor_cases.c.h"
+ #line 153 "Python/executor_cases.c.h"
break;
}
@@ -138,12 +164,12 @@
res = Py_False;
}
else {
- #line 142 "Python/executor_cases.c.h"
+ #line 168 "Python/executor_cases.c.h"
Py_DECREF(value);
#line 326 "Python/bytecodes.c"
res = Py_True;
}
- #line 147 "Python/executor_cases.c.h"
+ #line 173 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -155,7 +181,7 @@
DEOPT_IF(!PyList_CheckExact(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
res = Py_SIZE(value) ? Py_True : Py_False;
- #line 159 "Python/executor_cases.c.h"
+ #line 185 "Python/executor_cases.c.h"
Py_DECREF(value);
stack_pointer[-1] = res;
break;
@@ -169,7 +195,7 @@
DEOPT_IF(!Py_IsNone(value), TO_BOOL);
STAT_INC(TO_BOOL, hit);
res = Py_False;
- #line 173 "Python/executor_cases.c.h"
+ #line 199 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -186,12 +212,12 @@
}
else {
assert(Py_SIZE(value));
- #line 190 "Python/executor_cases.c.h"
+ #line 216 "Python/executor_cases.c.h"
Py_DECREF(value);
#line 354 "Python/bytecodes.c"
res = Py_True;
}
- #line 195 "Python/executor_cases.c.h"
+ #line 221 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -205,11 +231,11 @@
assert(version);
DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL);
STAT_INC(TO_BOOL, hit);
- #line 209 "Python/executor_cases.c.h"
+ #line 235 "Python/executor_cases.c.h"
Py_DECREF(value);
#line 364 "Python/bytecodes.c"
res = Py_True;
- #line 213 "Python/executor_cases.c.h"
+ #line 239 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -219,11 +245,11 @@
PyObject *res;
#line 368 "Python/bytecodes.c"
res = PyNumber_Invert(value);
- #line 223 "Python/executor_cases.c.h"
+ #line 249 "Python/executor_cases.c.h"
Py_DECREF(value);
#line 370 "Python/bytecodes.c"
if (res == NULL) goto pop_1_error;
- #line 227 "Python/executor_cases.c.h"
+ #line 253 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -234,7 +260,7 @@
#line 386 "Python/bytecodes.c"
DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
- #line 238 "Python/executor_cases.c.h"
+ #line 264 "Python/executor_cases.c.h"
break;
}
@@ -248,7 +274,7 @@
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
- #line 252 "Python/executor_cases.c.h"
+ #line 278 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -264,7 +290,7 @@
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
- #line 268 "Python/executor_cases.c.h"
+ #line 294 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -280,7 +306,7 @@
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
if (res == NULL) goto pop_2_error;
- #line 284 "Python/executor_cases.c.h"
+ #line 310 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -292,7 +318,7 @@
#line 422 "Python/bytecodes.c"
DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
- #line 296 "Python/executor_cases.c.h"
+ #line 322 "Python/executor_cases.c.h"
break;
}
@@ -306,7 +332,7 @@
((PyFloatObject *)left)->ob_fval *
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
- #line 310 "Python/executor_cases.c.h"
+ #line 336 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -322,7 +348,7 @@
((PyFloatObject *)left)->ob_fval +
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
- #line 326 "Python/executor_cases.c.h"
+ #line 352 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -338,7 +364,7 @@
((PyFloatObject *)left)->ob_fval -
((PyFloatObject *)right)->ob_fval;
DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);
- #line 342 "Python/executor_cases.c.h"
+ #line 368 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -350,7 +376,7 @@
#line 458 "Python/bytecodes.c"
DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP);
- #line 354 "Python/executor_cases.c.h"
+ #line 380 "Python/executor_cases.c.h"
break;
}
@@ -364,7 +390,35 @@
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (res == NULL) goto pop_2_error;
- #line 368 "Python/executor_cases.c.h"
+ #line 394 "Python/executor_cases.c.h"
+ STACK_SHRINK(1);
+ stack_pointer[-1] = res;
+ break;
+ }
+
+ case BINARY_SUBSCR: {
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 1, "incorrect cache size");
+ PyObject *sub = stack_pointer[-1];
+ PyObject *container = stack_pointer[-2];
+ PyObject *res;
+ #line 517 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ next_instr--;
+ _Py_Specialize_BinarySubscr(container, sub, next_instr);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(BINARY_SUBSCR, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ res = PyObject_GetItem(container, sub);
+ #line 417 "Python/executor_cases.c.h"
+ Py_DECREF(container);
+ Py_DECREF(sub);
+ #line 529 "Python/bytecodes.c"
+ if (res == NULL) goto pop_2_error;
+ #line 422 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -388,7 +442,7 @@
}
Py_DECREF(container);
if (res == NULL) goto pop_3_error;
- #line 392 "Python/executor_cases.c.h"
+ #line 446 "Python/executor_cases.c.h"
STACK_SHRINK(2);
stack_pointer[-1] = res;
break;
@@ -412,7 +466,7 @@
Py_DECREF(v);
Py_DECREF(container);
if (err) goto pop_4_error;
- #line 416 "Python/executor_cases.c.h"
+ #line 470 "Python/executor_cases.c.h"
STACK_SHRINK(4);
break;
}
@@ -435,7 +489,7 @@
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
- #line 439 "Python/executor_cases.c.h"
+ #line 493 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -459,7 +513,7 @@
Py_INCREF(res);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(tuple);
- #line 463 "Python/executor_cases.c.h"
+ #line 517 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -477,14 +531,14 @@
if (!_PyErr_Occurred(tstate)) {
_PyErr_SetKeyError(sub);
}
- #line 481 "Python/executor_cases.c.h"
+ #line 535 "Python/executor_cases.c.h"
Py_DECREF(dict);
Py_DECREF(sub);
#line 603 "Python/bytecodes.c"
if (true) goto pop_2_error;
}
Py_INCREF(res); // Do this before DECREF'ing dict, sub
- #line 488 "Python/executor_cases.c.h"
+ #line 542 "Python/executor_cases.c.h"
Py_DECREF(dict);
Py_DECREF(sub);
STACK_SHRINK(1);
@@ -497,7 +551,7 @@
PyObject *list = stack_pointer[-(2 + (oparg-1))];
#line 635 "Python/bytecodes.c"
if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error;
- #line 501 "Python/executor_cases.c.h"
+ #line 555 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -507,15 +561,47 @@
PyObject *set = stack_pointer[-(2 + (oparg-1))];
#line 639 "Python/bytecodes.c"
int err = PySet_Add(set, v);
- #line 511 "Python/executor_cases.c.h"
+ #line 565 "Python/executor_cases.c.h"
Py_DECREF(v);
#line 641 "Python/bytecodes.c"
if (err) goto pop_1_error;
- #line 515 "Python/executor_cases.c.h"
+ #line 569 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
+ case STORE_SUBSCR: {
+ static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size");
+ PyObject *sub = stack_pointer[-1];
+ PyObject *container = stack_pointer[-2];
+ PyObject *v = stack_pointer[-3];
+ uint16_t counter = (uint16_t)operand;
+ #line 651 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ if (ADAPTIVE_COUNTER_IS_ZERO(counter)) {
+ next_instr--;
+ _Py_Specialize_StoreSubscr(container, sub, next_instr);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(STORE_SUBSCR, deferred);
+ _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #else
+ (void)counter; // Unused.
+ #endif /* ENABLE_SPECIALIZATION */
+ /* container[sub] = v */
+ int err = PyObject_SetItem(container, sub, v);
+ #line 595 "Python/executor_cases.c.h"
+ Py_DECREF(v);
+ Py_DECREF(container);
+ Py_DECREF(sub);
+ #line 666 "Python/bytecodes.c"
+ if (err) goto pop_3_error;
+ #line 601 "Python/executor_cases.c.h"
+ STACK_SHRINK(3);
+ break;
+ }
+
case STORE_SUBSCR_LIST_INT: {
PyObject *sub = stack_pointer[-1];
PyObject *list = stack_pointer[-2];
@@ -537,7 +623,7 @@
Py_DECREF(old_value);
_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
Py_DECREF(list);
- #line 541 "Python/executor_cases.c.h"
+ #line 627 "Python/executor_cases.c.h"
STACK_SHRINK(3);
break;
}
@@ -552,7 +638,7 @@
int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
Py_DECREF(dict);
if (err) goto pop_3_error;
- #line 556 "Python/executor_cases.c.h"
+ #line 642 "Python/executor_cases.c.h"
STACK_SHRINK(3);
break;
}
@@ -563,12 +649,12 @@
#line 697 "Python/bytecodes.c"
/* del container[sub] */
int err = PyObject_DelItem(container, sub);
- #line 567 "Python/executor_cases.c.h"
+ #line 653 "Python/executor_cases.c.h"
Py_DECREF(container);
Py_DECREF(sub);
#line 700 "Python/bytecodes.c"
if (err) goto pop_2_error;
- #line 572 "Python/executor_cases.c.h"
+ #line 658 "Python/executor_cases.c.h"
STACK_SHRINK(2);
break;
}
@@ -579,11 +665,11 @@
#line 704 "Python/bytecodes.c"
assert(oparg <= MAX_INTRINSIC_1);
res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value);
- #line 583 "Python/executor_cases.c.h"
+ #line 669 "Python/executor_cases.c.h"
Py_DECREF(value);
#line 707 "Python/bytecodes.c"
if (res == NULL) goto pop_1_error;
- #line 587 "Python/executor_cases.c.h"
+ #line 673 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -595,12 +681,12 @@
#line 711 "Python/bytecodes.c"
assert(oparg <= MAX_INTRINSIC_2);
res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1);
- #line 599 "Python/executor_cases.c.h"
+ #line 685 "Python/executor_cases.c.h"
Py_DECREF(value2);
Py_DECREF(value1);
#line 714 "Python/bytecodes.c"
if (res == NULL) goto pop_2_error;
- #line 604 "Python/executor_cases.c.h"
+ #line 690 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -622,14 +708,14 @@
"'async for' requires an object with "
"__aiter__ method, got %.100s",
type->tp_name);
- #line 626 "Python/executor_cases.c.h"
+ #line 712 "Python/executor_cases.c.h"
Py_DECREF(obj);
#line 832 "Python/bytecodes.c"
if (true) goto pop_1_error;
}
iter = (*getter)(obj);
- #line 633 "Python/executor_cases.c.h"
+ #line 719 "Python/executor_cases.c.h"
Py_DECREF(obj);
#line 837 "Python/bytecodes.c"
if (iter == NULL) goto pop_1_error;
@@ -644,7 +730,7 @@
Py_DECREF(iter);
if (true) goto pop_1_error;
}
- #line 648 "Python/executor_cases.c.h"
+ #line 734 "Python/executor_cases.c.h"
stack_pointer[-1] = iter;
break;
}
@@ -695,7 +781,7 @@
Py_DECREF(next_iter);
}
}
- #line 699 "Python/executor_cases.c.h"
+ #line 785 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = awaitable;
break;
@@ -711,7 +797,7 @@
format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
}
- #line 715 "Python/executor_cases.c.h"
+ #line 801 "Python/executor_cases.c.h"
Py_DECREF(iterable);
#line 904 "Python/bytecodes.c"
@@ -730,7 +816,7 @@
}
if (iter == NULL) goto pop_1_error;
- #line 734 "Python/executor_cases.c.h"
+ #line 820 "Python/executor_cases.c.h"
stack_pointer[-1] = iter;
break;
}
@@ -740,7 +826,7 @@
#line 1034 "Python/bytecodes.c"
_PyErr_StackItem *exc_info = tstate->exc_info;
Py_XSETREF(exc_info->exc_value, exc_value);
- #line 744 "Python/executor_cases.c.h"
+ #line 830 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -749,7 +835,7 @@
PyObject *value;
#line 1085 "Python/bytecodes.c"
value = Py_NewRef(PyExc_AssertionError);
- #line 753 "Python/executor_cases.c.h"
+ #line 839 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = value;
break;
@@ -779,7 +865,7 @@
if (true) goto error;
}
}
- #line 783 "Python/executor_cases.c.h"
+ #line 869 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = bc;
break;
@@ -794,7 +880,7 @@
if (ns == NULL) {
_PyErr_Format(tstate, PyExc_SystemError,
"no locals found when storing %R", name);
- #line 798 "Python/executor_cases.c.h"
+ #line 884 "Python/executor_cases.c.h"
Py_DECREF(v);
#line 1121 "Python/bytecodes.c"
if (true) goto pop_1_error;
@@ -803,11 +889,11 @@
err = PyDict_SetItem(ns, name, v);
else
err = PyObject_SetItem(ns, name, v);
- #line 807 "Python/executor_cases.c.h"
+ #line 893 "Python/executor_cases.c.h"
Py_DECREF(v);
#line 1128 "Python/bytecodes.c"
if (err) goto pop_1_error;
- #line 811 "Python/executor_cases.c.h"
+ #line 897 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -830,7 +916,33 @@
name);
goto error;
}
- #line 834 "Python/executor_cases.c.h"
+ #line 920 "Python/executor_cases.c.h"
+ break;
+ }
+
+ case UNPACK_SEQUENCE: {
+ static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size");
+ PyObject *seq = stack_pointer[-1];
+ #line 1158 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ next_instr--;
+ _Py_Specialize_UnpackSequence(seq, next_instr, oparg);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(UNPACK_SEQUENCE, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ PyObject **top = stack_pointer + oparg - 1;
+ int res = unpack_iterable(tstate, seq, oparg, -1, top);
+ #line 940 "Python/executor_cases.c.h"
+ Py_DECREF(seq);
+ #line 1171 "Python/bytecodes.c"
+ if (res == 0) goto pop_1_error;
+ #line 944 "Python/executor_cases.c.h"
+ STACK_SHRINK(1);
+ STACK_GROW(oparg);
break;
}
@@ -844,7 +956,7 @@
STAT_INC(UNPACK_SEQUENCE, hit);
values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1));
values[1] = Py_NewRef(PyTuple_GET_ITEM(seq, 0));
- #line 848 "Python/executor_cases.c.h"
+ #line 960 "Python/executor_cases.c.h"
Py_DECREF(seq);
STACK_SHRINK(1);
STACK_GROW(oparg);
@@ -862,7 +974,7 @@
for (int i = oparg; --i >= 0; ) {
*values++ = Py_NewRef(items[i]);
}
- #line 866 "Python/executor_cases.c.h"
+ #line 978 "Python/executor_cases.c.h"
Py_DECREF(seq);
STACK_SHRINK(1);
STACK_GROW(oparg);
@@ -880,7 +992,7 @@
for (int i = oparg; --i >= 0; ) {
*values++ = Py_NewRef(items[i]);
}
- #line 884 "Python/executor_cases.c.h"
+ #line 996 "Python/executor_cases.c.h"
Py_DECREF(seq);
STACK_SHRINK(1);
STACK_GROW(oparg);
@@ -893,11 +1005,11 @@
int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8);
PyObject **top = stack_pointer + totalargs - 1;
int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top);
- #line 897 "Python/executor_cases.c.h"
+ #line 1009 "Python/executor_cases.c.h"
Py_DECREF(seq);
#line 1211 "Python/bytecodes.c"
if (res == 0) goto pop_1_error;
- #line 901 "Python/executor_cases.c.h"
+ #line 1013 "Python/executor_cases.c.h"
STACK_GROW((oparg & 0xFF) + (oparg >> 8));
break;
}
@@ -907,11 +1019,11 @@
#line 1242 "Python/bytecodes.c"
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyObject_SetAttr(owner, name, (PyObject *)NULL);
- #line 911 "Python/executor_cases.c.h"
+ #line 1023 "Python/executor_cases.c.h"
Py_DECREF(owner);
#line 1245 "Python/bytecodes.c"
if (err) goto pop_1_error;
- #line 915 "Python/executor_cases.c.h"
+ #line 1027 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -921,11 +1033,11 @@
#line 1249 "Python/bytecodes.c"
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
int err = PyDict_SetItem(GLOBALS(), name, v);
- #line 925 "Python/executor_cases.c.h"
+ #line 1037 "Python/executor_cases.c.h"
Py_DECREF(v);
#line 1252 "Python/bytecodes.c"
if (err) goto pop_1_error;
- #line 929 "Python/executor_cases.c.h"
+ #line 1041 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -943,7 +1055,7 @@
}
goto error;
}
- #line 947 "Python/executor_cases.c.h"
+ #line 1059 "Python/executor_cases.c.h"
break;
}
@@ -957,7 +1069,7 @@
if (true) goto error;
}
Py_INCREF(locals);
- #line 961 "Python/executor_cases.c.h"
+ #line 1073 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = locals;
break;
@@ -1023,17 +1135,81 @@
}
}
}
- #line 1027 "Python/executor_cases.c.h"
+ #line 1139 "Python/executor_cases.c.h"
stack_pointer[-1] = v;
break;
}
+ case LOAD_GLOBAL: {
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size");
+ PyObject *null = NULL;
+ PyObject *v;
+ #line 1351 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+ next_instr--;
+ _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(LOAD_GLOBAL, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+ if (PyDict_CheckExact(GLOBALS())
+ && PyDict_CheckExact(BUILTINS()))
+ {
+ v = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(),
+ (PyDictObject *)BUILTINS(),
+ name);
+ if (v == NULL) {
+ if (!_PyErr_Occurred(tstate)) {
+ /* _PyDict_LoadGlobal() returns NULL without raising
+ * an exception if the key doesn't exist */
+ format_exc_check_arg(tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
+ }
+ if (true) goto error;
+ }
+ Py_INCREF(v);
+ }
+ else {
+ /* Slow-path if globals or builtins is not a dict */
+
+ /* namespace 1: globals */
+ v = PyObject_GetItem(GLOBALS(), name);
+ if (v == NULL) {
+ if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) goto error;
+ _PyErr_Clear(tstate);
+
+ /* namespace 2: builtins */
+ v = PyObject_GetItem(BUILTINS(), name);
+ if (v == NULL) {
+ if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) {
+ format_exc_check_arg(
+ tstate, PyExc_NameError,
+ NAME_ERROR_MSG, name);
+ }
+ if (true) goto error;
+ }
+ }
+ }
+ null = NULL;
+ #line 1200 "Python/executor_cases.c.h"
+ STACK_GROW(1);
+ STACK_GROW(((oparg & 1) ? 1 : 0));
+ stack_pointer[-1] = v;
+ if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; }
+ break;
+ }
+
case DELETE_FAST: {
#line 1435 "Python/bytecodes.c"
PyObject *v = GETLOCAL(oparg);
if (v == NULL) goto unbound_local_error;
SETLOCAL(oparg, NULL);
- #line 1037 "Python/executor_cases.c.h"
+ #line 1213 "Python/executor_cases.c.h"
break;
}
@@ -1049,7 +1225,7 @@
}
PyCell_SET(cell, NULL);
Py_DECREF(oldobj);
- #line 1053 "Python/executor_cases.c.h"
+ #line 1229 "Python/executor_cases.c.h"
break;
}
@@ -1091,7 +1267,7 @@
}
Py_INCREF(value);
}
- #line 1095 "Python/executor_cases.c.h"
+ #line 1271 "Python/executor_cases.c.h"
stack_pointer[-1] = value;
break;
}
@@ -1106,7 +1282,7 @@
if (true) goto error;
}
Py_INCREF(value);
- #line 1110 "Python/executor_cases.c.h"
+ #line 1286 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = value;
break;
@@ -1119,7 +1295,7 @@
PyObject *oldobj = PyCell_GET(cell);
PyCell_SET(cell, v);
Py_XDECREF(oldobj);
- #line 1123 "Python/executor_cases.c.h"
+ #line 1299 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -1136,7 +1312,7 @@
PyObject *o = PyTuple_GET_ITEM(closure, i);
frame->localsplus[offset + i] = Py_NewRef(o);
}
- #line 1140 "Python/executor_cases.c.h"
+ #line 1316 "Python/executor_cases.c.h"
break;
}
@@ -1145,13 +1321,13 @@
PyObject *str;
#line 1532 "Python/bytecodes.c"
str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg);
- #line 1149 "Python/executor_cases.c.h"
+ #line 1325 "Python/executor_cases.c.h"
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(pieces[_i]);
}
#line 1534 "Python/bytecodes.c"
if (str == NULL) { STACK_SHRINK(oparg); goto error; }
- #line 1155 "Python/executor_cases.c.h"
+ #line 1331 "Python/executor_cases.c.h"
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = str;
@@ -1164,7 +1340,7 @@
#line 1538 "Python/bytecodes.c"
tup = _PyTuple_FromArraySteal(values, oparg);
if (tup == NULL) { STACK_SHRINK(oparg); goto error; }
- #line 1168 "Python/executor_cases.c.h"
+ #line 1344 "Python/executor_cases.c.h"
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = tup;
@@ -1177,7 +1353,7 @@
#line 1543 "Python/bytecodes.c"
list = _PyList_FromArraySteal(values, oparg);
if (list == NULL) { STACK_SHRINK(oparg); goto error; }
- #line 1181 "Python/executor_cases.c.h"
+ #line 1357 "Python/executor_cases.c.h"
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = list;
@@ -1198,13 +1374,13 @@
"Value after * must be an iterable, not %.200s",
Py_TYPE(iterable)->tp_name);
}
- #line 1202 "Python/executor_cases.c.h"
+ #line 1378 "Python/executor_cases.c.h"
Py_DECREF(iterable);
#line 1559 "Python/bytecodes.c"
if (true) goto pop_1_error;
}
assert(Py_IsNone(none_val));
- #line 1208 "Python/executor_cases.c.h"
+ #line 1384 "Python/executor_cases.c.h"
Py_DECREF(iterable);
STACK_SHRINK(1);
break;
@@ -1215,11 +1391,11 @@
PyObject *set = stack_pointer[-(2 + (oparg-1))];
#line 1566 "Python/bytecodes.c"
int err = _PySet_Update(set, iterable);
- #line 1219 "Python/executor_cases.c.h"
+ #line 1395 "Python/executor_cases.c.h"
Py_DECREF(iterable);
#line 1568 "Python/bytecodes.c"
if (err < 0) goto pop_1_error;
- #line 1223 "Python/executor_cases.c.h"
+ #line 1399 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -1242,7 +1418,7 @@
Py_DECREF(set);
if (true) { STACK_SHRINK(oparg); goto error; }
}
- #line 1246 "Python/executor_cases.c.h"
+ #line 1422 "Python/executor_cases.c.h"
STACK_SHRINK(oparg);
STACK_GROW(1);
stack_pointer[-1] = set;
@@ -1260,13 +1436,13 @@
if (map == NULL)
goto error;
- #line 1264 "Python/executor_cases.c.h"
+ #line 1440 "Python/executor_cases.c.h"
for (int _i = oparg*2; --_i >= 0;) {
Py_DECREF(values[_i]);
}
#line 1597 "Python/bytecodes.c"
if (map == NULL) { STACK_SHRINK(oparg*2); goto error; }
- #line 1270 "Python/executor_cases.c.h"
+ #line 1446 "Python/executor_cases.c.h"
STACK_SHRINK(oparg*2);
STACK_GROW(1);
stack_pointer[-1] = map;
@@ -1314,7 +1490,7 @@
Py_DECREF(ann_dict);
}
}
- #line 1318 "Python/executor_cases.c.h"
+ #line 1494 "Python/executor_cases.c.h"
break;
}
@@ -1332,14 +1508,14 @@
map = _PyDict_FromItems(
&PyTuple_GET_ITEM(keys, 0), 1,
values, 1, oparg);
- #line 1336 "Python/executor_cases.c.h"
+ #line 1512 "Python/executor_cases.c.h"
for (int _i = oparg; --_i >= 0;) {
Py_DECREF(values[_i]);
}
Py_DECREF(keys);
#line 1653 "Python/bytecodes.c"
if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; }
- #line 1343 "Python/executor_cases.c.h"
+ #line 1519 "Python/executor_cases.c.h"
STACK_SHRINK(oparg);
stack_pointer[-1] = map;
break;
@@ -1355,12 +1531,12 @@
"'%.200s' object is not a mapping",
Py_TYPE(update)->tp_name);
}
- #line 1359 "Python/executor_cases.c.h"
+ #line 1535 "Python/executor_cases.c.h"
Py_DECREF(update);
#line 1665 "Python/bytecodes.c"
if (true) goto pop_1_error;
}
- #line 1364 "Python/executor_cases.c.h"
+ #line 1540 "Python/executor_cases.c.h"
Py_DECREF(update);
STACK_SHRINK(1);
break;
@@ -1373,12 +1549,12 @@
if (_PyDict_MergeEx(dict, update, 2) < 0) {
format_kwargs_error(tstate, PEEK(3 + oparg), update);
- #line 1377 "Python/executor_cases.c.h"
+ #line 1553 "Python/executor_cases.c.h"
Py_DECREF(update);
#line 1676 "Python/bytecodes.c"
if (true) goto pop_1_error;
}
- #line 1382 "Python/executor_cases.c.h"
+ #line 1558 "Python/executor_cases.c.h"
Py_DECREF(update);
STACK_SHRINK(1);
break;
@@ -1393,7 +1569,7 @@
/* dict[key] = value */
// Do not DECREF INPUTS because the function steals the references
if (_PyDict_SetItem_Take2((PyDictObject *)dict, key, value) != 0) goto pop_2_error;
- #line 1397 "Python/executor_cases.c.h"
+ #line 1573 "Python/executor_cases.c.h"
STACK_SHRINK(2);
break;
}
@@ -1411,13 +1587,13 @@
STAT_INC(LOAD_SUPER_ATTR, hit);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2);
res = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL);
- #line 1415 "Python/executor_cases.c.h"
+ #line 1591 "Python/executor_cases.c.h"
Py_DECREF(global_super);
Py_DECREF(class);
Py_DECREF(self);
#line 1772 "Python/bytecodes.c"
if (res == NULL) goto pop_3_error;
- #line 1421 "Python/executor_cases.c.h"
+ #line 1597 "Python/executor_cases.c.h"
STACK_SHRINK(2);
STACK_GROW(((oparg & 1) ? 1 : 0));
stack_pointer[-1] = res;
@@ -1454,13 +1630,110 @@
res = res2;
res2 = NULL;
}
- #line 1458 "Python/executor_cases.c.h"
+ #line 1634 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
stack_pointer[-2] = res2;
break;
}
+ case LOAD_ATTR: {
+ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size");
+ PyObject *owner = stack_pointer[-1];
+ PyObject *res2 = NULL;
+ PyObject *res;
+ #line 1815 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyAttrCache *cache = (_PyAttrCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
+ next_instr--;
+ _Py_Specialize_LoadAttr(owner, next_instr, name);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(LOAD_ATTR, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1);
+ if (oparg & 1) {
+ /* Designed to work in tandem with CALL, pushes two values. */
+ PyObject* meth = NULL;
+ if (_PyObject_GetMethod(owner, name, &meth)) {
+ /* We can bypass temporary bound method object.
+ meth is unbound method and obj is self.
+
+ meth | self | arg1 | ... | argN
+ */
+ assert(meth != NULL); // No errors on this branch
+ res2 = meth;
+ res = owner; // Transfer ownership
+ }
+ else {
+ /* meth is not an unbound method (but a regular attr, or
+ something was returned by a descriptor protocol). Set
+ the second element of the stack to NULL, to signal
+ CALL that it's not a method call.
+
+ NULL | meth | arg1 | ... | argN
+ */
+ #line 1680 "Python/executor_cases.c.h"
+ Py_DECREF(owner);
+ #line 1849 "Python/bytecodes.c"
+ if (meth == NULL) goto pop_1_error;
+ res2 = NULL;
+ res = meth;
+ }
+ }
+ else {
+ /* Classic, pushes one value. */
+ res = PyObject_GetAttr(owner, name);
+ #line 1691 "Python/executor_cases.c.h"
+ Py_DECREF(owner);
+ #line 1858 "Python/bytecodes.c"
+ if (res == NULL) goto pop_1_error;
+ }
+ #line 1696 "Python/executor_cases.c.h"
+ STACK_GROW(((oparg & 1) ? 1 : 0));
+ stack_pointer[-1] = res;
+ if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; }
+ break;
+ }
+
+ case COMPARE_OP: {
+ static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size");
+ PyObject *right = stack_pointer[-1];
+ PyObject *left = stack_pointer[-2];
+ PyObject *res;
+ #line 2091 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ next_instr--;
+ _Py_Specialize_CompareOp(left, right, next_instr, oparg);
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(COMPARE_OP, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ assert((oparg >> 5) <= Py_GE);
+ res = PyObject_RichCompare(left, right, oparg >> 5);
+ #line 1721 "Python/executor_cases.c.h"
+ Py_DECREF(left);
+ Py_DECREF(right);
+ #line 2104 "Python/bytecodes.c"
+ if (res == NULL) goto pop_2_error;
+ if (oparg & 16) {
+ int res_bool = PyObject_IsTrue(res);
+ Py_DECREF(res);
+ if (res_bool < 0) goto pop_2_error;
+ res = res_bool ? Py_True : Py_False;
+ }
+ #line 1732 "Python/executor_cases.c.h"
+ STACK_SHRINK(1);
+ stack_pointer[-1] = res;
+ break;
+ }
+
case COMPARE_OP_FLOAT: {
PyObject *right = stack_pointer[-1];
PyObject *left = stack_pointer[-2];
@@ -1477,7 +1750,7 @@
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
- #line 1481 "Python/executor_cases.c.h"
+ #line 1754 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -1503,7 +1776,7 @@
_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
res = (sign_ish & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
- #line 1507 "Python/executor_cases.c.h"
+ #line 1780 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -1526,7 +1799,7 @@
assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS);
res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? Py_True : Py_False;
// It's always a bool, so we don't care about oparg & 16.
- #line 1530 "Python/executor_cases.c.h"
+ #line 1803 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -1538,12 +1811,12 @@
PyObject *b;
#line 2163 "Python/bytecodes.c"
int res = Py_Is(left, right) ^ oparg;
- #line 1542 "Python/executor_cases.c.h"
+ #line 1815 "Python/executor_cases.c.h"
Py_DECREF(left);
Py_DECREF(right);
#line 2165 "Python/bytecodes.c"
b = res ? Py_True : Py_False;
- #line 1547 "Python/executor_cases.c.h"
+ #line 1820 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = b;
break;
@@ -1555,13 +1828,13 @@
PyObject *b;
#line 2169 "Python/bytecodes.c"
int res = PySequence_Contains(right, left);
- #line 1559 "Python/executor_cases.c.h"
+ #line 1832 "Python/executor_cases.c.h"
Py_DECREF(left);
Py_DECREF(right);
#line 2171 "Python/bytecodes.c"
if (res < 0) goto pop_2_error;
b = (res ^ oparg) ? Py_True : Py_False;
- #line 1565 "Python/executor_cases.c.h"
+ #line 1838 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = b;
break;
@@ -1574,7 +1847,7 @@
PyObject *match;
#line 2176 "Python/bytecodes.c"
if (check_except_star_type_valid(tstate, match_type) < 0) {
- #line 1578 "Python/executor_cases.c.h"
+ #line 1851 "Python/executor_cases.c.h"
Py_DECREF(exc_value);
Py_DECREF(match_type);
#line 2178 "Python/bytecodes.c"
@@ -1585,7 +1858,7 @@
rest = NULL;
int res = exception_group_match(exc_value, match_type,
&match, &rest);
- #line 1589 "Python/executor_cases.c.h"
+ #line 1862 "Python/executor_cases.c.h"
Py_DECREF(exc_value);
Py_DECREF(match_type);
#line 2186 "Python/bytecodes.c"
@@ -1597,7 +1870,7 @@
if (!Py_IsNone(match)) {
PyErr_SetHandledException(match);
}
- #line 1601 "Python/executor_cases.c.h"
+ #line 1874 "Python/executor_cases.c.h"
stack_pointer[-1] = match;
stack_pointer[-2] = rest;
break;
@@ -1610,18 +1883,18 @@
#line 2197 "Python/bytecodes.c"
assert(PyExceptionInstance_Check(left));
if (check_except_type_valid(tstate, right) < 0) {
- #line 1614 "Python/executor_cases.c.h"
+ #line 1887 "Python/executor_cases.c.h"
Py_DECREF(right);
#line 2200 "Python/bytecodes.c"
if (true) goto pop_1_error;
}
int res = PyErr_GivenExceptionMatches(left, right);
- #line 1621 "Python/executor_cases.c.h"
+ #line 1894 "Python/executor_cases.c.h"
Py_DECREF(right);
#line 2205 "Python/bytecodes.c"
b = res ? Py_True : Py_False;
- #line 1625 "Python/executor_cases.c.h"
+ #line 1898 "Python/executor_cases.c.h"
stack_pointer[-1] = b;
break;
}
@@ -1635,7 +1908,7 @@
if (len_i < 0) goto error;
len_o = PyLong_FromSsize_t(len_i);
if (len_o == NULL) goto error;
- #line 1639 "Python/executor_cases.c.h"
+ #line 1912 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = len_o;
break;
@@ -1651,7 +1924,7 @@
// None on failure.
assert(PyTuple_CheckExact(names));
attrs = match_class(tstate, subject, type, oparg, names);
- #line 1655 "Python/executor_cases.c.h"
+ #line 1928 "Python/executor_cases.c.h"
Py_DECREF(subject);
Py_DECREF(type);
Py_DECREF(names);
@@ -1663,7 +1936,7 @@
if (_PyErr_Occurred(tstate)) goto pop_3_error;
attrs = Py_None; // Failure!
}
- #line 1667 "Python/executor_cases.c.h"
+ #line 1940 "Python/executor_cases.c.h"
STACK_SHRINK(2);
stack_pointer[-1] = attrs;
break;
@@ -1675,7 +1948,7 @@
#line 2332 "Python/bytecodes.c"
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING;
res = match ? Py_True : Py_False;
- #line 1679 "Python/executor_cases.c.h"
+ #line 1952 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = res;
break;
@@ -1687,7 +1960,7 @@
#line 2337 "Python/bytecodes.c"
int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE;
res = match ? Py_True : Py_False;
- #line 1691 "Python/executor_cases.c.h"
+ #line 1964 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = res;
break;
@@ -1701,7 +1974,7 @@
// On successful match, PUSH(values). Otherwise, PUSH(None).
values_or_none = match_keys(tstate, subject, keys);
if (values_or_none == NULL) goto error;
- #line 1705 "Python/executor_cases.c.h"
+ #line 1978 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = values_or_none;
break;
@@ -1713,11 +1986,11 @@
#line 2348 "Python/bytecodes.c"
/* before: [obj]; after [getiter(obj)] */
iter = PyObject_GetIter(iterable);
- #line 1717 "Python/executor_cases.c.h"
+ #line 1990 "Python/executor_cases.c.h"
Py_DECREF(iterable);
#line 2351 "Python/bytecodes.c"
if (iter == NULL) goto pop_1_error;
- #line 1721 "Python/executor_cases.c.h"
+ #line 1994 "Python/executor_cases.c.h"
stack_pointer[-1] = iter;
break;
}
@@ -1748,11 +2021,11 @@
if (iter == NULL) {
goto error;
}
- #line 1752 "Python/executor_cases.c.h"
+ #line 2025 "Python/executor_cases.c.h"
Py_DECREF(iterable);
#line 2378 "Python/bytecodes.c"
}
- #line 1756 "Python/executor_cases.c.h"
+ #line 2029 "Python/executor_cases.c.h"
stack_pointer[-1] = iter;
break;
}
@@ -1783,7 +2056,7 @@
res = PyObject_Vectorcall(exit_func, stack + 1,
3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL);
if (res == NULL) goto error;
- #line 1787 "Python/executor_cases.c.h"
+ #line 2060 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = res;
break;
@@ -1802,7 +2075,7 @@
}
assert(PyExceptionInstance_Check(new_exc));
exc_info->exc_value = Py_NewRef(new_exc);
- #line 1806 "Python/executor_cases.c.h"
+ #line 2079 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = new_exc;
stack_pointer[-2] = prev_exc;
@@ -1819,7 +2092,7 @@
Py_TYPE(should_be_none)->tp_name);
goto error;
}
- #line 1823 "Python/executor_cases.c.h"
+ #line 2096 "Python/executor_cases.c.h"
STACK_SHRINK(1);
break;
}
@@ -1839,7 +2112,7 @@
func_obj->func_version = ((PyCodeObject *)codeobj)->co_version;
func = (PyObject *)func_obj;
- #line 1843 "Python/executor_cases.c.h"
+ #line 2116 "Python/executor_cases.c.h"
stack_pointer[-1] = func;
break;
}
@@ -1872,7 +2145,7 @@
default:
Py_UNREACHABLE();
}
- #line 1876 "Python/executor_cases.c.h"
+ #line 2149 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = func;
break;
@@ -1885,13 +2158,13 @@
PyObject *slice;
#line 3497 "Python/bytecodes.c"
slice = PySlice_New(start, stop, step);
- #line 1889 "Python/executor_cases.c.h"
+ #line 2162 "Python/executor_cases.c.h"
Py_DECREF(start);
Py_DECREF(stop);
Py_XDECREF(step);
#line 3499 "Python/bytecodes.c"
if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; }
- #line 1895 "Python/executor_cases.c.h"
+ #line 2168 "Python/executor_cases.c.h"
STACK_SHRINK(((oparg == 3) ? 1 : 0));
STACK_SHRINK(1);
stack_pointer[-1] = slice;
@@ -1908,7 +2181,7 @@
result = conv_fn(value);
Py_DECREF(value);
if (result == NULL) goto pop_1_error;
- #line 1912 "Python/executor_cases.c.h"
+ #line 2185 "Python/executor_cases.c.h"
stack_pointer[-1] = result;
break;
}
@@ -1927,7 +2200,7 @@
else {
res = value;
}
- #line 1931 "Python/executor_cases.c.h"
+ #line 2204 "Python/executor_cases.c.h"
stack_pointer[-1] = res;
break;
}
@@ -1941,7 +2214,7 @@
Py_DECREF(value);
Py_DECREF(fmt_spec);
if (res == NULL) goto pop_2_error;
- #line 1945 "Python/executor_cases.c.h"
+ #line 2218 "Python/executor_cases.c.h"
STACK_SHRINK(1);
stack_pointer[-1] = res;
break;
@@ -1953,18 +2226,49 @@
#line 3532 "Python/bytecodes.c"
assert(oparg > 0);
top = Py_NewRef(bottom);
- #line 1957 "Python/executor_cases.c.h"
+ #line 2230 "Python/executor_cases.c.h"
STACK_GROW(1);
stack_pointer[-1] = top;
break;
}
+ case BINARY_OP: {
+ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size");
+ PyObject *rhs = stack_pointer[-1];
+ PyObject *lhs = stack_pointer[-2];
+ PyObject *res;
+ #line 3537 "Python/bytecodes.c"
+ #if ENABLE_SPECIALIZATION
+ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
+ if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) {
+ next_instr--;
+ _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
+ DISPATCH_SAME_OPARG();
+ }
+ STAT_INC(BINARY_OP, deferred);
+ DECREMENT_ADAPTIVE_COUNTER(cache->counter);
+ #endif /* ENABLE_SPECIALIZATION */
+ assert(0 <= oparg);
+ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops));
+ assert(binary_ops[oparg]);
+ res = binary_ops[oparg](lhs, rhs);
+ #line 2256 "Python/executor_cases.c.h"
+ Py_DECREF(lhs);
+ Py_DECREF(rhs);
+ #line 3552 "Python/bytecodes.c"
+ if (res == NULL) goto pop_2_error;
+ #line 2261 "Python/executor_cases.c.h"
+ STACK_SHRINK(1);
+ stack_pointer[-1] = res;
+ break;
+ }
+
case SWAP: {
PyObject *top = stack_pointer[-1];
PyObject *bottom = stack_pointer[-(2 + (oparg-2))];
#line 3557 "Python/bytecodes.c"
assert(oparg >= 2);
- #line 1968 "Python/executor_cases.c.h"
+ #line 2272 "Python/executor_cases.c.h"
stack_pointer[-1] = bottom;
stack_pointer[-(2 + (oparg-2))] = top;
break;
diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h
index d29f7216ea65e..70e1ca44ce766 100644
--- a/Python/opcode_metadata.h
+++ b/Python/opcode_metadata.h
@@ -1182,6 +1182,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } },
[UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } },
[UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } },
+ [TO_BOOL] = { .nuops = 1, .uops = { { TO_BOOL, 0, 0 } } },
[TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } },
[TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } },
[TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } },
@@ -1196,6 +1197,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } },
[BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } },
[BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } },
+ [BINARY_SUBSCR] = { .nuops = 1, .uops = { { BINARY_SUBSCR, 0, 0 } } },
[BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } },
[STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } },
[BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } },
@@ -1203,6 +1205,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } },
[LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } },
[SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } },
+ [STORE_SUBSCR] = { .nuops = 1, .uops = { { STORE_SUBSCR, 1, 0 } } },
[STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } },
[STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } },
[DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } },
@@ -1216,6 +1219,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } },
[STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } },
[DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } },
+ [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE, 0, 0 } } },
[UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } },
[UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } },
[UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } },
@@ -1226,6 +1230,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, 0, 0 } } },
[LOAD_NAME] = { .nuops = 2, .uops = { { _LOAD_LOCALS, 0, 0 }, { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } },
[LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { _LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } },
+ [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } },
[DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } },
[DELETE_DEREF] = { .nuops = 1, .uops = { { DELETE_DEREF, 0, 0 } } },
[LOAD_FROM_DICT_OR_DEREF] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_DEREF, 0, 0 } } },
@@ -1246,6 +1251,8 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } },
[LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } },
[LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } },
+ [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } },
+ [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } },
[COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } },
[COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } },
[COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } },
@@ -1270,6 +1277,7 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = {
[FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } },
[FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } },
[COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } },
+ [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } },
[SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } },
};
#ifdef NEED_OPCODE_METADATA
diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py
index 632834ce23166..14269ca8cbe75 100644
--- a/Tools/cases_generator/generate_cases.py
+++ b/Tools/cases_generator/generate_cases.py
@@ -425,8 +425,9 @@ def is_viable_uop(self) -> bool:
return False
res = True
for forbidden in FORBIDDEN_NAMES_IN_UOPS:
- # TODO: Don't check in '#ifdef ENABLE_SPECIALIZATION' regions
- if variable_used(self.inst, forbidden):
+ # NOTE: To disallow unspecialized uops, use
+ # if variable_used(self.inst, forbidden):
+ if variable_used_unspecialized(self.inst, forbidden):
# print(f"Skipping {self.name} because it uses {forbidden}")
res = False
return res
@@ -1644,6 +1645,27 @@ def variable_used(node: parser.Node, name: str) -> bool:
)
+def variable_used_unspecialized(node: parser.Node, name: str) -> bool:
+ """Like variable_used(), but skips #if ENABLE_SPECIALIZATION blocks."""
+ tokens: list[lx.Token] = []
+ skipping = False
+ for i, token in enumerate(node.tokens):
+ if token.kind == "MACRO":
+ text = "".join(token.text.split())
+ # TODO: Handle nested #if
+ if text == "#if":
+ if (
+ i + 1 < len(node.tokens)
+ and node.tokens[i + 1].text == "ENABLE_SPECIALIZATION"
+ ):
+ skipping = True
+ elif text in ("#else", "#endif"):
+ skipping = False
+ if not skipping:
+ tokens.append(token)
+ return any(token.kind == "IDENTIFIER" and token.text == name for token in tokens)
+
+
def main():
"""Parse command line, parse input, analyze, write output."""
args = arg_parser.parse_args() # Prints message and sys.exit(2) on error
More information about the Python-checkins
mailing list