[Python-checkins] gh-90997: bpo-46841: Disassembly of quickened code (GH-32099)
markshannon
webhook-mailer at python.org
Tue Apr 19 04:45:24 EDT 2022
https://github.com/python/cpython/commit/e590379197f065f52c8140c0edd7a59360216531
commit: e590379197f065f52c8140c0edd7a59360216531
branch: main
author: penguin_wwy <940375606 at qq.com>
committer: markshannon <mark at hotpy.org>
date: 2022-04-19T09:45:08+01:00
summary:
gh-90997: bpo-46841: Disassembly of quickened code (GH-32099)
files:
A Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst
M Lib/dis.py
M Lib/test/test__opcode.py
M Lib/test/test_dis.py
diff --git a/Lib/dis.py b/Lib/dis.py
index f7b38a82ab13e..205e9d8d193f3 100644
--- a/Lib/dis.py
+++ b/Lib/dis.py
@@ -7,7 +7,7 @@
from opcode import *
from opcode import __all__ as _opcodes_all
-from opcode import _nb_ops
+from opcode import _nb_ops, _inline_cache_entries, _specializations, _specialized_instructions
__all__ = ["code_info", "dis", "disassemble", "distb", "disco",
"findlinestarts", "findlabels", "show_code",
@@ -34,6 +34,18 @@
CACHE = opmap["CACHE"]
+_all_opname = list(opname)
+_all_opmap = dict(opmap)
+_empty_slot = [slot for slot, name in enumerate(_all_opname) if name.startswith("<")]
+for spec_op, specialized in zip(_empty_slot, _specialized_instructions):
+ # fill opname and opmap
+ _all_opname[spec_op] = specialized
+ _all_opmap[specialized] = spec_op
+
+deoptmap = {
+ specialized: base for base, family in _specializations.items() for specialized in family
+}
+
def _try_compile(source, name):
"""Attempts to compile the given source, first as an expression and
then as a statement if the first approach fails.
@@ -47,7 +59,7 @@ def _try_compile(source, name):
c = compile(source, name, 'exec')
return c
-def dis(x=None, *, file=None, depth=None, show_caches=False):
+def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False):
"""Disassemble classes, methods, functions, and other compiled objects.
With no argument, disassemble the last traceback.
@@ -57,7 +69,7 @@ def dis(x=None, *, file=None, depth=None, show_caches=False):
in a special attribute.
"""
if x is None:
- distb(file=file, show_caches=show_caches)
+ distb(file=file, show_caches=show_caches, adaptive=adaptive)
return
# Extract functions from methods.
if hasattr(x, '__func__'):
@@ -78,21 +90,21 @@ def dis(x=None, *, file=None, depth=None, show_caches=False):
if isinstance(x1, _have_code):
print("Disassembly of %s:" % name, file=file)
try:
- dis(x1, file=file, depth=depth, show_caches=show_caches)
+ dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
except TypeError as msg:
print("Sorry:", msg, file=file)
print(file=file)
elif hasattr(x, 'co_code'): # Code object
- _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches)
+ _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
elif isinstance(x, (bytes, bytearray)): # Raw bytecode
_disassemble_bytes(x, file=file, show_caches=show_caches)
elif isinstance(x, str): # Source code
- _disassemble_str(x, file=file, depth=depth, show_caches=show_caches)
+ _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive)
else:
raise TypeError("don't know how to disassemble %s objects" %
type(x).__name__)
-def distb(tb=None, *, file=None, show_caches=False):
+def distb(tb=None, *, file=None, show_caches=False, adaptive=False):
"""Disassemble a traceback (default: last traceback)."""
if tb is None:
try:
@@ -100,7 +112,7 @@ def distb(tb=None, *, file=None, show_caches=False):
except AttributeError:
raise RuntimeError("no last traceback to disassemble") from None
while tb.tb_next: tb = tb.tb_next
- disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches)
+ disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive)
# The inspect module interrogates this dictionary to build its
# list of CO_* constants. It is also used by pretty_flags to
@@ -162,6 +174,13 @@ def _get_code_object(x):
raise TypeError("don't know how to disassemble %s objects" %
type(x).__name__)
+def _deoptop(op):
+ name = _all_opname[op]
+ return _all_opmap[deoptmap[name]] if name in deoptmap else op
+
+def _get_code_array(co, adaptive):
+ return co._co_code_adaptive if adaptive else co.co_code
+
def code_info(x):
"""Formatted details of methods, functions, or code."""
return _format_code_info(_get_code_object(x))
@@ -302,7 +321,7 @@ def _disassemble(self, lineno_width=3, mark_as_current=False, offset_width=4):
return ' '.join(fields).rstrip()
-def get_instructions(x, *, first_line=None, show_caches=False):
+def get_instructions(x, *, first_line=None, show_caches=False, adaptive=False):
"""Iterator for the opcodes in methods, functions or code
Generates a series of Instruction named tuples giving the details of
@@ -319,7 +338,7 @@ def get_instructions(x, *, first_line=None, show_caches=False):
line_offset = first_line - co.co_firstlineno
else:
line_offset = 0
- return _get_instructions_bytes(co.co_code,
+ return _get_instructions_bytes(_get_code_array(co, adaptive),
co._varname_from_oparg,
co.co_names, co.co_consts,
linestarts, line_offset,
@@ -415,8 +434,13 @@ def _get_instructions_bytes(code, varname_from_oparg=None,
for i in range(start, end):
labels.add(target)
starts_line = None
+ cache_counter = 0
for offset, op, arg in _unpack_opargs(code):
- if not show_caches and op == CACHE:
+ if cache_counter > 0:
+ if show_caches:
+ yield Instruction("CACHE", 0, None, None, '',
+ offset, None, False, None)
+ cache_counter -= 1
continue
if linestarts is not None:
starts_line = linestarts.get(offset, None)
@@ -426,61 +450,63 @@ def _get_instructions_bytes(code, varname_from_oparg=None,
argval = None
argrepr = ''
positions = Positions(*next(co_positions, ()))
+ deop = _deoptop(op)
+ cache_counter = _inline_cache_entries[deop]
if arg is not None:
# Set argval to the dereferenced value of the argument when
# available, and argrepr to the string representation of argval.
# _disassemble_bytes needs the string repr of the
# raw name index for LOAD_GLOBAL, LOAD_CONST, etc.
argval = arg
- if op in hasconst:
- argval, argrepr = _get_const_info(op, arg, co_consts)
- elif op in hasname:
- if op == LOAD_GLOBAL:
+ if deop in hasconst:
+ argval, argrepr = _get_const_info(deop, arg, co_consts)
+ elif deop in hasname:
+ if deop == LOAD_GLOBAL:
argval, argrepr = _get_name_info(arg//2, get_name)
if (arg & 1) and argrepr:
argrepr = "NULL + " + argrepr
else:
argval, argrepr = _get_name_info(arg, get_name)
- elif op in hasjabs:
+ elif deop in hasjabs:
argval = arg*2
argrepr = "to " + repr(argval)
- elif op in hasjrel:
- signed_arg = -arg if _is_backward_jump(op) else arg
+ elif deop in hasjrel:
+ signed_arg = -arg if _is_backward_jump(deop) else arg
argval = offset + 2 + signed_arg*2
argrepr = "to " + repr(argval)
- elif op in haslocal or op in hasfree:
+ elif deop in haslocal or deop in hasfree:
argval, argrepr = _get_name_info(arg, varname_from_oparg)
- elif op in hascompare:
+ elif deop in hascompare:
argval = cmp_op[arg]
argrepr = argval
- elif op == FORMAT_VALUE:
+ elif deop == FORMAT_VALUE:
argval, argrepr = FORMAT_VALUE_CONVERTERS[arg & 0x3]
argval = (argval, bool(arg & 0x4))
if argval[1]:
if argrepr:
argrepr += ', '
argrepr += 'with format'
- elif op == MAKE_FUNCTION:
+ elif deop == MAKE_FUNCTION:
argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS)
if arg & (1<<i))
- elif op == BINARY_OP:
+ elif deop == BINARY_OP:
_, argrepr = _nb_ops[arg]
- yield Instruction(opname[op], op,
+ yield Instruction(_all_opname[op], op,
arg, argval, argrepr,
offset, starts_line, is_jump_target, positions)
-def disassemble(co, lasti=-1, *, file=None, show_caches=False):
+def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False):
"""Disassemble a code object."""
linestarts = dict(findlinestarts(co))
exception_entries = parse_exception_table(co)
- _disassemble_bytes(co.co_code, lasti,
- co._varname_from_oparg,
+ _disassemble_bytes(_get_code_array(co, adaptive),
+ lasti, co._varname_from_oparg,
co.co_names, co.co_consts, linestarts, file=file,
exception_entries=exception_entries,
co_positions=co.co_positions(), show_caches=show_caches)
-def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False):
- disassemble(co, file=file, show_caches=show_caches)
+def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False):
+ disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive)
if depth is None or depth > 0:
if depth is not None:
depth = depth - 1
@@ -489,7 +515,7 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False):
print(file=file)
print("Disassembly of %r:" % (x,), file=file)
_disassemble_recursive(
- x, file=file, depth=depth, show_caches=show_caches
+ x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive
)
def _disassemble_bytes(code, lasti=-1, varname_from_oparg=None,
@@ -548,7 +574,7 @@ def _unpack_opargs(code):
extended_arg = 0
for i in range(0, len(code), 2):
op = code[i]
- if op >= HAVE_ARGUMENT:
+ if _deoptop(op) >= HAVE_ARGUMENT:
arg = code[i+1] | extended_arg
extended_arg = (arg << 8) if op == EXTENDED_ARG else 0
# The oparg is stored as a signed integer
@@ -641,7 +667,7 @@ class Bytecode:
Iterating over this yields the bytecode operations as Instruction instances.
"""
- def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False):
+ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False):
self.codeobj = co = _get_code_object(x)
if first_line is None:
self.first_line = co.co_firstlineno
@@ -654,10 +680,11 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False
self.current_offset = current_offset
self.exception_entries = parse_exception_table(co)
self.show_caches = show_caches
+ self.adaptive = adaptive
def __iter__(self):
co = self.codeobj
- return _get_instructions_bytes(co.co_code,
+ return _get_instructions_bytes(_get_code_array(co, self.adaptive),
co._varname_from_oparg,
co.co_names, co.co_consts,
self._linestarts,
@@ -671,12 +698,12 @@ def __repr__(self):
self._original_object)
@classmethod
- def from_traceback(cls, tb, *, show_caches=False):
+ def from_traceback(cls, tb, *, show_caches=False, adaptive=False):
""" Construct a Bytecode from the given traceback """
while tb.tb_next:
tb = tb.tb_next
return cls(
- tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches
+ tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches, adaptive=adaptive
)
def info(self):
@@ -691,7 +718,7 @@ def dis(self):
else:
offset = -1
with io.StringIO() as output:
- _disassemble_bytes(co.co_code,
+ _disassemble_bytes(_get_code_array(co, self.adaptive),
varname_from_oparg=co._varname_from_oparg,
names=co.co_names, co_consts=co.co_consts,
linestarts=self._linestarts,
diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py
index 7c1c0cfdb069b..2a4c0d2eeb656 100644
--- a/Lib/test/test__opcode.py
+++ b/Lib/test/test__opcode.py
@@ -18,7 +18,7 @@ def test_stack_effect(self):
self.assertRaises(ValueError, stack_effect, dis.opmap['BUILD_SLICE'])
self.assertRaises(ValueError, stack_effect, dis.opmap['POP_TOP'], 0)
# All defined opcodes
- for name, code in dis.opmap.items():
+ for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
with self.subTest(opname=name):
if code < dis.HAVE_ARGUMENT:
stack_effect(code)
@@ -47,7 +47,7 @@ def test_stack_effect_jump(self):
self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0)
# All defined opcodes
has_jump = dis.hasjabs + dis.hasjrel
- for name, code in dis.opmap.items():
+ for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()):
with self.subTest(opname=name):
if code < dis.HAVE_ARGUMENT:
common = stack_effect(code)
diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
index fbc34a5dbe4ef..f560a5556c8b0 100644
--- a/Lib/test/test_dis.py
+++ b/Lib/test/test_dis.py
@@ -7,7 +7,7 @@
import sys
import types
import unittest
-from test.support import captured_stdout, requires_debug_ranges
+from test.support import captured_stdout, requires_debug_ranges, cpython_only
from test.support.bytecode_helper import BytecodeTestCase
import opcode
@@ -583,6 +583,58 @@ def foo(x):
_h.__code__.co_firstlineno + 3,
)
+def load_test(x, y=0):
+ a, b = x, y
+ return a, b
+
+dis_load_test_quickened_code = """\
+%3d 0 RESUME_QUICK 0
+
+%3d 2 LOAD_FAST__LOAD_FAST 0 (x)
+ 4 LOAD_FAST 1 (y)
+ 6 STORE_FAST__STORE_FAST 3 (b)
+ 8 STORE_FAST__LOAD_FAST 2 (a)
+
+%3d 10 LOAD_FAST__LOAD_FAST 2 (a)
+ 12 LOAD_FAST 3 (b)
+ 14 BUILD_TUPLE 2
+ 16 RETURN_VALUE
+""" % (load_test.__code__.co_firstlineno,
+ load_test.__code__.co_firstlineno + 1,
+ load_test.__code__.co_firstlineno + 2)
+
+def loop_test():
+ for i in [1, 2, 3] * 3:
+ load_test(i)
+
+dis_loop_test_quickened_code = """\
+%3d 0 RESUME_QUICK 0
+
+%3d 2 BUILD_LIST 0
+ 4 LOAD_CONST 1 ((1, 2, 3))
+ 6 LIST_EXTEND 1
+ 8 LOAD_CONST 2 (3)
+ 10 BINARY_OP_ADAPTIVE 5 (*)
+ 14 GET_ITER
+ 16 FOR_ITER 17 (to 52)
+ 18 STORE_FAST 0 (i)
+
+%3d 20 LOAD_GLOBAL_MODULE 1 (NULL + load_test)
+ 32 LOAD_FAST 0 (i)
+ 34 PRECALL_PYFUNC 1
+ 38 CALL_PY_WITH_DEFAULTS 1
+ 48 POP_TOP
+ 50 JUMP_BACKWARD_QUICK 18 (to 16)
+
+%3d >> 52 LOAD_CONST 0 (None)
+ 54 RETURN_VALUE
+""" % (loop_test.__code__.co_firstlineno,
+ loop_test.__code__.co_firstlineno + 1,
+ loop_test.__code__.co_firstlineno + 2,
+ loop_test.__code__.co_firstlineno + 1,)
+
+QUICKENING_WARMUP_DELAY = 8
+
class DisTestBase(unittest.TestCase):
"Common utilities for DisTests and TestDisTraceback"
@@ -860,6 +912,93 @@ def check(expected, **kwargs):
check(dis_nested_2, depth=None)
check(dis_nested_2)
+ @staticmethod
+ def code_quicken(f, times=QUICKENING_WARMUP_DELAY):
+ for _ in range(times):
+ f()
+
+ @cpython_only
+ def test_super_instructions(self):
+ self.code_quicken(lambda: load_test(0, 0))
+ got = self.get_disassembly(load_test, adaptive=True)
+ self.do_disassembly_compare(got, dis_load_test_quickened_code, True)
+
+ @cpython_only
+ def test_binary_specialize(self):
+ binary_op_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 LOAD_NAME 0 (a)
+ 4 LOAD_NAME 1 (b)
+ 6 %s
+ 10 RETURN_VALUE
+"""
+ co_int = compile('a + b', "<int>", "eval")
+ self.code_quicken(lambda: exec(co_int, {}, {'a': 1, 'b': 2}))
+ got = self.get_disassembly(co_int, adaptive=True)
+ self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_INT 0 (+)", True)
+
+ co_unicode = compile('a + b', "<unicode>", "eval")
+ self.code_quicken(lambda: exec(co_unicode, {}, {'a': 'a', 'b': 'b'}))
+ got = self.get_disassembly(co_unicode, adaptive=True)
+ self.do_disassembly_compare(got, binary_op_quicken % "BINARY_OP_ADD_UNICODE 0 (+)", True)
+
+ binary_subscr_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 LOAD_NAME 0 (a)
+ 4 LOAD_CONST 0 (0)
+ 6 %s
+ 16 RETURN_VALUE
+"""
+ co_list = compile('a[0]', "<list>", "eval")
+ self.code_quicken(lambda: exec(co_list, {}, {'a': [0]}))
+ got = self.get_disassembly(co_list, adaptive=True)
+ self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT", True)
+
+ co_dict = compile('a[0]', "<dict>", "eval")
+ self.code_quicken(lambda: exec(co_dict, {}, {'a': {0: '1'}}))
+ got = self.get_disassembly(co_dict, adaptive=True)
+ self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT", True)
+
+ @cpython_only
+ def test_load_attr_specialize(self):
+ load_attr_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 LOAD_CONST 0 ('a')
+ 4 LOAD_ATTR_SLOT 0 (__class__)
+ 14 RETURN_VALUE
+"""
+ co = compile("'a'.__class__", "", "eval")
+ self.code_quicken(lambda: exec(co, {}, {}))
+ got = self.get_disassembly(co, adaptive=True)
+ self.do_disassembly_compare(got, load_attr_quicken, True)
+
+ @cpython_only
+ def test_call_specialize(self):
+ call_quicken = """\
+ 0 RESUME_QUICK 0
+
+ 1 2 PUSH_NULL
+ 4 LOAD_NAME 0 (str)
+ 6 LOAD_CONST 0 (1)
+ 8 PRECALL_NO_KW_STR_1 1
+ 12 CALL_ADAPTIVE 1
+ 22 RETURN_VALUE
+"""
+ co = compile("str(1)", "", "eval")
+ self.code_quicken(lambda: exec(co, {}, {}))
+ got = self.get_disassembly(co, adaptive=True)
+ self.do_disassembly_compare(got, call_quicken, True)
+
+ @cpython_only
+ def test_loop_quicken(self):
+ # Loop can trigger a quicken where the loop is located
+ self.code_quicken(loop_test, 1)
+ got = self.get_disassembly(loop_test, adaptive=True)
+ self.do_disassembly_compare(got, dis_loop_test_quickened_code, True)
+
class DisWithFileTests(DisTests):
diff --git a/Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst b/Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst
new file mode 100644
index 0000000000000..0e778047593a7
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-03-25-22-18-45.bpo-46841.NUEsXW.rst
@@ -0,0 +1 @@
+Disassembly of quickened code.
More information about the Python-checkins
mailing list