[pypy-commit] pypy winoverlapped: Merged py3.6 changes
andrewjlawrence
pypy.commits at gmail.com
Thu Mar 28 18:20:26 EDT 2019
Author: andrewjlawrence
Branch: winoverlapped
Changeset: r96386:6c5e88b3724e
Date: 2019-03-28 22:18 +0000
http://bitbucket.org/pypy/pypy/changeset/6c5e88b3724e/
Log: Merged py3.6 changes
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -21,6 +21,7 @@
symbols = symtable.SymtableBuilder(space, module, info)
return TopLevelCodeGenerator(space, module, symbols, info).assemble()
+MAX_STACKDEPTH_CONTAINERS = 100
name_ops_default = misc.dict_to_switch({
ast.Load: ops.LOAD_NAME,
@@ -1222,8 +1223,29 @@
ifexp.orelse.walkabout(self)
self.use_next_block(end)
- def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op):
+ def _visit_starunpack(self, node, elts, single_op, inner_op, outer_op, add_op):
elt_count = len(elts) if elts else 0
+ contains_starred = False
+ for i in range(elt_count):
+ elt = elts[i]
+ if isinstance(elt, ast.Starred):
+ contains_starred = True
+ break
+ if elt_count > MAX_STACKDEPTH_CONTAINERS and not contains_starred:
+ tuplecase = False
+ if add_op == -1: # tuples
+ self.emit_op_arg(ops.BUILD_LIST, 0)
+ add_op = ops.LIST_APPEND
+ tuplecase = True
+ else:
+ self.emit_op_arg(single_op, 0)
+ for elt in elts:
+ elt.walkabout(self)
+ self.emit_op_arg(add_op, 1)
+ if tuplecase:
+ self.emit_op_arg(ops.BUILD_TUPLE_UNPACK, 1)
+ return
+
seen_star = 0
elt_subitems = 0
for i in range(elt_count):
@@ -1279,7 +1301,7 @@
if tup.ctx == ast.Store:
self._visit_assignment(tup, tup.elts, tup.ctx)
elif tup.ctx == ast.Load:
- self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK)
+ self._visit_starunpack(tup, tup.elts, ops.BUILD_TUPLE, ops.BUILD_TUPLE, ops.BUILD_TUPLE_UNPACK, -1)
else:
self.visit_sequence(tup.elts)
@@ -1288,7 +1310,8 @@
if l.ctx == ast.Store:
self._visit_assignment(l, l.elts, l.ctx)
elif l.ctx == ast.Load:
- self._visit_starunpack(l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK)
+ self._visit_starunpack(
+ l, l.elts, ops.BUILD_LIST, ops.BUILD_TUPLE, ops.BUILD_LIST_UNPACK, ops.LIST_APPEND)
else:
self.visit_sequence(l.elts)
@@ -1299,6 +1322,21 @@
is_unpacking = False
all_constant_keys_w = None
if d.values:
+ unpacking_anywhere = False
+ for key in d.keys:
+ if key is None:
+ unpacking_anywhere = True
+ break
+ if not unpacking_anywhere and len(d.keys) > MAX_STACKDEPTH_CONTAINERS:
+ # do it in a small amount of stack
+ self.emit_op_arg(ops.BUILD_MAP, 0)
+ for i in range(len(d.values)):
+ d.values[i].walkabout(self)
+ key = d.keys[i]
+ assert key is not None
+ key.walkabout(self)
+ self.emit_op_arg(ops.MAP_ADD, 1)
+ return
if len(d.keys) < 0xffff:
all_constant_keys_w = []
for key in d.keys:
@@ -1343,7 +1381,7 @@
is_unpacking = False
def visit_Set(self, s):
- self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK)
+ self._visit_starunpack(s, s.elts, ops.BUILD_SET, ops.BUILD_SET, ops.BUILD_SET_UNPACK, ops.SET_ADD)
def visit_Name(self, name):
self.update_position(name.lineno)
diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -302,6 +302,9 @@
node = tup.elts[i]
w_const = node.as_constant()
if w_const is None:
+ new_elts = self._optimize_constant_star_unpacks(tup.elts)
+ if new_elts is not None:
+ return ast.Tuple(new_elts, ast.Load, tup.lineno, tup.col_offset)
return tup
consts_w[i] = w_const
# intern the string constants packed into the tuple here,
@@ -314,6 +317,62 @@
w_consts = self.space.newtuple(consts_w)
return ast.Constant(w_consts, tup.lineno, tup.col_offset)
+ def _make_starred_tuple_const(self, consts_w, firstelt):
+ w_consts = self.space.newtuple(consts_w[:])
+ return ast.Starred(ast.Constant(
+ w_consts, firstelt.lineno, firstelt.col_offset),
+ ast.Load, firstelt.lineno, firstelt.col_offset)
+
+ def _optimize_constant_star_unpacks(self, elts):
+ # turn (1, 2, 3, *a) into (*(1, 2, 3), *a) with a constant (1, 2, 3)
+ # or similarly, for lists
+ contains_starred = False
+ for i in range(len(elts)):
+ elt = elts[i]
+ if isinstance(elt, ast.Starred):
+ contains_starred = True
+ break
+ if not contains_starred:
+ return None
+ new_elts = []
+ changed = False
+ const_since_last_star_w = []
+ after_last_star_index = 0
+ for i in range(len(elts)):
+ elt = elts[i]
+ if isinstance(elt, ast.Starred):
+ if const_since_last_star_w is not None:
+ firstelt = elts[after_last_star_index]
+ new_elts.append(self._make_starred_tuple_const(
+ const_since_last_star_w, firstelt))
+ changed = True
+ const_since_last_star_w = []
+ after_last_star_index = i + 1
+ new_elts.append(elt)
+ elif const_since_last_star_w is not None:
+ w_const = elt.as_constant()
+ if w_const is None:
+ new_elts.extend(elts[after_last_star_index:i + 1])
+ const_since_last_star_w = None
+ else:
+ const_since_last_star_w.append(w_const)
+ else:
+ new_elts.append(elt)
+ if after_last_star_index != len(elts) and const_since_last_star_w is not None:
+ firstelt = elts[after_last_star_index]
+ new_elts.append(self._make_starred_tuple_const(
+ const_since_last_star_w, firstelt))
+ changed = True
+ if changed:
+ return new_elts
+
+ def visit_List(self, l):
+ if l.ctx == ast.Load and l.elts:
+ new_elts = self._optimize_constant_star_unpacks(l.elts)
+ if new_elts:
+ return ast.List(new_elts, ast.Load, l.lineno, l.col_offset)
+ return l
+
def visit_Subscript(self, subs):
if subs.ctx == ast.Load:
w_obj = subs.value.as_constant()
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1099,14 +1099,23 @@
return a, b, c
"""
yield self.st, func, "f()", (1, [2, 3], 4)
+
+ def test_unpacking_while_building(self):
func = """def f():
b = [4,5,6]
- c = 7
- a = [*b, c]
+ a = (*b, 7)
+ return a
+ """
+ yield self.st, func, "f()", (4, 5, 6, 7)
+
+ func = """def f():
+ b = [4,5,6]
+ a = [*b, 7]
return a
"""
yield self.st, func, "f()", [4, 5, 6, 7]
+
def test_extended_unpacking_fail(self):
exc = py.test.raises(SyntaxError, self.simple_test, "*a, *b = [1, 2]",
None, None).value
@@ -1542,4 +1551,60 @@
counts = self.count_instructions(source)
assert ops.BUILD_TUPLE not in counts
+ def test_constant_tuples_star(self):
+ source = """def f(a, c):
+ return (u"a", 1, *a, 3, 5, 3, *c)
+ """
+ counts = self.count_instructions(source)
+ assert ops.BUILD_TUPLE not in counts
+ source = """def f(a, c, d):
+ return (u"a", 1, *a, c, 1, *d, 1, 2, 3)
+ """
+ counts = self.count_instructions(source)
+ assert counts[ops.BUILD_TUPLE] == 1
+
+ def test_constant_list_star(self):
+ source = """def f(a, c):
+ return [u"a", 1, *a, 3, 5, 3, *c]
+ """
+ counts = self.count_instructions(source)
+ assert ops.BUILD_TUPLE not in counts
+
+ source = """def f(a, c, d):
+ return [u"a", 1, *a, c, 1, *d, 1, 2, 3]
+ """
+ counts = self.count_instructions(source)
+ assert counts[ops.BUILD_TUPLE] == 1
+
+
+class TestHugeStackDepths:
+ def run_and_check_stacksize(self, source):
+ space = self.space
+ code = compile_with_astcompiler("a = " + source, 'exec', space)
+ assert code.co_stacksize < 100
+ w_dict = space.newdict()
+ code.exec_code(space, w_dict, w_dict)
+ return space.getitem(w_dict, space.newtext("a"))
+
+ def test_tuple(self):
+ source = "(" + ",".join([str(i) for i in range(200)]) + ")\n"
+ w_res = self.run_and_check_stacksize(source)
+ assert self.space.unwrap(w_res) == tuple(range(200))
+
+ def test_list(self):
+ source = "[" + ",".join([str(i) for i in range(200)]) + "]\n"
+ w_res = self.run_and_check_stacksize(source)
+ assert self.space.unwrap(w_res) == range(200)
+
+ def test_set(self):
+ source = "{" + ",".join([str(i) for i in range(200)]) + "}\n"
+ w_res = self.run_and_check_stacksize(source)
+ space = self.space
+ assert [space.int_w(w_x)
+ for w_x in space.unpackiterable(w_res)] == range(200)
+
+ def test_dict(self):
+ source = "{" + ",".join(['%s: None' % (i, ) for i in range(200)]) + "}\n"
+ w_res = self.run_and_check_stacksize(source)
+ assert self.space.unwrap(w_res) == dict.fromkeys(range(200))
diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -662,14 +662,17 @@
def wcharpsize2utf8(space, wcharp, size):
"""Safe version of rffi.wcharpsize2utf8.
- Raises app-level rutf8.OutOfRange if any wchar value is outside the valid
+ Raises app-level ValueError if any wchar value is outside the valid
codepoint range.
"""
try:
return rffi.wcharpsize2utf8(wcharp, size)
except rutf8.OutOfRange as e:
- raise oefmt(space.w_ValueError,
- "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
+ raise wrap_unicode_out_of_range_error(space, e)
+
+def wrap_unicode_out_of_range_error(space, e):
+ raise oefmt(space.w_ValueError,
+ "character %s is not in range [U+0000; U+10ffff]", 'U+%x' % e.code)
# ____________________________________________________________
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -5,6 +5,8 @@
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import interp_attrproperty
from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
+from pypy.interpreter.unicodehelper import wrap_unicode_out_of_range_error
from rpython.rlib.clibffi import *
from rpython.rlib.objectmodel import we_are_translated
@@ -598,10 +600,13 @@
if address == 0:
return space.w_None
wcharp_addr = rffi.cast(rffi.CWCHARP, address)
- if maxlength == -1:
- s, lgt = rffi.wcharp2utf8(wcharp_addr)
- else:
- s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+ try:
+ if maxlength == -1:
+ s, lgt = rffi.wcharp2utf8(wcharp_addr)
+ else:
+ s, lgt = rffi.wcharp2utf8n(wcharp_addr, maxlength)
+ except rutf8.OutOfRange as e:
+ raise wrap_unicode_out_of_range_error(space, e)
return space.newutf8(s, lgt)
@unwrap_spec(address=r_uint, maxlength=int)
@@ -617,7 +622,7 @@
return wcharp2unicode(space, address)
elif maxlength < 0:
maxlength = 0
- s = rffi.wcharpsize2utf8(rffi.cast(rffi.CWCHARP, address), maxlength)
+ s = wcharpsize2utf8(space, rffi.cast(rffi.CWCHARP, address), maxlength)
return space.newutf8(s, maxlength)
@unwrap_spec(address=r_uint, newcontent='bufferstr', offset=int, size=int)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -1225,6 +1225,23 @@
lib = _rawffi.CDLL(self.lib_name)
assert lib.name == self.lib_name
+ def test_wcharp2rawunicode(self):
+ import _rawffi
+ A = _rawffi.Array('i')
+ arg = A(1)
+ arg[0] = 0x1234
+ u = _rawffi.wcharp2rawunicode(arg.itemaddress(0))
+ assert u == u'\u1234'
+ u = _rawffi.wcharp2rawunicode(arg.itemaddress(0), 1)
+ assert u == u'\u1234'
+ arg[0] = -1
+ raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+ raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+ arg[0] = 0x110000
+ raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0))
+ raises(ValueError, _rawffi.wcharp2rawunicode, arg.itemaddress(0), 1)
+ arg.free()
+
class AppTestAutoFree:
spaceconfig = dict(usemodules=['_rawffi', 'struct'])
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -14,6 +14,7 @@
interp2app, interpindirect2app, unwrap_spec)
from pypy.interpreter.typedef import (
GetSetProperty, TypeDef, make_weakref_descr)
+from pypy.interpreter.unicodehelper import wcharpsize2utf8
@unwrap_spec(typecode='text')
@@ -509,7 +510,8 @@
if s < 0:
s = 0
buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
- return space.newutf8(rffi.wcharpsize2utf8(buf, s), s)
+ utf8 = wcharpsize2utf8(space, buf, s)
+ return space.newutf8(utf8, s)
else:
raise oefmt(space.w_ValueError,
"tounicode() may only be called on type 'u' arrays")
@@ -767,8 +769,16 @@
if self.len == 0:
return space.newtext("array('%s')" % self.typecode)
elif self.typecode == "u":
- r = space.repr(self.descr_tounicode(space))
- s = "array('%s', %s)" % (self.typecode, space.text_w(r))
+ try:
+ w_unicode = self.descr_tounicode(space)
+ except OperationError as e:
+ if not e.match(space, space.w_ValueError):
+ raise
+ w_exc_value = e.get_w_value(space)
+ r = "<%s>" % (space.text_w(w_exc_value),)
+ else:
+ r = space.text_w(space.repr(w_unicode))
+ s = "array('%s', %s)" % (self.typecode, r)
return space.newtext(s)
else:
r = space.repr(self.descr_tolist(space))
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -893,10 +893,16 @@
assert repr(mya('i', (1, 2, 3))) == "array('i', [1, 2, 3])"
def test_unicode_outofrange(self):
- a = self.array('u', u'\x01\u263a\x00\ufeff')
- b = self.array('u', u'\x01\u263a\x00\ufeff')
+ input_unicode = u'\x01\u263a\x00\ufeff'
+ a = self.array('u', input_unicode)
+ b = self.array('u', input_unicode)
b.byteswap()
raises(ValueError, "a != b")
+ assert str(a) == "array('u', %r)" % (input_unicode,)
+ assert str(b) == ("array('u', <character U+1000000 is not in"
+ " range [U+0000; U+10ffff]>)")
+ assert a.tounicode() == input_unicode
+ raises(ValueError, b.tounicode) # doesn't work
def test_weakref(self):
import weakref
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -20,7 +20,7 @@
# inherits from W_Root for internal reasons. Such instances don't
# have a typedef at all (or have a null typedef after translation).
if not we_are_translated():
- if not hasattr(w_obj, 'typedef'):
+ if getattr(w_obj, 'typedef', None) is None:
return None
else:
if w_obj is None or not w_obj.typedef:
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1023,7 +1023,7 @@
def wcharpsize2utf8(w, size):
""" Helper to convert WCHARP pointer to utf8 in one go.
Equivalent to wcharpsize2unicode().encode("utf8")
- Raises ValueError if characters are outside range(0x110000)!
+ Raises rutf8.OutOfRange if characters are outside range(0x110000)!
"""
from rpython.rlib import rutf8
@@ -1033,6 +1033,9 @@
return s.build()
def wcharp2utf8(w):
+ """
+ Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+ """
from rpython.rlib import rutf8
s = rutf8.Utf8StringBuilder()
@@ -1043,6 +1046,9 @@
return s.build(), i
def wcharp2utf8n(w, maxlen):
+ """
+ Raises rutf8.OutOfRange if characters are outside range(0x110000)!
+ """
from rpython.rlib import rutf8
s = rutf8.Utf8StringBuilder(maxlen)
More information about the pypy-commit
mailing list