From pypy.commits at gmail.com Thu Mar 1 05:07:26 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 01 Mar 2018 02:07:26 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Merged in alcarithemad/pypy/pep526 (pull request #595) Message-ID: <5a97d0de.2196df0a.42e87.2f31@mx.google.com> Author: Armin Rigo Branch: py3.6 Changeset: r93928:552eccd54e83 Date: 2018-03-01 10:06 +0000 http://bitbucket.org/pypy/pypy/changeset/552eccd54e83/ Log: Merged in alcarithemad/pypy/pep526 (pull request #595) PEP 526: Type annotations for variables Approved-by: Armin Rigo Approved-by: Amaury Forgeot d'Arc diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py --- a/lib-python/3/opcode.py +++ b/lib-python/3/opcode.py @@ -121,7 +121,7 @@ def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) - +def_op('SETUP_ANNOTATIONS', 85) def_op('YIELD_VALUE', 86) def_op('POP_BLOCK', 87) def_op('END_FINALLY', 88) @@ -171,6 +171,7 @@ haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) +name_op('STORE_ANNOTATION', 127) # Index in name list def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -577,6 +577,7 @@ mainmodule = type(sys)('__main__') mainmodule.__loader__ = sys.__loader__ mainmodule.__builtins__ = os.__builtins__ + mainmodule.__annotations__ = {} sys.modules['__main__'] = mainmodule if not no_site: diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -690,6 +690,9 @@ ops.POP_JUMP_IF_FALSE: -1, ops.JUMP_IF_NOT_DEBUG: 0, + ops.SETUP_ANNOTATIONS: 0, + ops.STORE_ANNOTATION: -1, + # TODO ops.BUILD_LIST_FROM_ARG: 1, diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -339,6 +339,8 @@ return Assign.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_AugAssign): return AugAssign.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_AnnAssign): + return AnnAssign.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_For): return For.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_AsyncFor): @@ -816,6 +818,64 @@ State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value']) +class AnnAssign(stmt): + + def __init__(self, target, annotation, value, simple, lineno, col_offset): + self.target = target + self.annotation = annotation + self.value = value + self.simple = simple + stmt.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_AnnAssign(self) + + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.annotation = self.annotation.mutate_over(visitor) + if self.value: + self.value = self.value.mutate_over(visitor) + return visitor.visit_AnnAssign(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_AnnAssign) + w_target = self.target.to_object(space) # expr + space.setattr(w_node, space.newtext('target'), w_target) + w_annotation = self.annotation.to_object(space) # expr + space.setattr(w_node, space.newtext('annotation'), w_annotation) + w_value = self.value.to_object(space) if self.value is not None else space.w_None # expr + space.setattr(w_node, space.newtext('value'), w_value) + w_simple = space.newint(self.simple) # int + space.setattr(w_node, space.newtext('simple'), w_simple) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_target = get_field(space, w_node, 'target', False) + w_annotation = get_field(space, w_node, 'annotation', False) + w_value = get_field(space, w_node, 'value', True) + w_simple = get_field(space, w_node, 'simple', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _target = expr.from_object(space, w_target) + if _target is None: + raise_required_value(space, w_node, 'target') + _annotation = expr.from_object(space, w_annotation) + if _annotation is None: + raise_required_value(space, w_node, 'annotation') + _value = expr.from_object(space, w_value) + _simple = obj_to_int(space, w_simple) + _lineno = obj_to_int(space, w_lineno) + _col_offset = obj_to_int(space, w_col_offset) + return AnnAssign(_target, _annotation, _value, _simple, _lineno, _col_offset) + +State.ast_type('AnnAssign', 'stmt', ['target', 'annotation', 'value', 'simple']) + + class For(stmt): def __init__(self, target, iter, body, orelse, lineno, col_offset): @@ -3673,10 +3733,11 @@ class comprehension(AST): - def __init__(self, target, iter, ifs): + def __init__(self, target, iter, ifs, is_async): self.target = target self.iter = iter self.ifs = ifs + self.is_async = is_async def mutate_over(self, visitor): self.target = self.target.mutate_over(visitor) @@ -3702,6 +3763,8 @@ ifs_w = [node.to_object(space) for node in self.ifs] # expr w_ifs = space.newlist(ifs_w) space.setattr(w_node, space.newtext('ifs'), w_ifs) + w_is_async = space.newint(self.is_async) # int + space.setattr(w_node, space.newtext('is_async'), w_is_async) return w_node @staticmethod @@ -3709,6 +3772,7 @@ w_target = get_field(space, w_node, 'target', False) w_iter = get_field(space, w_node, 'iter', False) w_ifs = get_field(space, w_node, 'ifs', False) + w_is_async = get_field(space, w_node, 'is_async', False) _target = expr.from_object(space, w_target) if _target is None: raise_required_value(space, w_node, 'target') @@ -3717,9 +3781,10 @@ raise_required_value(space, w_node, 'iter') ifs_w = space.unpackiterable(w_ifs) _ifs = [expr.from_object(space, w_item) for w_item in ifs_w] - return comprehension(_target, _iter, _ifs) - -State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs']) + _is_async = obj_to_int(space, w_is_async) + return comprehension(_target, _iter, _ifs, _is_async) + +State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs', 'is_async']) class excepthandler(AST): @@ -4066,6 +4131,8 @@ return self.default_visitor(node) def visit_AugAssign(self, node): return self.default_visitor(node) + def visit_AnnAssign(self, node): + return self.default_visitor(node) def visit_For(self, node): return self.default_visitor(node) def visit_AsyncFor(self, node): @@ -4230,6 +4297,12 @@ node.target.walkabout(self) node.value.walkabout(self) + def visit_AnnAssign(self, node): + node.target.walkabout(self) + node.annotation.walkabout(self) + if node.value: + node.value.walkabout(self) + def visit_For(self, node): node.target.walkabout(self) node.iter.walkabout(self) diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -737,6 +737,7 @@ raise AssertionError("unknown statment type") def handle_expr_stmt(self, stmt): + from pypy.interpreter.pyparser.parser import AbstractNonterminal if stmt.num_children() == 1: expression = self.handle_testlist(stmt.get_child(0)) return ast.Expr(expression, stmt.get_lineno(), stmt.get_column()) @@ -754,6 +755,44 @@ operator = augassign_operator_map[op_str] return ast.AugAssign(target_expr, operator, value_expr, stmt.get_lineno(), stmt.get_column()) + elif stmt.get_child(1).type == syms.annassign: + # Variable annotation (PEP 526), which may or may not include assignment. + target = stmt.get_child(0) + target_expr = self.handle_testlist(target) + simple = 0 + # target is a name, nothing funky + if isinstance(target_expr, ast.Name): + # The PEP demands that `(x): T` be treated differently than `x: T` + # however, the parser does not easily expose the wrapping parens, which are a no-op + # they are elided by handle_testlist if they existed. + # so here we walk down the parse tree until we hit a terminal, and check whether it's + # a left paren + simple_test = target.get_child(0) + while isinstance(simple_test, AbstractNonterminal): + simple_test = simple_test.get_child(0) + if simple_test.type != tokens.LPAR: + simple = 1 + # subscripts are allowed with nothing special + elif isinstance(target_expr, ast.Subscript): + pass + # attributes are also fine here + elif isinstance(target_expr, ast.Attribute): + pass + # tuples and lists get special error messages + elif isinstance(target_expr, ast.Tuple): + self.error("only single target (not tuple) can be annotated", target) + elif isinstance(target_expr, ast.List): + self.error("only single target (not list) can be annotated", target) + # and everything else gets a generic error + else: + self.error("illegal target for annoation", target) + self.set_context(target_expr, ast.Store) + second = stmt.get_child(1) + annotation = self.handle_expr(second.get_child(1)) + value_expr = None + if second.num_children() == 4: + value_expr = self.handle_testlist(second.get_child(-1)) + return ast.AnnAssign(target_expr, annotation, value_expr, simple, stmt.get_lineno(), stmt.get_column()) else: # Normal assignment. targets = [] @@ -1315,7 +1354,8 @@ expr = self.handle_expr(comp_node.get_child(3)) assert isinstance(expr, ast.expr) if for_node.num_children() == 1: - comp = ast.comprehension(for_targets[0], expr, None) + # FIXME: determine whether this is actually async + comp = ast.comprehension(for_targets[0], expr, None, 0) else: # Modified in python2.7, see http://bugs.python.org/issue6704 # Fixing unamed tuple location @@ -1324,7 +1364,8 @@ col = expr_node.col_offset line = expr_node.lineno target = ast.Tuple(for_targets, ast.Store, line, col) - comp = ast.comprehension(target, expr, None) + # FIXME: determine whether this is actually async + comp = ast.comprehension(target, expr, None, 0) if comp_node.num_children() == 5: comp_node = comp_iter = comp_node.get_child(4) assert comp_iter.type == syms.comp_iter diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -299,6 +299,12 @@ else: return False + def _maybe_setup_annotations(self): + # if the scope contained an annotated variable assignemt, + # this will emit the requisite SETUP_ANNOTATIONS + if self.scope.contains_annotated and not isinstance(self, AbstractFunctionCodeGenerator): + self.emit_op(ops.SETUP_ANNOTATIONS) + def visit_Module(self, mod): if not self._handle_body(mod.body): self.first_lineno = self.lineno = 1 @@ -925,6 +931,66 @@ self.visit_sequence(targets) return True + def _annotation_evaluate(self, item): + # PEP 526 requires that some things be evaluated, to avoid bugs + # where a non-assigning variable annotation references invalid items + # this is effectively a NOP, but will fail if e.g. item is an + # Attribute and one of the chained names does not exist + item.walkabout(self) + self.emit_op(ops.POP_TOP) + + def _annotation_eval_slice(self, target): + if isinstance(target, ast.Index): + self._annotation_evaluate(target.value) + elif isinstance(target, ast.Slice): + for val in [target.lower, target.upper, target.step]: + if val: + self._annotation_evaluate(val) + elif isinstance(target, ast.ExtSlice): + for val in target.dims: + if isinstance(val, ast.Index) or isinstance(val, ast.Slice): + self._annotation_eval_slice(val) + else: + self.error("Invalid nested slice", val) + else: + self.error("Invalid slice?", target) + + def visit_AnnAssign(self, assign): + self.update_position(assign.lineno, True) + target = assign.target + # if there's an assignment to be done, do it + if assign.value: + assign.value.walkabout(self) + target.walkabout(self) + # the PEP requires that certain parts of the target be evaluated at runtime + # to avoid silent annotation-related errors + if isinstance(target, ast.Name): + # if it's just a simple name and we're not in a function, store + # the annotation in __annotations__ + if assign.simple and not isinstance(self.scope, symtable.FunctionScope): + assign.annotation.walkabout(self) + name = target.id + self.emit_op_arg(ops.STORE_ANNOTATION, self.add_name(self.names, name)) + elif isinstance(target, ast.Attribute): + # the spec requires that `a.b: int` evaluates `a` + # and in a non-function scope, also evaluates `int` + # (N.B.: if the target is of the form `a.b.c`, `a.b` will be evaluated) + if not assign.value: + attr = target.value + self._annotation_evaluate(attr) + elif isinstance(target, ast.Subscript): + # similar to the above, `a[0:5]: int` evaluates the name and the slice argument + # and if not in a function, also evaluates the annotation + sl = target.slice + self._annotation_evaluate(target.value) + self._annotation_eval_slice(sl) + else: + self.error("can't handle annotation with %s" % (target,), target) + # if this is not in a function, evaluate the annotation + if not (assign.simple or isinstance(self.scope, symtable.FunctionScope)): + self._annotation_evaluate(assign.annotation) + + def visit_With(self, wih): self.update_position(wih.lineno, True) self.handle_withitem(wih, 0, is_async=False) @@ -1527,6 +1593,7 @@ symbols, compile_info, qualname=None) def _compile(self, tree): + self._maybe_setup_annotations() tree.walkabout(self) def _get_code_flags(self): @@ -1656,6 +1723,7 @@ w_qualname = self.space.newtext(self.qualname) self.load_const(w_qualname) self.name_op("__qualname__", ast.Store) + self._maybe_setup_annotations() # compile the body proper self._handle_body(cls.body) # return the (empty) __class__ cell diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -12,6 +12,7 @@ SYM_PARAM = 2 << 1 SYM_NONLOCAL = 2 << 2 SYM_USED = 2 << 3 +SYM_ANNOTATED = 2 << 4 SYM_BOUND = (SYM_PARAM | SYM_ASSIGNED) # codegen.py actually deals with these: @@ -44,6 +45,7 @@ self.child_has_free = False self.nested = False self.doc_removable = False + self.contains_annotated = False self._in_try_body_depth = 0 def lookup(self, name): @@ -139,7 +141,7 @@ self.free_vars.append(name) free[name] = None self.has_free = True - elif flags & SYM_BOUND: + elif flags & (SYM_BOUND | SYM_ANNOTATED): self.symbols[name] = SCOPE_LOCAL local[name] = None try: @@ -420,6 +422,20 @@ self.scope.note_return(ret) ast.GenericASTVisitor.visit_Return(self, ret) + def visit_AnnAssign(self, assign): + # __annotations__ is not setup or used in functions. + if not isinstance(self.scope, FunctionScope): + self.scope.contains_annotated = True + target = assign.target + if isinstance(target, ast.Name): + scope = SYM_ANNOTATED + name = target.id + if assign.value: + scope |= SYM_USED + self.note_symbol(name, scope) + else: + target.walkabout(self) + def visit_ClassDef(self, clsdef): self.note_symbol(clsdef.name, SYM_ASSIGNED) self.visit_sequence(clsdef.bases) @@ -485,10 +501,13 @@ msg = "name '%s' is nonlocal and global" % (name,) raise SyntaxError(msg, glob.lineno, glob.col_offset) - if old_role & (SYM_USED | SYM_ASSIGNED): + if old_role & (SYM_USED | SYM_ASSIGNED | SYM_ANNOTATED): if old_role & SYM_ASSIGNED: msg = "name '%s' is assigned to before global declaration"\ % (name,) + elif old_role & SYM_ANNOTATED: + msg = "annotated name '%s' can't be global" \ + % (name,) else: msg = "name '%s' is used prior to global declaration" % \ (name,) @@ -498,6 +517,7 @@ def visit_Nonlocal(self, nonl): for name in nonl.names: old_role = self.scope.lookup_role(name) + print(name, old_role) msg = "" if old_role & SYM_GLOBAL: msg = "name '%s' is nonlocal and global" % (name,) @@ -505,6 +525,9 @@ msg = "name '%s' is parameter and nonlocal" % (name,) if isinstance(self.scope, ModuleScope): msg = "nonlocal declaration not allowed at module level" + if old_role & SYM_ANNOTATED: + msg = "annotated name '%s' can't be nonlocal" \ + % (name,) if msg is not "": raise SyntaxError(msg, nonl.lineno, nonl.col_offset) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -614,6 +614,44 @@ assert len(dec.args) == 2 assert dec.keywords is None + def test_annassign(self): + simple = self.get_first_stmt('a: int') + assert isinstance(simple, ast.AnnAssign) + assert isinstance(simple.target, ast.Name) + assert simple.target.ctx == ast.Store + assert isinstance(simple.annotation, ast.Name) + assert simple.value == None + assert simple.simple == 1 + + with_value = self.get_first_stmt('x: str = "test"') + assert isinstance(with_value, ast.AnnAssign) + assert isinstance(with_value.value, ast.Str) + assert self.space.eq_w(with_value.value.s, self.space.wrap("test")) + + not_simple = self.get_first_stmt('(a): int') + assert isinstance(not_simple, ast.AnnAssign) + assert isinstance(not_simple.target, ast.Name) + assert not_simple.target.ctx == ast.Store + assert not_simple.simple == 0 + + attrs = self.get_first_stmt('a.b.c: int') + assert isinstance(attrs, ast.AnnAssign) + assert isinstance(attrs.target, ast.Attribute) + + subscript = self.get_first_stmt('a[0:2]: int') + assert isinstance(subscript, ast.AnnAssign) + assert isinstance(subscript.target, ast.Subscript) + + exc_tuple = py.test.raises(SyntaxError, self.get_ast, 'a, b: int').value + assert exc_tuple.msg == "only single target (not tuple) can be annotated" + + exc_list = py.test.raises(SyntaxError, self.get_ast, '[]: int').value + assert exc_list.msg == "only single target (not list) can be annotated" + + exc_bad_target = py.test.raises(SyntaxError, self.get_ast, '{}: int').value + assert exc_bad_target.msg == "illegal target for annoation" + + def test_augassign(self): aug_assigns = ( ("+=", ast.Add), diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -486,6 +486,37 @@ scp = self.mod_scope("with x: pass") assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL + def test_annotation_global(self): + src_global = ("def f():\n" + " x: int\n" + " global x\n") + exc_global = py.test.raises(SyntaxError, self.func_scope, src_global).value + assert exc_global.msg == "annotated name 'x' can't be global" + assert exc_global.lineno == 3 + + def test_annotation_nonlocal(self): + src_nonlocal = ("def f():\n" + " x: int\n" + " nonlocal x\n") + exc_nonlocal = py.test.raises(SyntaxError, self.func_scope, src_nonlocal).value + assert exc_nonlocal.msg == "annotated name 'x' can't be nonlocal" + assert exc_nonlocal.lineno == 3 + + def test_annotation_assignment(self): + scp = self.mod_scope("x: int = 1") + assert scp.contains_annotated == True + + scp2 = self.mod_scope("x = 1") + assert scp2.contains_annotated == False + + fscp = self.func_scope("def f(): x: int") + assert fscp.contains_annotated == False + assert fscp.lookup("x") == symtable.SCOPE_LOCAL + + def test_nonsimple_annotation(self): + fscp = self.func_scope("def f(): implicit_global[0]: int") + assert fscp.lookup("implicit_global") == symtable.SCOPE_GLOBAL_IMPLICIT + def test_issue13343(self): scp = self.mod_scope("lambda *, k1=x, k2: None") assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl --- a/pypy/interpreter/astcompiler/tools/Python.asdl +++ b/pypy/interpreter/astcompiler/tools/Python.asdl @@ -17,6 +17,7 @@ stmt* body, expr* decorator_list, expr? returns) | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, expr* bases, keyword* keywords, @@ -27,6 +28,8 @@ | Delete(expr* targets) | Assign(expr* targets, expr value) | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) -- use 'orelse' because else is a keyword in target languages | For(expr target, expr iter, stmt* body, stmt* orelse) @@ -107,7 +110,7 @@ cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - comprehension = (expr target, expr iter, expr* ifs) + comprehension = (expr target, expr iter, expr* ifs, int is_async) excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) attributes (int lineno, int col_offset) diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py --- a/pypy/interpreter/astcompiler/validate.py +++ b/pypy/interpreter/astcompiler/validate.py @@ -212,6 +212,12 @@ self._validate_exprs(node.targets, ast.Store) self._validate_expr(node.value) + def visit_AnnAssign(self, node): + self._validate_expr(node.target, ast.Store) + self._validate_expr(node.annotation) + if node.value: + self._validate_expr(node.value) + def visit_AugAssign(self, node): self._validate_expr(node.target, ast.Store) self._validate_expr(node.value) diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py --- a/pypy/interpreter/main.py +++ b/pypy/interpreter/main.py @@ -13,6 +13,8 @@ raise mainmodule = module.Module(space, w_main) space.setitem(w_modules, w_main, mainmodule) + w_annotations = space.newdict() + space.setitem_str(mainmodule.w_dict, '__annotations__', w_annotations) return mainmodule diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -292,6 +292,10 @@ self.DELETE_DEREF(oparg, next_instr) elif opcode == opcodedesc.DELETE_FAST.index: self.DELETE_FAST(oparg, next_instr) + elif opcode == opcodedesc.SETUP_ANNOTATIONS.index: + self.SETUP_ANNOTATIONS(oparg, next_instr) + elif opcode == opcodedesc.STORE_ANNOTATION.index: + self.STORE_ANNOTATION(oparg, next_instr) elif opcode == opcodedesc.DELETE_GLOBAL.index: self.DELETE_GLOBAL(oparg, next_instr) elif opcode == opcodedesc.DELETE_NAME.index: @@ -947,6 +951,18 @@ varname) self.locals_cells_stack_w[varindex] = None + def SETUP_ANNOTATIONS(self, oparg, next_instr): + w_locals = self.getorcreatedebug().w_locals + if not self.space.finditem_str(w_locals, '__annotations__'): + w_annotations = self.space.newdict() + self.space.setitem_str(w_locals, '__annotations__', w_annotations) + + def STORE_ANNOTATION(self, varindex, next_instr): + varname = self.getname_u(varindex) + w_newvalue = self.popvalue() + self.space.setitem_str(self.getorcreatedebug().w_locals.getitem_str('__annotations__'), varname, + w_newvalue) + def BUILD_TUPLE(self, itemcount, next_instr): items = self.popvalues(itemcount) w_tuple = self.space.newtuple(items) diff --git a/pypy/interpreter/pyparser/data/Grammar3.6 b/pypy/interpreter/pyparser/data/Grammar3.6 new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/data/Grammar3.6 @@ -0,0 +1,149 @@ +# Grammar for Python + +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ + +# Start symbols for the grammar: +# single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; +# eval_input is the input for the eval() functions. +# NB: compound_stmt in single_input is followed by extra NEWLINE! +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER +eval_input: testlist NEWLINE* ENDMARKER + +decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE +decorators: decorator+ +decorated: decorators (classdef | funcdef | async_funcdef) + +async_funcdef: ASYNC funcdef +funcdef: 'def' NAME parameters ['->' test] ':' suite + +parameters: '(' [typedargslist] ')' +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) +tfpdef: NAME [':' test] +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] +) +vfpdef: NAME + +stmt: simple_stmt | compound_stmt +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | nonlocal_stmt | assert_stmt) +expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +annassign: ':' test ['=' test] +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') +# For normal and annotated assignments, additional restrictions enforced by the interpreter +del_stmt: 'del' exprlist +pass_stmt: 'pass' +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +break_stmt: 'break' +continue_stmt: 'continue' +return_stmt: 'return' [testlist] +yield_stmt: yield_expr +raise_stmt: 'raise' [test ['from' test]] +import_stmt: import_name | import_from +import_name: 'import' dotted_as_names +# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS +import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) + 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_as_name: NAME ['as' NAME] +dotted_as_name: dotted_name ['as' NAME] +import_as_names: import_as_name (',' import_as_name)* [','] +dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_name: NAME ('.' NAME)* +global_stmt: 'global' NAME (',' NAME)* +nonlocal_stmt: 'nonlocal' NAME (',' NAME)* +assert_stmt: 'assert' test [',' test] + +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +while_stmt: 'while' test ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +try_stmt: ('try' ':' suite + ((except_clause ':' suite)+ + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' with_item (',' with_item)* ':' suite +with_item: test ['as' expr] +# NB compile.c makes sure that the default except clause is last +except_clause: 'except' [test ['as' NAME]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + +test: or_test ['if' or_test 'else' test] | lambdef +test_nocond: or_test | lambdef_nocond +lambdef: 'lambda' [varargslist] ':' test +lambdef_nocond: 'lambda' [varargslist] ':' test_nocond +or_test: and_test ('or' and_test)* +and_test: not_test ('and' not_test)* +not_test: 'not' not_test | comparison +comparison: expr (comp_op expr)* +# <> isn't actually a valid comparison operator in Python. It's here for the +# sake of a __future__ import described in PEP 401 (which really works :-) +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr +expr: xor_expr ('|' xor_expr)* +xor_expr: and_expr ('^' and_expr)* +and_expr: shift_expr ('&' shift_expr)* +shift_expr: arith_expr (('<<'|'>>') arith_expr)* +arith_expr: term (('+'|'-') term)* +term: factor (('*'|'@'|'/'|'%'|'//') factor)* +factor: ('+'|'-'|'~') factor | power +power: atom_expr ['**' factor] +atom_expr: [AWAIT] atom trailer* +atom: ('(' [yield_expr|testlist_comp] ')' | + '[' [testlist_comp] ']' | + '{' [dictorsetmaker] '}' | + NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') +testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +subscriptlist: subscript (',' subscript)* [','] +subscript: test | [test] ':' [test] [sliceop] +sliceop: ':' [test] +exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +testlist: test (',' test)* [','] +dictorsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) + +classdef: 'class' NAME ['(' [arglist] ')'] ':' suite + +arglist: argument (',' argument)* [','] + +# The reason that keywords are test nodes instead of NAME is that using NAME +# results in an ambiguity. ast.c makes sure it's a NAME. +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguments are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test '=' test | + '**' test | + '*' test ) + +comp_iter: comp_for | comp_if +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' test_nocond [comp_iter] + +# not used in grammar, but may appear in "node" passed from Parser to Compiler +encoding_decl: NAME + +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist diff --git a/pypy/interpreter/pyparser/pygram.py b/pypy/interpreter/pyparser/pygram.py --- a/pypy/interpreter/pyparser/pygram.py +++ b/pypy/interpreter/pyparser/pygram.py @@ -9,7 +9,7 @@ def _get_python_grammar(): here = os.path.dirname(__file__) - fp = open(os.path.join(here, "data", "Grammar3.5")) + fp = open(os.path.join(here, "data", "Grammar3.6")) try: gram_source = fp.read() finally: diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/test_annotations.py @@ -0,0 +1,113 @@ +class AppTestAnnotations: + + def test_toplevel_annotation(self): + # exec because this needs to be in "top level" scope + # whereas the docstring-based tests are inside a function + # (or don't care) + exec("a: int; assert __annotations__['a'] == int") + + def test_toplevel_invalid(self): + exec('try: a: invalid\nexcept NameError: pass\n') + + def test_non_simple_annotation(self): + ''' + class C: + (a): int + assert "a" not in __annotations__ + ''' + + def test_simple_with_target(self): + ''' + class C: + a: int = 1 + assert __annotations__["a"] == int + assert a == 1 + ''' + + def test_attribute_target(self): + ''' + class C: + a = 1 + a.x: int + assert __annotations__ == {} + ''' + + def test_subscript_target(self): + ''' + # ensure that these type annotations don't raise exceptions + # during compilation + class C: + a = 1 + a[0]: int + a[1:2]: int + a[1:2:2]: int + a[1:2:2,...]: int + assert __annotations__ == {} + ''' + + def test_class_annotation(self): + ''' + class C: + a: int + b: str + assert "__annotations__" in locals() + assert C.__annotations__ == {"a": int, "b": str} + ''' + + def test_unevaluated_name(self): + ''' + class C: + def __init__(self): + self.x: invalid_name = 1 + assert self.x == 1 + C() + ''' + + def test_nonexistent_target(self): + ''' + try: + # this is invalid because `y` is undefined + # it should raise a NameError + y[0]: invalid + except NameError: + ... + ''' + + def test_repeated_setup(self): + # each exec will run another SETUP_ANNOTATIONS + # we want to confirm that this doesn't blow away + # the previous __annotations__ + d = {} + exec('a: int', d) + exec('b: int', d) + exec('assert __annotations__ == {"a": int, "b": int}', d) + + def test_function_no___annotations__(self): + ''' + a: int + assert "__annotations__" not in locals() + ''' + + def test_unboundlocal(self): + # a simple variable annotation implies its target is a local + ''' + a: int + try: + print(a) + except UnboundLocalError: + return + assert False + ''' + + def test_reassigned___annotations__(self): + ''' + class C: + __annotations__ = None + try: + a: int + raise + except TypeError: + pass + except: + assert False + ''' From pypy.commits at gmail.com Thu Mar 1 05:07:38 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:38 -0800 (PST) Subject: [pypy-commit] pypy pep526: Update grammer to 3.6 Message-ID: <5a97d0ea.4484df0a.8f707.5a60@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93908:8a37c2a3ef7c Date: 2018-02-10 14:40 -0800 http://bitbucket.org/pypy/pypy/changeset/8a37c2a3ef7c/ Log: Update grammer to 3.6 diff --git a/pypy/interpreter/pyparser/data/Grammar3.6 b/pypy/interpreter/pyparser/data/Grammar3.6 new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/data/Grammar3.6 @@ -0,0 +1,149 @@ +# Grammar for Python + +# NOTE WELL: You should also follow all the steps listed at +# https://devguide.python.org/grammar/ + +# Start symbols for the grammar: +# single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; +# eval_input is the input for the eval() functions. +# NB: compound_stmt in single_input is followed by extra NEWLINE! +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +file_input: (NEWLINE | stmt)* ENDMARKER +eval_input: testlist NEWLINE* ENDMARKER + +decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE +decorators: decorator+ +decorated: decorators (classdef | funcdef | async_funcdef) + +async_funcdef: ASYNC funcdef +funcdef: 'def' NAME parameters ['->' test] ':' suite + +parameters: '(' [typedargslist] ')' +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) +tfpdef: NAME [':' test] +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] +) +vfpdef: NAME + +stmt: simple_stmt | compound_stmt +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | nonlocal_stmt | assert_stmt) +expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +annassign: ':' test ['=' test] +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') +# For normal and annotated assignments, additional restrictions enforced by the interpreter +del_stmt: 'del' exprlist +pass_stmt: 'pass' +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +break_stmt: 'break' +continue_stmt: 'continue' +return_stmt: 'return' [testlist] +yield_stmt: yield_expr +raise_stmt: 'raise' [test ['from' test]] +import_stmt: import_name | import_from +import_name: 'import' dotted_as_names +# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS +import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) + 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_as_name: NAME ['as' NAME] +dotted_as_name: dotted_name ['as' NAME] +import_as_names: import_as_name (',' import_as_name)* [','] +dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_name: NAME ('.' NAME)* +global_stmt: 'global' NAME (',' NAME)* +nonlocal_stmt: 'nonlocal' NAME (',' NAME)* +assert_stmt: 'assert' test [',' test] + +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +async_stmt: ASYNC (funcdef | with_stmt | for_stmt) +if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +while_stmt: 'while' test ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +try_stmt: ('try' ':' suite + ((except_clause ':' suite)+ + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' with_item (',' with_item)* ':' suite +with_item: test ['as' expr] +# NB compile.c makes sure that the default except clause is last +except_clause: 'except' [test ['as' NAME]] +suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + +test: or_test ['if' or_test 'else' test] | lambdef +test_nocond: or_test | lambdef_nocond +lambdef: 'lambda' [varargslist] ':' test +lambdef_nocond: 'lambda' [varargslist] ':' test_nocond +or_test: and_test ('or' and_test)* +and_test: not_test ('and' not_test)* +not_test: 'not' not_test | comparison +comparison: expr (comp_op expr)* +# <> isn't actually a valid comparison operator in Python. It's here for the +# sake of a __future__ import described in PEP 401 (which really works :-) +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr +expr: xor_expr ('|' xor_expr)* +xor_expr: and_expr ('^' and_expr)* +and_expr: shift_expr ('&' shift_expr)* +shift_expr: arith_expr (('<<'|'>>') arith_expr)* +arith_expr: term (('+'|'-') term)* +term: factor (('*'|'@'|'/'|'%'|'//') factor)* +factor: ('+'|'-'|'~') factor | power +power: atom_expr ['**' factor] +atom_expr: [AWAIT] atom trailer* +atom: ('(' [yield_expr|testlist_comp] ')' | + '[' [testlist_comp] ']' | + '{' [dictorsetmaker] '}' | + NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') +testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +subscriptlist: subscript (',' subscript)* [','] +subscript: test | [test] ':' [test] [sliceop] +sliceop: ':' [test] +exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +testlist: test (',' test)* [','] +dictorsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) + +classdef: 'class' NAME ['(' [arglist] ')'] ':' suite + +arglist: argument (',' argument)* [','] + +# The reason that keywords are test nodes instead of NAME is that using NAME +# results in an ambiguity. ast.c makes sure it's a NAME. +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguments are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test '=' test | + '**' test | + '*' test ) + +comp_iter: comp_for | comp_if +comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' test_nocond [comp_iter] + +# not used in grammar, but may appear in "node" passed from Parser to Compiler +encoding_decl: NAME + +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist diff --git a/pypy/interpreter/pyparser/pygram.py b/pypy/interpreter/pyparser/pygram.py --- a/pypy/interpreter/pyparser/pygram.py +++ b/pypy/interpreter/pyparser/pygram.py @@ -9,7 +9,7 @@ def _get_python_grammar(): here = os.path.dirname(__file__) - fp = open(os.path.join(here, "data", "Grammar3.5")) + fp = open(os.path.join(here, "data", "Grammar3.6")) try: gram_source = fp.read() finally: From pypy.commits at gmail.com Thu Mar 1 05:07:42 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:42 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add minimal update to comprehensions to produce valid AST nodes for new asdl. Message-ID: <5a97d0ee.06321c0a.d550c.3489@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93910:89c267ae1715 Date: 2018-02-12 23:29 -0800 http://bitbucket.org/pypy/pypy/changeset/89c267ae1715/ Log: Add minimal update to comprehensions to produce valid AST nodes for new asdl. diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1315,7 +1315,8 @@ expr = self.handle_expr(comp_node.get_child(3)) assert isinstance(expr, ast.expr) if for_node.num_children() == 1: - comp = ast.comprehension(for_targets[0], expr, None) + # FIXME: determine whether this is actually async + comp = ast.comprehension(for_targets[0], expr, None, 0) else: # Modified in python2.7, see http://bugs.python.org/issue6704 # Fixing unamed tuple location @@ -1324,7 +1325,8 @@ col = expr_node.col_offset line = expr_node.lineno target = ast.Tuple(for_targets, ast.Store, line, col) - comp = ast.comprehension(target, expr, None) + # FIXME: determine whether this is actually async + comp = ast.comprehension(target, expr, None, 0) if comp_node.num_children() == 5: comp_node = comp_iter = comp_node.get_child(4) assert comp_iter.type == syms.comp_iter From pypy.commits at gmail.com Thu Mar 1 05:07:40 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:40 -0800 (PST) Subject: [pypy-commit] pypy pep526: Update Python.asdl to 3.6. Message-ID: <5a97d0ec.020a1c0a.6cde2.949b@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93909:031ac5343b06 Date: 2018-02-12 23:28 -0800 http://bitbucket.org/pypy/pypy/changeset/031ac5343b06/ Log: Update Python.asdl to 3.6. diff --git a/pypy/interpreter/astcompiler/tools/Python.asdl b/pypy/interpreter/astcompiler/tools/Python.asdl --- a/pypy/interpreter/astcompiler/tools/Python.asdl +++ b/pypy/interpreter/astcompiler/tools/Python.asdl @@ -17,6 +17,7 @@ stmt* body, expr* decorator_list, expr? returns) | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns) + | ClassDef(identifier name, expr* bases, keyword* keywords, @@ -27,6 +28,8 @@ | Delete(expr* targets) | Assign(expr* targets, expr value) | AugAssign(expr target, operator op, expr value) + -- 'simple' indicates that we annotate simple name without parens + | AnnAssign(expr target, expr annotation, expr? value, int simple) -- use 'orelse' because else is a keyword in target languages | For(expr target, expr iter, stmt* body, stmt* orelse) @@ -107,7 +110,7 @@ cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn - comprehension = (expr target, expr iter, expr* ifs) + comprehension = (expr target, expr iter, expr* ifs, int is_async) excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) attributes (int lineno, int col_offset) From pypy.commits at gmail.com Thu Mar 1 05:07:48 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:48 -0800 (PST) Subject: [pypy-commit] pypy pep526: Update ast.py, generated from Python.asdl. Message-ID: <5a97d0f4.aab6df0a.a8157.fba1@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93913:8cf248660bb4 Date: 2018-02-13 23:22 -0800 http://bitbucket.org/pypy/pypy/changeset/8cf248660bb4/ Log: Update ast.py, generated from Python.asdl. diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -339,6 +339,8 @@ return Assign.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_AugAssign): return AugAssign.from_object(space, w_node) + if space.isinstance_w(w_node, get(space).w_AnnAssign): + return AnnAssign.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_For): return For.from_object(space, w_node) if space.isinstance_w(w_node, get(space).w_AsyncFor): @@ -816,6 +818,64 @@ State.ast_type('AugAssign', 'stmt', ['target', 'op', 'value']) +class AnnAssign(stmt): + + def __init__(self, target, annotation, value, simple, lineno, col_offset): + self.target = target + self.annotation = annotation + self.value = value + self.simple = simple + stmt.__init__(self, lineno, col_offset) + + def walkabout(self, visitor): + visitor.visit_AnnAssign(self) + + def mutate_over(self, visitor): + self.target = self.target.mutate_over(visitor) + self.annotation = self.annotation.mutate_over(visitor) + if self.value: + self.value = self.value.mutate_over(visitor) + return visitor.visit_AnnAssign(self) + + def to_object(self, space): + w_node = space.call_function(get(space).w_AnnAssign) + w_target = self.target.to_object(space) # expr + space.setattr(w_node, space.newtext('target'), w_target) + w_annotation = self.annotation.to_object(space) # expr + space.setattr(w_node, space.newtext('annotation'), w_annotation) + w_value = self.value.to_object(space) if self.value is not None else space.w_None # expr + space.setattr(w_node, space.newtext('value'), w_value) + w_simple = space.newint(self.simple) # int + space.setattr(w_node, space.newtext('simple'), w_simple) + w_lineno = space.newint(self.lineno) # int + space.setattr(w_node, space.newtext('lineno'), w_lineno) + w_col_offset = space.newint(self.col_offset) # int + space.setattr(w_node, space.newtext('col_offset'), w_col_offset) + return w_node + + @staticmethod + def from_object(space, w_node): + w_target = get_field(space, w_node, 'target', False) + w_annotation = get_field(space, w_node, 'annotation', False) + w_value = get_field(space, w_node, 'value', True) + w_simple = get_field(space, w_node, 'simple', False) + w_lineno = get_field(space, w_node, 'lineno', False) + w_col_offset = get_field(space, w_node, 'col_offset', False) + _target = expr.from_object(space, w_target) + if _target is None: + raise_required_value(space, w_node, 'target') + _annotation = expr.from_object(space, w_annotation) + if _annotation is None: + raise_required_value(space, w_node, 'annotation') + _value = expr.from_object(space, w_value) + _simple = obj_to_int(space, w_simple) + _lineno = obj_to_int(space, w_lineno) + _col_offset = obj_to_int(space, w_col_offset) + return AnnAssign(_target, _annotation, _value, _simple, _lineno, _col_offset) + +State.ast_type('AnnAssign', 'stmt', ['target', 'annotation', 'value', 'simple']) + + class For(stmt): def __init__(self, target, iter, body, orelse, lineno, col_offset): @@ -3673,10 +3733,11 @@ class comprehension(AST): - def __init__(self, target, iter, ifs): + def __init__(self, target, iter, ifs, is_async): self.target = target self.iter = iter self.ifs = ifs + self.is_async = is_async def mutate_over(self, visitor): self.target = self.target.mutate_over(visitor) @@ -3702,6 +3763,8 @@ ifs_w = [node.to_object(space) for node in self.ifs] # expr w_ifs = space.newlist(ifs_w) space.setattr(w_node, space.newtext('ifs'), w_ifs) + w_is_async = space.newint(self.is_async) # int + space.setattr(w_node, space.newtext('is_async'), w_is_async) return w_node @staticmethod @@ -3709,6 +3772,7 @@ w_target = get_field(space, w_node, 'target', False) w_iter = get_field(space, w_node, 'iter', False) w_ifs = get_field(space, w_node, 'ifs', False) + w_is_async = get_field(space, w_node, 'is_async', False) _target = expr.from_object(space, w_target) if _target is None: raise_required_value(space, w_node, 'target') @@ -3717,9 +3781,10 @@ raise_required_value(space, w_node, 'iter') ifs_w = space.unpackiterable(w_ifs) _ifs = [expr.from_object(space, w_item) for w_item in ifs_w] - return comprehension(_target, _iter, _ifs) - -State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs']) + _is_async = obj_to_int(space, w_is_async) + return comprehension(_target, _iter, _ifs, _is_async) + +State.ast_type('comprehension', 'AST', ['target', 'iter', 'ifs', 'is_async']) class excepthandler(AST): @@ -4066,6 +4131,8 @@ return self.default_visitor(node) def visit_AugAssign(self, node): return self.default_visitor(node) + def visit_AnnAssign(self, node): + return self.default_visitor(node) def visit_For(self, node): return self.default_visitor(node) def visit_AsyncFor(self, node): @@ -4230,6 +4297,12 @@ node.target.walkabout(self) node.value.walkabout(self) + def visit_AnnAssign(self, node): + node.target.walkabout(self) + node.annotation.walkabout(self) + if node.value: + node.value.walkabout(self) + def visit_For(self, node): node.target.walkabout(self) node.iter.walkabout(self) From pypy.commits at gmail.com Thu Mar 1 05:07:44 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:44 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add WIP test for variable annotation in symtable. Message-ID: <5a97d0f0.02a1df0a.14a1a.dec3@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93911:2ddcba2e06c4 Date: 2018-02-13 23:21 -0800 http://bitbucket.org/pypy/pypy/changeset/2ddcba2e06c4/ Log: Add WIP test for variable annotation in symtable. diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -486,6 +486,34 @@ scp = self.mod_scope("with x: pass") assert scp.lookup("_[1]") == symtable.SCOPE_LOCAL + def test_annotation_global(self): + src_global = ("def f():\n" + " x: int\n" + " global x\n") + exc_global = py.test.raises(SyntaxError, self.func_scope, src_global).value + assert exc_global.msg == "annotated name 'x' can't be global" + assert exc_global.lineno == 3 + + def test_annotation_nonlocal(self): + src_nonlocal = ("def f():\n" + " x: int\n" + " nonlocal x\n") + exc_nonlocal = py.test.raises(SyntaxError, self.func_scope, src_nonlocal).value + assert exc_nonlocal.msg == "annotated name 'x' can't be nonlocal" + assert exc_nonlocal.lineno == 3 + + def test_annotation_assignment(self): + scp = self.mod_scope("x: int = 1") + assert scp.contains_annotated == True + + scp2 = self.mod_scope("x = 1") + assert scp2.contains_annotated == False + + fscp = self.func_scope("def f(): x: int") + assert fscp.contains_annotated == False + assert fscp.lookup("x") == symtable.SCOPE_LOCAL + + def test_issue13343(self): scp = self.mod_scope("lambda *, k1=x, k2: None") assert scp.lookup("x") == symtable.SCOPE_GLOBAL_IMPLICIT From pypy.commits at gmail.com Thu Mar 1 05:07:50 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:50 -0800 (PST) Subject: [pypy-commit] pypy pep526: Make astbuilder produce AnnAssign nodes for variable annotations. Message-ID: <5a97d0f6.0ee51c0a.64f6c.3ad1@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93914:61124ea0a2b7 Date: 2018-02-13 23:23 -0800 http://bitbucket.org/pypy/pypy/changeset/61124ea0a2b7/ Log: Make astbuilder produce AnnAssign nodes for variable annotations. diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -737,6 +737,7 @@ raise AssertionError("unknown statment type") def handle_expr_stmt(self, stmt): + from pypy.interpreter.pyparser.parser import AbstractNonterminal if stmt.num_children() == 1: expression = self.handle_testlist(stmt.get_child(0)) return ast.Expr(expression, stmt.get_lineno(), stmt.get_column()) @@ -754,6 +755,44 @@ operator = augassign_operator_map[op_str] return ast.AugAssign(target_expr, operator, value_expr, stmt.get_lineno(), stmt.get_column()) + elif stmt.get_child(1).type == syms.annassign: + # Variable annotation (PEP 526), which may or may not include assignment. + target = stmt.get_child(0) + target_expr = self.handle_testlist(target) + simple = 0 + # target is a name, nothing funky + if isinstance(target_expr, ast.Name): + # The PEP demands that `(x): T` be treated differently than `x: T` + # however, the parser does not easily expose the wrapping parens, which are a no-op + # they are elided by handle_testlist if they existed. + # so here we walk down the parse tree until we hit a terminal, and check whether it's + # a left paren + simple_test = target.get_child(0) + while isinstance(simple_test, AbstractNonterminal): + simple_test = simple_test.get_child(0) + if simple_test.type != tokens.LPAR: + simple = 1 + # subscripts are allowed with nothing special + elif isinstance(target_expr, ast.Subscript): + pass + # attributes are also fine here + elif isinstance(target_expr, ast.Attribute): + pass + # tuples and lists get special error messages + elif isinstance(target_expr, ast.Tuple): + self.error("only single target (not tuple) can be annotated", target) + elif isinstance(target_expr, ast.List): + self.error("only single target (not list) can be annotated", target) + # and everything else gets a generic error + else: + self.error("illegal target for annoation", target) + self.set_context(target_expr, ast.Store) + second = stmt.get_child(1) + annotation = self.handle_expr(second.get_child(1)) + value_expr = None + if second.num_children() == 4: + value_expr = self.handle_testlist(second.get_child(-1)) + return ast.AnnAssign(target_expr, annotation, value_expr, simple, stmt.get_lineno(), stmt.get_column()) else: # Normal assignment. targets = [] From pypy.commits at gmail.com Thu Mar 1 05:07:46 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:46 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add test for parsing variable annotations. Message-ID: <5a97d0f2.ce851c0a.addf8.3511@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93912:27758040aaa8 Date: 2018-02-13 23:21 -0800 http://bitbucket.org/pypy/pypy/changeset/27758040aaa8/ Log: Add test for parsing variable annotations. diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -614,6 +614,44 @@ assert len(dec.args) == 2 assert dec.keywords is None + def test_annassign(self): + simple = self.get_first_stmt('a: int') + assert isinstance(simple, ast.AnnAssign) + assert isinstance(simple.target, ast.Name) + assert simple.target.ctx == ast.Store + assert isinstance(simple.annotation, ast.Name) + assert simple.value == None + assert simple.simple == 1 + + with_value = self.get_first_stmt('x: str = "test"') + assert isinstance(with_value, ast.AnnAssign) + assert isinstance(with_value.value, ast.Str) + assert self.space.eq_w(with_value.value.s, self.space.wrap("test")) + + not_simple = self.get_first_stmt('(a): int') + assert isinstance(not_simple, ast.AnnAssign) + assert isinstance(not_simple.target, ast.Name) + assert not_simple.target.ctx == ast.Store + assert not_simple.simple == 0 + + attrs = self.get_first_stmt('a.b.c: int') + assert isinstance(attrs, ast.AnnAssign) + assert isinstance(attrs.target, ast.Attribute) + + subscript = self.get_first_stmt('a[0:2]: int') + assert isinstance(subscript, ast.AnnAssign) + assert isinstance(subscript.target, ast.Subscript) + + exc_tuple = py.test.raises(SyntaxError, self.get_ast, 'a, b: int').value + assert exc_tuple.msg == "only single target (not tuple) can be annotated" + + exc_list = py.test.raises(SyntaxError, self.get_ast, '[]: int').value + assert exc_list.msg == "only single target (not list) can be annotated" + + exc_bad_target = py.test.raises(SyntaxError, self.get_ast, '{}: int').value + assert exc_bad_target.msg == "illegal target for annoation" + + def test_augassign(self): aug_assigns = ( ("+=", ast.Add), From pypy.commits at gmail.com Thu Mar 1 05:07:52 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:52 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add AST validator for AnnAssign. Message-ID: <5a97d0f8.cf061c0a.23540.b3c8@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93915:9bfa5258df88 Date: 2018-02-13 23:24 -0800 http://bitbucket.org/pypy/pypy/changeset/9bfa5258df88/ Log: Add AST validator for AnnAssign. diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py --- a/pypy/interpreter/astcompiler/validate.py +++ b/pypy/interpreter/astcompiler/validate.py @@ -212,6 +212,12 @@ self._validate_exprs(node.targets, ast.Store) self._validate_expr(node.value) + def visit_AnnAssign(self, node): + self._validate_expr(node.target, ast.Store) + self._validate_expr(node.annotation) + if node.value: + self._validate_expr(node.value) + def visit_AugAssign(self, node): self._validate_expr(node.target, ast.Store) self._validate_expr(node.value) From pypy.commits at gmail.com Thu Mar 1 05:07:54 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:54 -0800 (PST) Subject: [pypy-commit] pypy pep526: Handle AnnAssign in symtable building (WIP). Message-ID: <5a97d0fa.88c0df0a.5d017.316a@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93916:e3f0d3b1a562 Date: 2018-02-13 23:25 -0800 http://bitbucket.org/pypy/pypy/changeset/e3f0d3b1a562/ Log: Handle AnnAssign in symtable building (WIP). diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -12,6 +12,7 @@ SYM_PARAM = 2 << 1 SYM_NONLOCAL = 2 << 2 SYM_USED = 2 << 3 +SYM_ANNOTATED = 2 << 4 SYM_BOUND = (SYM_PARAM | SYM_ASSIGNED) # codegen.py actually deals with these: @@ -44,6 +45,7 @@ self.child_has_free = False self.nested = False self.doc_removable = False + self.contains_annotated = False self._in_try_body_depth = 0 def lookup(self, name): @@ -139,7 +141,7 @@ self.free_vars.append(name) free[name] = None self.has_free = True - elif flags & SYM_BOUND: + elif flags & (SYM_BOUND | SYM_ANNOTATED): self.symbols[name] = SCOPE_LOCAL local[name] = None try: @@ -420,6 +422,18 @@ self.scope.note_return(ret) ast.GenericASTVisitor.visit_Return(self, ret) + def visit_AnnAssign(self, assign): + # __annotations__ is not setup or used in functions. + if not isinstance(self.scope, FunctionScope): + self.scope.contains_annotated = True + target = assign.target + if isinstance(target, ast.Name): + scope = SYM_ANNOTATED + name = target.id + if assign.value: + scope |= SYM_USED + self.note_symbol(name, scope) + def visit_ClassDef(self, clsdef): self.note_symbol(clsdef.name, SYM_ASSIGNED) self.visit_sequence(clsdef.bases) @@ -485,10 +499,13 @@ msg = "name '%s' is nonlocal and global" % (name,) raise SyntaxError(msg, glob.lineno, glob.col_offset) - if old_role & (SYM_USED | SYM_ASSIGNED): + if old_role & (SYM_USED | SYM_ASSIGNED | SYM_ANNOTATED): if old_role & SYM_ASSIGNED: msg = "name '%s' is assigned to before global declaration"\ % (name,) + elif old_role & SYM_ANNOTATED: + msg = "annotated name '%s' can't be global" \ + % (name,) else: msg = "name '%s' is used prior to global declaration" % \ (name,) @@ -498,6 +515,7 @@ def visit_Nonlocal(self, nonl): for name in nonl.names: old_role = self.scope.lookup_role(name) + print(name, old_role) msg = "" if old_role & SYM_GLOBAL: msg = "name '%s' is nonlocal and global" % (name,) @@ -505,6 +523,9 @@ msg = "name '%s' is parameter and nonlocal" % (name,) if isinstance(self.scope, ModuleScope): msg = "nonlocal declaration not allowed at module level" + if old_role & SYM_ANNOTATED: + msg = "annotated name '%s' can't be nonlocal" \ + % (name,) if msg is not "": raise SyntaxError(msg, nonl.lineno, nonl.col_offset) From pypy.commits at gmail.com Thu Mar 1 05:07:56 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:56 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add some app-level test for variable annotations. Message-ID: <5a97d0fc.ccaadf0a.96866.da81@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93917:8d9b6685e41e Date: 2018-02-13 23:27 -0800 http://bitbucket.org/pypy/pypy/changeset/8d9b6685e41e/ Log: Add some app-level test for variable annotations. diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/test_annotations.py @@ -0,0 +1,95 @@ +import py + +class AppTestAnnotations: + + def test_simple_annotation(self): + # exec because this needs to be in "top level" scope + # whereas the docstring-based tests are inside a function + # (or don't care) + exec("a: int; assert __annotations__['a'] == int") + + def test_non_simple_annotation(self): + ''' + class C: + (a): int + assert "a" not in __annotations__ + ''' + + def test_simple_with_target(self): + ''' + class C: + a: int = 1 + assert __annotations__["a"] == int + assert a == 1 + ''' + + def test_attribute_target(self): + ''' + class C: + a = 1 + a.x: int + assert __annotations__ == {} + ''' + + def test_subscript_target(self): + ''' + # this test exists to ensure that these type annotations + # don't raise exceptions during compilation + class C: + a = 1 + a[0]: int + a[1:2]: int + a[1:2:2]: int + a[1:2:2,...]: int + assert __annotations__ == {} + ''' + + def test_class_annotation(self): + ''' + class C: + a: int + b: str = "s" + assert "__annotations__" in locals() + assert C.__annotations__ == {"a": int, "b": str} + assert C.b == "s" + ''' + + def test_unevaluated_name(self): + ''' + class C: + def __init__(self): + self.x: invalid_name = 1 + y[0]: also_invalid + assert self.x == 1 + C() + ''' + + def test_function_no___annotations__(self): + ''' + a: int + assert "__annotations__" not in locals() + ''' + + def test_unboundlocal(self): + # this test and the one below it are adapted from PEP 526 + ''' + a: int + try: + print(a) + except UnboundLocalError: + pass + except: + assert False + ''' + + def test_nameerror(self): + # there's no annotation here, but it's present for contrast with + # the test above + ''' + try: + print(a) + except NameError: + pass + except: + raise + ''' From pypy.commits at gmail.com Thu Mar 1 05:08:02 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:02 -0800 (PST) Subject: [pypy-commit] pypy pep526: Make symtable handle non-Name targets properly in variable annotation. Message-ID: <5a97d102.2381df0a.f2ce8.1b20@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93920:509be3f27434 Date: 2018-02-14 22:08 -0800 http://bitbucket.org/pypy/pypy/changeset/509be3f27434/ Log: Make symtable handle non-Name targets properly in variable annotation. diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -433,6 +433,8 @@ if assign.value: scope |= SYM_USED self.note_symbol(name, scope) + else: + target.walkabout(self) def visit_ClassDef(self, clsdef): self.note_symbol(clsdef.name, SYM_ASSIGNED) From pypy.commits at gmail.com Thu Mar 1 05:07:58 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:07:58 -0800 (PST) Subject: [pypy-commit] pypy pep526: Stack effect values for new annotation opcodes. Message-ID: <5a97d0fe.c30c1c0a.1f493.6acb@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93918:d6987b09241a Date: 2018-02-14 07:14 -0800 http://bitbucket.org/pypy/pypy/changeset/d6987b09241a/ Log: Stack effect values for new annotation opcodes. diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -690,6 +690,9 @@ ops.POP_JUMP_IF_FALSE: -1, ops.JUMP_IF_NOT_DEBUG: 0, + ops.SETUP_ANNOTATIONS: 0, + ops.STORE_ANNOTATION: -1, + # TODO ops.BUILD_LIST_FROM_ARG: 1, From pypy.commits at gmail.com Thu Mar 1 05:08:04 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:04 -0800 (PST) Subject: [pypy-commit] pypy pep526: Catch expected exception in test_annotations. Message-ID: <5a97d104.b4b0df0a.29449.904a@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93921:c9cec4aab2cc Date: 2018-02-14 22:09 -0800 http://bitbucket.org/pypy/pypy/changeset/c9cec4aab2cc/ Log: Catch expected exception in test_annotations. diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py --- a/pypy/interpreter/test/test_annotations.py +++ b/pypy/interpreter/test/test_annotations.py @@ -59,8 +59,13 @@ class C: def __init__(self): self.x: invalid_name = 1 - y[0]: also_invalid assert self.x == 1 + try: + # this is invalid because `y` is undefined + # it should raise a NameError + y[0]: also_invalid + except NameError: + ... C() ''' From pypy.commits at gmail.com Thu Mar 1 05:08:00 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:00 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add another symtable test to ensure proper scoping of non-Name targets. Message-ID: <5a97d100.e4a6df0a.566bd.3821@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93919:7a50e0c20060 Date: 2018-02-14 22:04 -0800 http://bitbucket.org/pypy/pypy/changeset/7a50e0c20060/ Log: Add another symtable test to ensure proper scoping of non-Name targets. diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -513,6 +513,9 @@ assert fscp.contains_annotated == False assert fscp.lookup("x") == symtable.SCOPE_LOCAL + def test_nonsimple_annotation(self): + fscp = self.func_scope("def f(): implicit_global[0]: int") + assert fscp.lookup("implicit_global") == symtable.SCOPE_GLOBAL_IMPLICIT def test_issue13343(self): scp = self.mod_scope("lambda *, k1=x, k2: None") From pypy.commits at gmail.com Thu Mar 1 05:08:06 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:06 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add new instructions to opcode.py. Message-ID: <5a97d106.8a8bdf0a.6194a.25ef@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93922:59ecffe9844c Date: 2018-02-14 22:12 -0800 http://bitbucket.org/pypy/pypy/changeset/59ecffe9844c/ Log: Add new instructions to opcode.py. diff --git a/lib-python/3/opcode.py b/lib-python/3/opcode.py --- a/lib-python/3/opcode.py +++ b/lib-python/3/opcode.py @@ -121,7 +121,7 @@ def_op('RETURN_VALUE', 83) def_op('IMPORT_STAR', 84) - +def_op('SETUP_ANNOTATIONS', 85) def_op('YIELD_VALUE', 86) def_op('POP_BLOCK', 87) def_op('END_FINALLY', 88) @@ -171,6 +171,7 @@ haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) +name_op('STORE_ANNOTATION', 127) # Index in name list def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args + (#kwargs << 8) From pypy.commits at gmail.com Thu Mar 1 05:08:12 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:12 -0800 (PST) Subject: [pypy-commit] pypy pep526: Add test to ensure repeated calls to SETUP_ANNOTATIONS work. Message-ID: <5a97d10c.8a8bdf0a.6194a.25f4@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93925:37e249a9c04b Date: 2018-02-14 22:53 -0800 http://bitbucket.org/pypy/pypy/changeset/37e249a9c04b/ Log: Add test to ensure repeated calls to SETUP_ANNOTATIONS work. diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py --- a/pypy/interpreter/test/test_annotations.py +++ b/pypy/interpreter/test/test_annotations.py @@ -69,6 +69,14 @@ C() ''' + def test_repeated_setup(self): + ''' + d = {} + exec('a: int', d) + exec('b: int', d) + exec('assert __annotations__ == {"a": int, "b": int}', d) + ''' + def test_function_no___annotations__(self): ''' a: int From pypy.commits at gmail.com Thu Mar 1 05:08:08 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:08 -0800 (PST) Subject: [pypy-commit] pypy pep526: Implement the variable annotation bytecodes. Message-ID: <5a97d108.88d31c0a.86f04.b244@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93923:52cd5d8635c2 Date: 2018-02-14 22:13 -0800 http://bitbucket.org/pypy/pypy/changeset/52cd5d8635c2/ Log: Implement the variable annotation bytecodes. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -292,6 +292,10 @@ self.DELETE_DEREF(oparg, next_instr) elif opcode == opcodedesc.DELETE_FAST.index: self.DELETE_FAST(oparg, next_instr) + elif opcode == opcodedesc.SETUP_ANNOTATIONS.index: + self.SETUP_ANNOTATIONS(oparg, next_instr) + elif opcode == opcodedesc.STORE_ANNOTATION.index: + self.STORE_ANNOTATION(oparg, next_instr) elif opcode == opcodedesc.DELETE_GLOBAL.index: self.DELETE_GLOBAL(oparg, next_instr) elif opcode == opcodedesc.DELETE_NAME.index: @@ -947,6 +951,18 @@ varname) self.locals_cells_stack_w[varindex] = None + def SETUP_ANNOTATIONS(self, oparg, next_instr): + w_locals = self.getorcreatedebug().w_locals + if not self.space.finditem_str(w_locals, '__annotations__'): + w_annotations = self.space.newdict() + self.space.setitem_str(w_locals, '__annotations__', w_annotations) + + def STORE_ANNOTATION(self, varindex, next_instr): + varname = self.getname_u(varindex) + w_newvalue = self.popvalue() + self.space.setitem_str(self.getorcreatedebug().w_locals.getitem_str('__annotations__'), varname, + w_newvalue) + def BUILD_TUPLE(self, itemcount, next_instr): items = self.popvalues(itemcount) w_tuple = self.space.newtuple(items) From pypy.commits at gmail.com Thu Mar 1 05:08:10 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:10 -0800 (PST) Subject: [pypy-commit] pypy pep526: Support code generation with variable annotations. Message-ID: <5a97d10a.42e61c0a.8f2af.b828@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93924:2588a45dab22 Date: 2018-02-14 22:14 -0800 http://bitbucket.org/pypy/pypy/changeset/2588a45dab22/ Log: Support code generation with variable annotations. diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -299,6 +299,12 @@ else: return False + def _maybe_setup_annotations(self): + # if the scope contained an annotated variable assignemt, + # this will emit the requisite SETUP_ANNOTATIONS + if self.scope.contains_annotated and not isinstance(self, AbstractFunctionCodeGenerator): + self.emit_op(ops.SETUP_ANNOTATIONS) + def visit_Module(self, mod): if not self._handle_body(mod.body): self.first_lineno = self.lineno = 1 @@ -925,6 +931,66 @@ self.visit_sequence(targets) return True + def _annotation_evaluate(self, item): + # PEP 526 requires that some things be evaluated, to avoid bugs + # where a non-assigning variable annotation references invalid items + # this is effectively a NOP, but will fail if e.g. item is an + # Attribute and one of the chained names does not exist + item.walkabout(self) + self.emit_op(ops.POP_TOP) + + def _annotation_eval_slice(self, target): + if isinstance(target, ast.Index): + self._annotation_evaluate(target.value) + elif isinstance(target, ast.Slice): + for val in [target.lower, target.upper, target.step]: + if val: + self._annotation_evaluate(val) + elif isinstance(target, ast.ExtSlice): + for val in target.dims: + if isinstance(val, ast.Index) or isinstance(val, ast.Slice): + self._annotation_eval_slice(val) + else: + self.error("Invalid nested slice", val) + else: + self.error("Invalid slice?", target) + + def visit_AnnAssign(self, assign): + self.update_position(assign.lineno, True) + target = assign.target + # if there's an assignment to be done, do it + if assign.value: + assign.value.walkabout(self) + target.walkabout(self) + # the PEP requires that certain parts of the target be evaluated at runtime + # to avoid silent annotation-related errors + if isinstance(target, ast.Name): + # if it's just a simple name and we're not in a function, store + # the annotation in __annotations__ + if assign.simple and not isinstance(self.scope, symtable.FunctionScope): + assign.annotation.walkabout(self) + name = target.id + self.emit_op_arg(ops.STORE_ANNOTATION, self.add_name(self.names, name)) + elif isinstance(target, ast.Attribute): + # the spec requires that `a.b: int` evaluates `a` + # and in a non-function scope, also evaluates `int` + # (N.B.: if the target is of the form `a.b.c`, `a.b` will be evaluated) + if not assign.value: + attr = target.value + self._annotation_evaluate(attr) + elif isinstance(target, ast.Subscript): + # similar to the above, `a[0:5]: int` evaluates the name and the slice argument + # and if not in a function, also evaluates the annotation + sl = target.slice + self._annotation_evaluate(target.value) + self._annotation_eval_slice(sl) + else: + self.error("can't handle annotation with %s" % (target,), target) + # if this is not in a function, evaluate the annotation + if not (assign.simple or isinstance(self.scope, symtable.FunctionScope)): + self._annotation_evaluate(assign.annotation) + + def visit_With(self, wih): self.update_position(wih.lineno, True) self.handle_withitem(wih, 0, is_async=False) @@ -1527,6 +1593,7 @@ symbols, compile_info, qualname=None) def _compile(self, tree): + self._maybe_setup_annotations() tree.walkabout(self) def _get_code_flags(self): @@ -1656,6 +1723,7 @@ w_qualname = self.space.newtext(self.qualname) self.load_const(w_qualname) self.name_op("__qualname__", ast.Store) + self._maybe_setup_annotations() # compile the body proper self._handle_body(cls.body) # return the (empty) __class__ cell From pypy.commits at gmail.com Thu Mar 1 05:08:17 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:17 -0800 (PST) Subject: [pypy-commit] pypy pep526: Improve annotation tests. Message-ID: <5a97d111.c89edf0a.6c209.24ef@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93927:250a545b5734 Date: 2018-02-18 10:42 -0800 http://bitbucket.org/pypy/pypy/changeset/250a545b5734/ Log: Improve annotation tests. diff --git a/pypy/interpreter/test/test_annotations.py b/pypy/interpreter/test/test_annotations.py --- a/pypy/interpreter/test/test_annotations.py +++ b/pypy/interpreter/test/test_annotations.py @@ -1,13 +1,14 @@ -import py - class AppTestAnnotations: - def test_simple_annotation(self): + def test_toplevel_annotation(self): # exec because this needs to be in "top level" scope # whereas the docstring-based tests are inside a function # (or don't care) exec("a: int; assert __annotations__['a'] == int") + def test_toplevel_invalid(self): + exec('try: a: invalid\nexcept NameError: pass\n') + def test_non_simple_annotation(self): ''' class C: @@ -33,8 +34,8 @@ def test_subscript_target(self): ''' - # this test exists to ensure that these type annotations - # don't raise exceptions during compilation + # ensure that these type annotations don't raise exceptions + # during compilation class C: a = 1 a[0]: int @@ -48,10 +49,9 @@ ''' class C: a: int - b: str = "s" + b: str assert "__annotations__" in locals() assert C.__annotations__ == {"a": int, "b": str} - assert C.b == "s" ''' def test_unevaluated_name(self): @@ -60,22 +60,27 @@ def __init__(self): self.x: invalid_name = 1 assert self.x == 1 - try: - # this is invalid because `y` is undefined - # it should raise a NameError - y[0]: also_invalid - except NameError: - ... C() ''' + def test_nonexistent_target(self): + ''' + try: + # this is invalid because `y` is undefined + # it should raise a NameError + y[0]: invalid + except NameError: + ... + ''' + def test_repeated_setup(self): - ''' + # each exec will run another SETUP_ANNOTATIONS + # we want to confirm that this doesn't blow away + # the previous __annotations__ d = {} exec('a: int', d) exec('b: int', d) exec('assert __annotations__ == {"a": int, "b": int}', d) - ''' def test_function_no___annotations__(self): ''' @@ -84,25 +89,25 @@ ''' def test_unboundlocal(self): - # this test and the one below it are adapted from PEP 526 + # a simple variable annotation implies its target is a local ''' a: int try: print(a) except UnboundLocalError: - pass - except: - assert False + return + assert False ''' - def test_nameerror(self): - # there's no annotation here, but it's present for contrast with - # the test above + def test_reassigned___annotations__(self): ''' - try: - print(a) - except NameError: - pass - except: - raise + class C: + __annotations__ = None + try: + a: int + raise + except TypeError: + pass + except: + assert False ''' From pypy.commits at gmail.com Thu Mar 1 05:08:14 2018 From: pypy.commits at gmail.com (alcarithemad) Date: Thu, 01 Mar 2018 02:08:14 -0800 (PST) Subject: [pypy-commit] pypy pep526: Set __annotations__ to an empty dict in __main__. Message-ID: <5a97d10e.0a9fdf0a.62b68.2240@mx.google.com> Author: Colin Valliant Branch: pep526 Changeset: r93926:103116c17ec7 Date: 2018-02-18 09:24 -0800 http://bitbucket.org/pypy/pypy/changeset/103116c17ec7/ Log: Set __annotations__ to an empty dict in __main__. diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -577,6 +577,7 @@ mainmodule = type(sys)('__main__') mainmodule.__loader__ = sys.__loader__ mainmodule.__builtins__ = os.__builtins__ + mainmodule.__annotations__ = {} sys.modules['__main__'] = mainmodule if not no_site: diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py --- a/pypy/interpreter/main.py +++ b/pypy/interpreter/main.py @@ -13,6 +13,8 @@ raise mainmodule = module.Module(space, w_main) space.setitem(w_modules, w_main, mainmodule) + w_annotations = space.newdict() + space.setitem_str(mainmodule.w_dict, '__annotations__', w_annotations) return mainmodule From pypy.commits at gmail.com Thu Mar 1 05:37:33 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Mar 2018 02:37:33 -0800 (PST) Subject: [pypy-commit] pypy py3.5: generalize win Reg* calls for A and W variants, use and fix tests Message-ID: <5a97d7ed.7499df0a.f07bd.e59a@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93929:0684dbb33360 Date: 2018-03-01 05:35 -0500 http://bitbucket.org/pypy/pypy/changeset/0684dbb33360/ Log: generalize win Reg* calls for A and W variants, use and fix tests diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -175,7 +175,7 @@ c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with rffi.scoped_unicode2wcharp(filename) as wide_filename: c_filename = rffi.cast(rffi.CCHARP, wide_filename) - ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename) + ret = rwinreg.RegLoadKeyW(hkey, c_subkey, c_filename) if ret != 0: raiseWindowsError(space, ret, 'RegLoadKey') @@ -196,7 +196,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(filename) as wide_filename: c_filename = rffi.cast(rffi.CCHARP, wide_filename) - ret = rwinreg.RegSaveKey(hkey, c_filename, None) + ret = rwinreg.RegSaveKeyW(hkey, c_filename, None) if ret != 0: raiseWindowsError(space, ret, 'RegSaveKey') @@ -226,7 +226,7 @@ c_subkey = rffi.cast(rffi.CCHARP, subkey) with rffi.scoped_unicode2wcharp(value) as dataptr: c_dataptr = rffi.cast(rffi.CCHARP, dataptr) - ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ, + ret = rwinreg.RegSetValueW(hkey, c_subkey, rwinreg.REG_SZ, c_dataptr, len(value)) if ret != 0: raiseWindowsError(space, ret, 'RegSetValue') @@ -250,7 +250,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p) + ret = rwinreg.RegQueryValueW(hkey, c_subkey, None, bufsize_p) bufSize = intmask(bufsize_p[0]) if ret == rwinreg.ERROR_MORE_DATA: bufSize = 256 @@ -259,7 +259,7 @@ while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p) + ret = rwinreg.RegQueryValueW(hkey, c_subkey, buf, bufsize_p) if ret == rwinreg.ERROR_MORE_DATA: print 'bufSize was %d, too small' % bufSize # Resize and retry @@ -440,7 +440,7 @@ try: with rffi.scoped_unicode2wcharp(value_name) as wide_vn: c_vn = rffi.cast(rffi.CCHARP, wide_vn) - ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen) + ret = rwinreg.RegSetValueExW(hkey, c_vn, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -460,7 +460,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, null_dword, + ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword, null_dword, None, retDataSize) bufSize = intmask(retDataSize[0]) if ret == rwinreg.ERROR_MORE_DATA: @@ -472,7 +472,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, + ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -505,7 +505,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey) + ret = rwinreg.RegCreateKeyW(hkey, c_subkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'CreateKey') return W_HKEY(space, rethkey[0]) @@ -527,7 +527,7 @@ with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key: c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, + ret = rwinreg.RegCreateKeyExW(hkey, c_subkey, reserved, None, 0, access, None, rethkey, lltype.nullptr(rwin32.LPDWORD.TO)) if ret != 0: @@ -549,7 +549,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) - ret = rwinreg.RegDeleteKey(hkey, c_subkey) + ret = rwinreg.RegDeleteKeyW(hkey, c_subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteKey') @@ -562,7 +562,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) - ret = rwinreg.RegDeleteValue(hkey, c_subkey) + ret = rwinreg.RegDeleteValueW(hkey, c_subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteValue') @@ -582,7 +582,7 @@ with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access, rethkey) + ret = rwinreg.RegOpenKeyExW(hkey, c_subkey, reserved, access, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegOpenKeyEx') return W_HKEY(space, rethkey[0]) @@ -607,7 +607,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyW( hkey, None, null_dword, null_dword, null_dword, null_dword, null_dword, null_dword, retValueSize, retDataSize, @@ -628,7 +628,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf) - ret = rwinreg.RegEnumValue( + ret = rwinreg.RegEnumValueW( hkey, index, c_valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: @@ -673,7 +673,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, 257) as buf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: retValueSize[0] = r_uint(257) # includes NULL terminator - ret = rwinreg.RegEnumKeyEx(hkey, index, buf, retValueSize, + ret = rwinreg.RegEnumKeyExW(hkey, index, buf, retValueSize, null_dword, None, null_dword, lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: @@ -695,7 +695,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as nValues: with lltype.scoped_alloc(rwin32.PFILETIME.TO, 1) as ft: null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyW( hkey, None, null_dword, null_dword, nSubKeys, null_dword, null_dword, nValues, null_dword, null_dword, @@ -722,7 +722,7 @@ machine = space.text_or_none_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) + ret = rwinreg.RegConnectRegistryW(machine, hkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegConnectRegistry') return W_HKEY(space, rethkey[0]) diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -207,18 +207,20 @@ except: pass - key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) - SaveKey(key, self.tmpfilename) + with OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) as key: + SaveKey(key, self.tmpfilename) def test_expand_environment_string(self): from winreg import ExpandEnvironmentStrings import nt r = ExpandEnvironmentStrings("%windir%\\test") assert isinstance(r, str) - if 'WINDIR' in list(nt.environ.keys()): + if 'WINDIR' in nt.environ: assert r == nt.environ["WINDIR"] + "\\test" + elif 'windir' in nt.environ: + assert r == nt.environ["windir"] + "\\test" else: - assert r == nt.environ["windir"] + "\\test" + skip('nt.environ not filled in for untranslated tests') def test_long_key(self): from winreg import ( diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,77 +47,107 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'A' +suffix = 'W' +def get_traits(suffix): + RegSetValue = external( + 'RegSetValue' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) -RegSetValue = external( - 'RegSetValue' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) + RegSetValueEx = external( + 'RegSetValueEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, + rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) -RegSetValueEx = external( - 'RegSetValueEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, - rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) + RegQueryValue = external( + 'RegQueryValue' + suffix, + [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], + rffi.LONG) -RegQueryValue = external( - 'RegQueryValue' + suffix, - [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], - rffi.LONG) + RegQueryValueEx = external( + 'RegQueryValueEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) -RegQueryValueEx = external( - 'RegQueryValueEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) + RegCreateKey = external( + 'RegCreateKey' + suffix, + [HKEY, rffi.CCHARP, PHKEY], + rffi.LONG) -RegCreateKey = external( - 'RegCreateKey' + suffix, - [HKEY, rffi.CCHARP, PHKEY], - rffi.LONG) + RegCreateKeyEx = external( + 'RegCreateKeyEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, + REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], + rffi.LONG) -RegCreateKeyEx = external( - 'RegCreateKeyEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, - REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], - rffi.LONG) + RegDeleteValue = external( + 'RegDeleteValue' + suffix, + [HKEY, rffi.CCHARP], + rffi.LONG) -RegDeleteValue = external( - 'RegDeleteValue' + suffix, - [HKEY, rffi.CCHARP], - rffi.LONG) + RegDeleteKey = external( + 'RegDeleteKey' + suffix, + [HKEY, rffi.CCHARP], + rffi.LONG) -RegDeleteKey = external( - 'RegDeleteKey' + suffix, - [HKEY, rffi.CCHARP], - rffi.LONG) + RegOpenKeyEx = external( + 'RegOpenKeyEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], + rffi.LONG) -RegOpenKeyEx = external( - 'RegOpenKeyEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], - rffi.LONG) + RegEnumValue = external( + 'RegEnumValue' + suffix, + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) -RegEnumValue = external( - 'RegEnumValue' + suffix, - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) + RegEnumKeyEx = external( + 'RegEnumKeyEx' + suffix, + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) -RegEnumKeyEx = external( - 'RegEnumKeyEx' + suffix, - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) + RegQueryInfoKey = external( + 'RegQueryInfoKey' + suffix, + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) -RegQueryInfoKey = external( - 'RegQueryInfoKey' + suffix, - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) + RegLoadKey = external( + 'RegLoadKey' + suffix, + [HKEY, rffi.CCHARP, rffi.CCHARP], + rffi.LONG) + + RegSaveKey = external( + 'RegSaveKey' + suffix, + [HKEY, rffi.CCHARP, rffi.VOIDP], + rffi.LONG) + + RegConnectRegistry = external( + 'RegConnectRegistry' + suffix, + [rffi.CCHARP, HKEY, PHKEY], + rffi.LONG) + + return (RegSetValue, RegSetValueEx, RegQueryValue, RegQueryValueEx, + RegCreateKey, RegCreateKeyEx, RegDeleteValue, RegDeleteKey, + RegOpenKeyEx, RegEnumValue, RegEnumKeyEx, RegQueryInfoKey, + RegLoadKey, RegSaveKey, RegConnectRegistry) + +RegSetValueW, RegSetValueExW, RegQueryValueW, RegQueryValueExW, \ + RegCreateKeyW, RegCreateKeyExW, RegDeleteValueW, RegDeleteKeyW, \ + RegOpenKeyExW, RegEnumValueW, RegEnumKeyExW, RegQueryInfoKeyW, \ + RegLoadKeyW, RegSaveKeyW, RegConnectRegistryW = get_traits('W') + +RegSetValueA, RegSetValueExA, RegQueryValueA, RegQueryValueExA, \ + RegCreateKeyA, RegCreateKeyExA, RegDeleteValueA, RegDeleteKeyA, \ + RegOpenKeyExA, RegEnumValueA, RegEnumKeyExA, RegQueryInfoKeyA, \ + RegLoadKeyA, RegSaveKeyA, RegConnectRegistryA = get_traits('A') RegCloseKey = external( 'RegCloseKey', @@ -129,21 +159,6 @@ [HKEY], rffi.LONG) -RegLoadKey = external( - 'RegLoadKey' + suffix, - [HKEY, rffi.CCHARP, rffi.CCHARP], - rffi.LONG) - -RegSaveKey = external( - 'RegSaveKey' + suffix, - [HKEY, rffi.CCHARP, rffi.VOIDP], - rffi.LONG) - -RegConnectRegistry = external( - 'RegConnectRegistry' + suffix, - [rffi.CCHARP, HKEY, PHKEY], - rffi.LONG) - _ExpandEnvironmentStringsW = external( 'ExpandEnvironmentStringsW', [rffi.CWCHARP, rffi.CWCHARP, rwin32.DWORD], From pypy.commits at gmail.com Thu Mar 1 05:37:35 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Mar 2018 02:37:35 -0800 (PST) Subject: [pypy-commit] pypy py3.5: trivial test fixes for win Message-ID: <5a97d7ef.f4a0df0a.47c25.c368@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93930:934b5f719f0d Date: 2018-03-01 05:36 -0500 http://bitbucket.org/pypy/pypy/changeset/934b5f719f0d/ Log: trivial test fixes for win diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -82,7 +82,7 @@ import imp for suffix, mode, type in imp.get_suffixes(): if type == imp.PY_SOURCE: - assert suffix == '.py' + assert suffix in ('.py', '.pyw') assert mode == 'r' elif type == imp.PY_COMPILED: assert suffix == '.pyc' diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1478,6 +1478,8 @@ def test_environ(self): import sys, os environ = os.environ + if not environ: + skip('environ not filled in for untranslated tests') for k, v in environ.items(): assert type(k) is str assert type(v) is str From pypy.commits at gmail.com Thu Mar 1 06:41:55 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 01 Mar 2018 03:41:55 -0800 (PST) Subject: [pypy-commit] pypy py3.6: CPython Issue #26129: grp.getgrgid() sends a DeprecationWarning when passing non-integers Message-ID: <5a97e703.8ec11c0a.8830a.9ccd@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93932:0ebac2fc9fb1 Date: 2018-02-26 01:13 +0100 http://bitbucket.org/pypy/pypy/changeset/0ebac2fc9fb1/ Log: CPython Issue #26129: grp.getgrgid() sends a DeprecationWarning when passing non-integers diff --git a/lib_pypy/grp.py b/lib_pypy/grp.py --- a/lib_pypy/grp.py +++ b/lib_pypy/grp.py @@ -33,7 +33,13 @@ @builtinify def getgrgid(gid): - res = lib.getgrgid(gid) + try: + res = lib.getgrgid(gid) + except TypeError: + gid = int(gid) + res = lib.getgrgid(gid) + import warnings + warnings.warn("group id must be int", DeprecationWarning) if not res: # XXX maybe check error eventually raise KeyError(gid) From pypy.commits at gmail.com Thu Mar 1 06:41:53 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 01 Mar 2018 03:41:53 -0800 (PST) Subject: [pypy-commit] pypy py3.6: hg merge py3.5 Message-ID: <5a97e701.51951c0a.94951.86a6@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93931:eb5abc38e4b2 Date: 2018-02-25 23:09 +0100 http://bitbucket.org/pypy/pypy/changeset/eb5abc38e4b2/ Log: hg merge py3.5 diff too long, truncating to 2000 out of 5407 lines diff --git a/extra_tests/test_pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/__init__.py @@ -0,0 +1,1 @@ + diff --git a/extra_tests/test_pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/infrastructure.py @@ -0,0 +1,87 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import print_function +from contextlib import contextmanager +import os + +from pyrepl.reader import Reader +from pyrepl.console import Console, Event + + +class EqualsAnything(object): + def __eq__(self, other): + return True + + +EA = EqualsAnything() + + +class TestConsole(Console): + height = 24 + width = 80 + encoding = 'utf-8' + + def __init__(self, events, verbose=False): + self.events = events + self.next_screen = None + self.verbose = verbose + + def refresh(self, screen, xy): + if self.next_screen is not None: + assert screen == self.next_screen, "[ %s != %s after %r ]" % ( + screen, self.next_screen, self.last_event_name) + + def get_event(self, block=1): + ev, sc = self.events.pop(0) + self.next_screen = sc + if not isinstance(ev, tuple): + ev = (ev, None) + self.last_event_name = ev[0] + if self.verbose: + print("event", ev) + return Event(*ev) + + +class BaseTestReader(Reader): + + def get_prompt(self, lineno, cursor_on_line): + return '' + + def refresh(self): + Reader.refresh(self) + self.dirty = True + + +def read_spec(test_spec, reader_class=BaseTestReader): + # remember to finish your test_spec with 'accept' or similar! + con = TestConsole(test_spec, verbose=True) + reader = reader_class(con) + reader.readline() + + + at contextmanager +def sane_term(): + """Ensure a TERM that supports clear""" + old_term, os.environ['TERM'] = os.environ.get('TERM'), 'xterm' + yield + if old_term is not None: + os.environ['TERM'] = old_term + else: + del os.environ['TERM'] diff --git a/extra_tests/test_pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_basic.py @@ -0,0 +1,116 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import pytest +from .infrastructure import read_spec + + +def test_basic(): + read_spec([(('self-insert', 'a'), ['a']), + ( 'accept', ['a'])]) + + +def test_repeat(): + read_spec([(('digit-arg', '3'), ['']), + (('self-insert', 'a'), ['aaa']), + ( 'accept', ['aaa'])]) + + +def test_kill_line(): + read_spec([(('self-insert', 'abc'), ['abc']), + ( 'left', None), + ( 'kill-line', ['ab']), + ( 'accept', ['ab'])]) + + +def test_unix_line_discard(): + read_spec([(('self-insert', 'abc'), ['abc']), + ( 'left', None), + ( 'unix-word-rubout', ['c']), + ( 'accept', ['c'])]) + + +def test_kill_word(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'beginning-of-line', ['ab cd']), + ( 'kill-word', [' cd']), + ( 'accept', [' cd'])]) + + +def test_backward_kill_word(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'backward-kill-word', ['ab ']), + ( 'accept', ['ab '])]) + + +def test_yank(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'backward-kill-word', ['ab ']), + ( 'beginning-of-line', ['ab ']), + ( 'yank', ['cdab ']), + ( 'accept', ['cdab '])]) + + +def test_yank_pop(): + read_spec([(('self-insert', 'ab cd'), ['ab cd']), + ( 'backward-kill-word', ['ab ']), + ( 'left', ['ab ']), + ( 'backward-kill-word', [' ']), + ( 'yank', ['ab ']), + ( 'yank-pop', ['cd ']), + ( 'accept', ['cd '])]) + + +# interrupt uses os.kill which doesn't go through signal handlers on windows + at pytest.mark.skipif("os.name == 'nt'") +def test_interrupt(): + with pytest.raises(KeyboardInterrupt): + read_spec([('interrupt', [''])]) + + +# test_suspend -- hah +def test_up(): + read_spec([(('self-insert', 'ab\ncd'), ['ab', 'cd']), + ( 'up', ['ab', 'cd']), + (('self-insert', 'e'), ['abe', 'cd']), + ( 'accept', ['abe', 'cd'])]) + + +def test_down(): + read_spec([(('self-insert', 'ab\ncd'), ['ab', 'cd']), + ( 'up', ['ab', 'cd']), + (('self-insert', 'e'), ['abe', 'cd']), + ( 'down', ['abe', 'cd']), + (('self-insert', 'f'), ['abe', 'cdf']), + ( 'accept', ['abe', 'cdf'])]) + + +def test_left(): + read_spec([(('self-insert', 'ab'), ['ab']), + ( 'left', ['ab']), + (('self-insert', 'c'), ['acb']), + ( 'accept', ['acb'])]) + + +def test_right(): + read_spec([(('self-insert', 'ab'), ['ab']), + ( 'left', ['ab']), + (('self-insert', 'c'), ['acb']), + ( 'right', ['acb']), + (('self-insert', 'd'), ['acbd']), + ( 'accept', ['acbd'])]) diff --git a/extra_tests/test_pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_bugs.py @@ -0,0 +1,75 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from pyrepl.historical_reader import HistoricalReader +from .infrastructure import EA, BaseTestReader, sane_term, read_spec + +# this test case should contain as-verbatim-as-possible versions of +# (applicable) bug reports + +import pytest + + +class HistoricalTestReader(HistoricalReader, BaseTestReader): + pass + + + at pytest.mark.xfail(reason='event missing', run=False) +def test_transpose_at_start(): + read_spec([ + ('transpose', [EA, '']), + ('accept', [''])]) + + +def test_cmd_instantiation_crash(): + spec = [ + ('reverse-history-isearch', ["(r-search `') "]), + (('key', 'left'), ['']), + ('accept', ['']) + ] + read_spec(spec, HistoricalTestReader) + + + at pytest.mark.skipif("os.name != 'posix' or 'darwin' in sys.platform or " + "'kfreebsd' in sys.platform") +def test_signal_failure(monkeypatch): + import os + import pty + import signal + from pyrepl.unix_console import UnixConsole + + def failing_signal(a, b): + raise ValueError + + def really_failing_signal(a, b): + raise AssertionError + + mfd, sfd = pty.openpty() + try: + with sane_term(): + c = UnixConsole(sfd, sfd) + c.prepare() + c.restore() + monkeypatch.setattr(signal, 'signal', failing_signal) + c.prepare() + monkeypatch.setattr(signal, 'signal', really_failing_signal) + c.restore() + finally: + os.close(mfd) + os.close(sfd) diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_functional.py @@ -0,0 +1,28 @@ +# Copyright 2000-2007 Michael Hudson-Doyle +# Maciek Fijalkowski +# License: MIT +# some functional tests, to see if this is really working + +import pytest +import sys + + + at pytest.fixture() +def child(): + try: + import pexpect + except ImportError: + pytest.skip("no pexpect module") + except SyntaxError: + pytest.skip('pexpect wont work on py3k') + child = pexpect.spawn(sys.executable, ['-S'], timeout=10) + child.logfile = sys.stdout + child.sendline('from pyrepl.python_reader import main') + child.sendline('main()') + return child + + +def test_basic(child): + child.sendline('a = 3') + child.sendline('a') + child.expect('3') diff --git a/extra_tests/test_pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_keymap.py @@ -0,0 +1,10 @@ +from pyrepl.keymap import compile_keymap + + +def test_compile_keymap(): + k = compile_keymap({ + b'a': 'test', + b'bc': 'test2', + }) + + assert k == {b'a': 'test', b'b': {b'c': 'test2'}} diff --git a/extra_tests/test_pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_reader.py @@ -0,0 +1,9 @@ + +def test_process_prompt(): + from pyrepl.reader import Reader + r = Reader(None) + assert r.process_prompt("hi!") == ("hi!", 3) + assert r.process_prompt("h\x01i\x02!") == ("hi!", 2) + assert r.process_prompt("hi\033[11m!") == ("hi\033[11m!", 3) + assert r.process_prompt("h\x01i\033[11m!\x02") == ("hi\033[11m!", 1) + assert r.process_prompt("h\033[11m\x01i\x02!") == ("h\033[11mi!", 2) diff --git a/extra_tests/test_pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_readline.py @@ -0,0 +1,22 @@ +import pytest + +from .infrastructure import sane_term + + + at pytest.mark.skipif("os.name != 'posix' or 'darwin' in sys.platform or " + "'freebsd' in sys.platform") +def test_raw_input(): + import os + import pty + from pyrepl.readline import _ReadlineWrapper + + master, slave = pty.openpty() + readline_wrapper = _ReadlineWrapper(slave, slave) + os.write(master, b'input\n') + + with sane_term(): + result = readline_wrapper.get_reader().readline() + #result = readline_wrapper.raw_input('prompt:') + assert result == 'input' + # A bytes string on python2, a unicode string on python3. + assert isinstance(result, str) diff --git a/extra_tests/test_pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_wishes.py @@ -0,0 +1,31 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from .infrastructure import read_spec + +# this test case should contain as-verbatim-as-possible versions of +# (applicable) feature requests + + +def test_quoted_insert_repeat(): + read_spec([ + (('digit-arg', '3'), ['']), + (('quoted-insert', None), ['']), + (('self-insert', '\033'), ['^[^[^[']), + (('accept', None), None)]) diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -255,7 +255,7 @@ """Launch vcvarsall.bat and read the settings from its environment """ vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) + interesting = set(("include", "lib", "path")) result = {} if vcvarsall is None: diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py --- a/lib-python/3/distutils/unixccompiler.py +++ b/lib-python/3/distutils/unixccompiler.py @@ -222,6 +222,10 @@ return "-L" + dir def _is_gcc(self, compiler_name): + if "__pypy__" in sys.builtin_module_names: # issue #2747 + if (compiler_name.startswith('cc') or + compiler_name.startswith('c++')): + return True return "gcc" in compiler_name or "g++" in compiler_name def runtime_library_dir_option(self, dir): diff --git a/lib_pypy/_libmpdec/vccompat.h b/lib_pypy/_libmpdec/vccompat.h --- a/lib_pypy/_libmpdec/vccompat.h +++ b/lib_pypy/_libmpdec/vccompat.h @@ -32,7 +32,11 @@ /* Visual C fixes: no stdint.h, no snprintf ... */ #ifdef _MSC_VER + #if _MSC_VER < 1900 #include "vcstdint.h" + #else + #include "stdint.h" + #endif #undef inline #define inline __inline #undef random diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py --- a/lib_pypy/_pypy_winbase_build.py +++ b/lib_pypy/_pypy_winbase_build.py @@ -63,6 +63,12 @@ HANDLE hStdError; } STARTUPINFO, *LPSTARTUPINFO; +typedef struct _SECURITY_ATTRIBUTES { + DWORD nLength; + LPVOID lpSecurityDescriptor; + BOOL bInheritHandle; +} SECURITY_ATTRIBUTES, *PSECURITY_ATTRIBUTES, *LPSECURITY_ATTRIBUTES; + typedef struct { HANDLE hProcess; HANDLE hThread; @@ -70,9 +76,41 @@ DWORD dwThreadId; } PROCESS_INFORMATION, *LPPROCESS_INFORMATION; +typedef struct _OVERLAPPED { + ULONG_PTR Internal; + ULONG_PTR InternalHigh; + union { + struct { + DWORD Offset; + DWORD OffsetHigh; + } DUMMYSTRUCTNAME; + PVOID Pointer; + } DUMMYUNIONNAME; + + HANDLE hEvent; +} OVERLAPPED, *LPOVERLAPPED; + + DWORD WINAPI GetVersion(void); BOOL WINAPI CreatePipe(PHANDLE, PHANDLE, void *, DWORD); +HANDLE WINAPI CreateNamedPipeA(LPCSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateNamedPipeW(LPWSTR, DWORD, DWORD, DWORD, DWORD, DWORD, + DWORD , LPSECURITY_ATTRIBUTES); +HANDLE WINAPI CreateFileA(LPCSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +HANDLE WINAPI CreateFileW(LPCWSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, + DWORD, DWORD, HANDLE); +BOOL WINAPI SetNamedPipeHandleState(HANDLE, LPDWORD, LPDWORD, LPDWORD); +BOOL WINAPI ConnectNamedPipe(HANDLE, LPOVERLAPPED); +HANDLE WINAPI CreateEventA(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCSTR); +HANDLE WINAPI CreateEventW(LPSECURITY_ATTRIBUTES, BOOL, BOOL, LPCWSTR); +VOID WINAPI SetEvent(HANDLE); +BOOL WINAPI CancelIoEx(HANDLE, LPOVERLAPPED); BOOL WINAPI CloseHandle(HANDLE); +DWORD WINAPI GetLastError(VOID); +BOOL WINAPI GetOverlappedResult(HANDLE, LPOVERLAPPED, LPDWORD, BOOL); + HANDLE WINAPI GetCurrentProcess(void); BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD); diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py --- a/lib_pypy/_pypy_winbase_cffi.py +++ b/lib_pypy/_pypy_winbase_cffi.py @@ -3,8 +3,8 @@ ffi = _cffi_backend.FFI('_pypy_winbase_cffi', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01', - _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0), - _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')), - _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'), + _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xAB\x03\x00\x00\x13\x11\x00\x00\xB0\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\xAA\x03\x00\x00\xA8\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\xA7\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x29\x11\x00\x00\x18\x03\x00\x00\x07\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x00\x2E\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x6B\x03\x00\x00\x49\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x33\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x66\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x00\x0F\x00\x00\x66\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\xA9\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xAB\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x6E\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6B\x03\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x71\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x49\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x77\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x6E\x11\x00\x00\x0A\x01\x00\x00\x0A\x01\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\xB0\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x04\x09\x00\x00\x02\x09\x00\x00\x05\x09\x00\x00\x03\x09\x00\x00\x02\x01\x00\x00\x01\x09\x00\x00\x00\x09\x00\x00\xAF\x03\x00\x00\x04\x01\x00\x00\x00\x01', + _globals = (b'\x00\x00\x27\x23CancelIoEx',0,b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x27\x23ConnectNamedPipe',0,b'\x00\x00\x6D\x23CreateEventA',0,b'\x00\x00\x73\x23CreateEventW',0,b'\x00\x00\x79\x23CreateFileA',0,b'\x00\x00\x9B\x23CreateFileW',0,b'\x00\x00\x82\x23CreateNamedPipeA',0,b'\x00\x00\x91\x23CreateNamedPipeW',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x48\x23CreateProcessW',0,b'\x00\x00\x3F\x23DuplicateHandle',0,b'\x00\x00\x8F\x23GetCurrentProcess',0,b'\x00\x00\x35\x23GetExitCodeProcess',0,b'\x00\x00\x63\x23GetLastError',0,b'\x00\x00\x5E\x23GetModuleFileNameW',0,b'\x00\x00\x2B\x23GetOverlappedResult',0,b'\x00\x00\x8C\x23GetStdHandle',0,b'\x00\x00\x63\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x57\x23SetErrorMode',0,b'\x00\x00\xA4\x23SetEvent',0,b'\x00\x00\x39\x23SetNamedPipeHandleState',0,b'\x00\x00\x31\x23TerminateProcess',0,b'\x00\x00\x5A\x23WaitForSingleObject',0,b'\x00\x00\x54\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x68\x23_getwch',0,b'\x00\x00\x68\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x6A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x65\x23_ungetwch',0), + _struct_unions = ((b'\x00\x00\x00\xAD\x00\x00\x00\x03$1',b'\x00\x00\xAC\x11DUMMYSTRUCTNAME',b'\x00\x00\x15\x11Pointer'),(b'\x00\x00\x00\xAC\x00\x00\x00\x02$2',b'\x00\x00\x18\x11Offset',b'\x00\x00\x18\x11OffsetHigh'),(b'\x00\x00\x00\xA8\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\xAA\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x66\x11wShowWindow',b'\x00\x00\x66\x11cbReserved2',b'\x00\x00\xAE\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError'),(b'\x00\x00\x00\xA7\x00\x00\x00\x02_OVERLAPPED',b'\x00\x00\x18\x11Internal',b'\x00\x00\x18\x11InternalHigh',b'\x00\x00\xAD\x11DUMMYUNIONNAME',b'\x00\x00\x15\x11hEvent'),(b'\x00\x00\x00\xA9\x00\x00\x00\x02_SECURITY_ATTRIBUTES',b'\x00\x00\x18\x11nLength',b'\x00\x00\x15\x11lpSecurityDescriptor',b'\x00\x00\x01\x11bInheritHandle')), + _typenames = (b'\x00\x00\x00\x29LPOVERLAPPED',b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x6ELPSECURITY_ATTRIBUTES',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\xA7OVERLAPPED',b'\x00\x00\x00\xA8PROCESS_INFORMATION',b'\x00\x00\x00\x6EPSECURITY_ATTRIBUTES',b'\x00\x00\x00\xA9SECURITY_ATTRIBUTES',b'\x00\x00\x00\xAASTARTUPINFO',b'\x00\x00\x00\x66wint_t'), ) diff --git a/lib_pypy/_winapi.py b/lib_pypy/_winapi.py --- a/lib_pypy/_winapi.py +++ b/lib_pypy/_winapi.py @@ -1,12 +1,12 @@ """ -Support routines for subprocess module. +Support routines for subprocess and multiprocess module. Currently, this extension module is only required when using the -subprocess module on Windows. +modules on Windows. """ import sys if sys.platform != 'win32': - raise ImportError("The '_subprocess' module is only available on Windows") + raise ImportError("The '_winapi' module is only available on Windows") # Declare external Win32 functions @@ -14,7 +14,7 @@ _kernel32 = _ffi.dlopen('kernel32') GetVersion = _kernel32.GetVersion - +NULL = _ffi.NULL # Now the _subprocess module implementation @@ -33,13 +33,116 @@ def CreatePipe(attributes, size): handles = _ffi.new("HANDLE[2]") - res = _kernel32.CreatePipe(handles, handles + 1, _ffi.NULL, size) + res = _kernel32.CreatePipe(handles, handles + 1, NULL, size) if not res: raise _WinError() return _handle2int(handles[0]), _handle2int(handles[1]) +def CreateNamedPipe(*args): + handle = _kernel32.CreateNamedPipeW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def CreateFile(*args): + handle = _kernel32.CreateFileW(*args) + if handle == INVALID_HANDLE_VALUE: + raise _WinError() + return handle + +def SetNamedPipeHandleState(namedpipe, mode, max_collection_count, collect_data_timeout): + d0 = _ffi.new('DWORD[1]', [mode]) + if max_collection_count is None: + d1 = NULL + else: + d1 = _ffi.new('DWORD[1]', [max_collection_count]) + if collect_data_timeout is None: + d2 = NULL + else: + d2 = _ffi.new('DWORD[1]', [collect_data_timeout]) + ret = _kernel32.SetNamedPipeHandleState(namedpipe, d0, d1, d2) + if not ret: + raise _WinError() + +class Overlapped(object): + def __init__(self, handle): + self.overlapped = _ffi.new('OVERLAPPED[1]') + self.handle = handle + self.readbuffer = None + self.pending = 0 + self.completed = 0 + self.writebuffer = None + self.overlapped[0].hEvent = \ + _kernel32.CreateEventW(NULL, True, False, NULL) + + def __del__(self): + # do this somehow else + xxx + err = _kernel32.GetLastError() + bytes = _ffi.new('DWORD[1]') + o = overlapped[0] + if overlapped[0].pending: + if _kernel32.CancelIoEx(o.handle, o.overlapped) & \ + self.GetOverlappedResult(o.handle, o.overlapped, _ffi.addressof(bytes), True): + # The operation is no longer pending, nothing to do + pass + else: + raise RuntimeError('deleting an overlapped strucwith a pending operation not supported') + + @property + def event(self): + return None + + def GetOverlappedResult(self, wait): + transferred = _ffi.new('DWORD[1]', [0]) + res = _kernel32.GetOverlappedResult(self.handle, self.overlapped, transferred, wait != 0) + if not res: + res = GetLastError() + if res in (ERROR_SUCCESS, ERROR_MORE_DATA, ERROR_OPERATION_ABORTED): + self.completed = 1 + self.pending = 0 + elif res == ERROR_IO_INCOMPLETE: + pass + else: + self.pending = 0 + raise _WinError() + if self.completed and self.read_buffer: + if transferred != len(self.read_buffer): + raise _WinError() + return transferred[0], err + + def getbuffer(self): + xxx + return None + + def cancel(self): + xxx + return None + + +def ConnectNamedPipe(handle, overlapped=False): + if overlapped: + ov = Overlapped(handle) + else: + ov = Overlapped(None) + success = _kernel32.ConnectNamedPipe(handle, ov.overlapped) + if overlapped: + # Overlapped ConnectNamedPipe never returns a success code + assert success == 0 + err = _kernel32.GetLastError() + if err == ERROR_IO_PENDING: + overlapped[0].pending = 1 + elif err == ERROR_PIPE_CONNECTED: + _kernel32.SetEvent(ov.overlapped[0].hEvent) + else: + del ov + raise _WinError() + return ov + elif not success: + raise _WinError() + def GetCurrentProcess(): return _handle2int(_kernel32.GetCurrentProcess()) @@ -155,6 +258,7 @@ raise _WinError() return _ffi.string(buf) +# #define macros from WinBase.h and elsewhere STD_INPUT_HANDLE = -10 STD_OUTPUT_HANDLE = -11 STD_ERROR_HANDLE = -12 @@ -171,3 +275,52 @@ CREATE_UNICODE_ENVIRONMENT = 0x400 STILL_ACTIVE = 259 _MAX_PATH = 260 + +ERROR_SUCCESS = 0 +ERROR_NETNAME_DELETED = 64 +ERROR_BROKEN_PIPE = 109 +ERROR_MORE_DATA = 234 +ERROR_PIPE_CONNECTED = 535 +ERROR_OPERATION_ABORTED = 995 +ERROR_IO_INCOMPLETE = 996 +ERROR_IO_PENDING = 997 + +PIPE_ACCESS_INBOUND = 0x00000001 +PIPE_ACCESS_OUTBOUND = 0x00000002 +PIPE_ACCESS_DUPLEX = 0x00000003 +PIPE_WAIT = 0x00000000 +PIPE_NOWAIT = 0x00000001 +PIPE_READMODE_BYTE = 0x00000000 +PIPE_READMODE_MESSAGE = 0x00000002 +PIPE_TYPE_BYTE = 0x00000000 +PIPE_TYPE_MESSAGE = 0x00000004 +PIPE_ACCEPT_REMOTE_CLIENTS = 0x00000000 +PIPE_REJECT_REMOTE_CLIENTS = 0x00000008 + +GENERIC_READ = 0x80000000 +GENERIC_WRITE = 0x40000000 +GENERIC_EXECUTE= 0x20000000 +GENERIC_ALL = 0x10000000 +INVALID_HANDLE_VALUE = -1 +FILE_FLAG_WRITE_THROUGH = 0x80000000 +FILE_FLAG_OVERLAPPED = 0x40000000 +FILE_FLAG_NO_BUFFERING = 0x20000000 +FILE_FLAG_RANDOM_ACCESS = 0x10000000 +FILE_FLAG_SEQUENTIAL_SCAN = 0x08000000 +FILE_FLAG_DELETE_ON_CLOSE = 0x04000000 +FILE_FLAG_BACKUP_SEMANTICS = 0x02000000 +FILE_FLAG_POSIX_SEMANTICS = 0x01000000 +FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 +FILE_FLAG_OPEN_NO_RECALL = 0x00100000 +FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000 + +NMPWAIT_WAIT_FOREVER = 0xffffffff +NMPWAIT_NOWAIT = 0x00000001 +NMPWAIT_USE_DEFAULT_WAIT = 0x00000000 + +CREATE_NEW = 1 +CREATE_ALWAYS = 2 +OPEN_EXISTING = 3 +OPEN_ALWAYS = 4 +TRUNCATE_EXISTING = 5 + diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -40,3 +40,11 @@ .. branch: memory-accounting Improve way to describe memory + +.. branch: msvc14 + +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: winapi + +Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -1,6 +1,7 @@ import py import pytest import struct +import sys from pypy.interpreter.unicodehelper import ( encode_utf8, decode_utf8, unicode_encode_utf_32_be, str_decode_utf_32_be) from pypy.interpreter.unicodehelper import encode_utf8sp, decode_utf8sp @@ -51,7 +52,10 @@ py.test.raises(Hit, decode_utf8, space, "\xed\xb0\x80") py.test.raises(Hit, decode_utf8, space, "\xed\xa0\x80\xed\xb0\x80") got = decode_utf8(space, "\xf0\x90\x80\x80") - assert map(ord, got) == [0x10000] + if sys.maxunicode > 65535: + assert map(ord, got) == [0x10000] + else: + assert map(ord, got) == [55296, 56320] def test_decode_utf8_allow_surrogates(): sp = FakeSpace() diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -148,6 +148,7 @@ # which never raises UnicodeEncodeError. Surrogate pairs are then # allowed, either paired or lone. A paired surrogate is considered # like the non-BMP character it stands for. See also *_utf8sp(). + assert isinstance(uni, unicode) return runicode.unicode_encode_utf_8( uni, len(uni), "strict", errorhandler=encode_error_handler(space), diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -87,7 +87,6 @@ 'hidden_applevel' : 'interp_magic.hidden_applevel', 'lookup_special' : 'interp_magic.lookup_special', 'do_what_I_mean' : 'interp_magic.do_what_I_mean', - 'validate_fd' : 'interp_magic.validate_fd', 'resizelist_hint' : 'interp_magic.resizelist_hint', 'newlist_hint' : 'interp_magic.newlist_hint', 'add_memory_pressure' : 'interp_magic.add_memory_pressure', diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -105,14 +105,6 @@ raise oefmt(space.w_TypeError, "expecting dict or list or set object") return space.newtext(name) - - at unwrap_spec(fd='c_int') -def validate_fd(space, fd): - try: - rposix.validate_fd(fd) - except OSError as e: - raise wrap_oserror(space, e) - @unwrap_spec(sizehint=int) def resizelist_hint(space, w_list, sizehint): """ Reallocate the underlying storage of the argument list to sizehint """ diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -288,6 +288,15 @@ ffi.cast("unsigned short *", c)[1] += 500 assert list(a) == [10000, 20500, 30000] + def test_from_buffer_BytesIO(self): + from _cffi_backend import FFI + import _io + ffi = FFI() + a = _io.BytesIO(b"Hello, world!") + buf = a.getbuffer() + # used to segfault + raises(TypeError, ffi.from_buffer, buf) + def test_memmove(self): import sys import _cffi_backend as _cffi1_backend diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -589,12 +589,17 @@ assert b'\x00'.decode('unicode-internal', 'ignore') == '' def test_backslashreplace(self): + import sys import codecs sin = u"a\xac\u1234\u20ac\u8000\U0010ffff" - expected = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" - assert sin.encode('ascii', 'backslashreplace') == expected - expected = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" - assert sin.encode("iso-8859-15", "backslashreplace") == expected + if sys.maxunicode > 65535: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" + else: + expected_ascii = b"a\\xac\\u1234\\u20ac\\u8000\\udbff\\udfff" + expected_8859 = b"a\xac\\u1234\xa4\\u8000\\udbff\\udfff" + assert sin.encode('ascii', 'backslashreplace') == expected_ascii + assert sin.encode("iso-8859-15", "backslashreplace") == expected_8859 assert 'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == b'a\\xac\u1234\u20ac\u8000' assert b'\x00\x60\x80'.decode( diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -48,6 +48,9 @@ finally: w_bytesio.seek(tell) + def get_raw_address(self): + raise ValueError("BytesIOBuffer does not have a raw address") + class W_BytesIO(W_BufferedIOBase): import_from_mixin(RStringIO) diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1202,12 +1202,16 @@ start, stop, step, size = self.space.decode_index4(w_idx, self.len) assert step != 0 if w_item.len != size or self is w_item: - # XXX this is a giant slow hack - w_lst = self.descr_tolist(space) - w_item = space.call_method(w_item, 'tolist') - space.setitem(w_lst, w_idx, w_item) - self.setlen(0) - self.fromsequence(w_lst) + if start == self.len and step > 0: + # we actually want simply extend() + self.extend(w_item) + else: + # XXX this is a giant slow hack + w_lst = self.descr_tolist(space) + w_item = space.call_method(w_item, 'tolist') + space.setitem(w_lst, w_idx, w_item) + self.setlen(0) + self.fromsequence(w_lst) else: j = 0 buf = self.get_buffer() diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -300,6 +300,12 @@ b = self.array('u', u'hi') assert len(b) == 2 and b[0] == 'h' and b[1] == 'i' + def test_setslice_to_extend(self): + a = self.array('i') + a[0:1] = self.array('i', [9]) + a[1:5] = self.array('i', [99]) + assert list(a) == [9, 99] + def test_sequence(self): a = self.array('i', [1, 2, 3, 4]) assert len(a) == 4 diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -31,7 +31,7 @@ from pypy.module.__builtin__.descriptor import W_Property #from pypy.module.micronumpy.base import W_NDimArray from rpython.rlib.entrypoint import entrypoint_lowlevel -from rpython.rlib.rposix import is_valid_fd, validate_fd +from rpython.rlib.rposix import FdValidator from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize from pypy.module import exceptions @@ -97,25 +97,24 @@ dash = '' def fclose(fp): - if not is_valid_fd(c_fileno(fp)): + try: + with FdValidator(c_fileno(fp)): + return c_fclose(fp) + except IOError: return -1 - return c_fclose(fp) def fwrite(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fwrite(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fwrite(buf, sz, n, fp) def fread(buf, sz, n, fp): - validate_fd(c_fileno(fp)) - return c_fread(buf, sz, n, fp) + with FdValidator(c_fileno(fp)): + return c_fread(buf, sz, n, fp) _feof = rffi.llexternal('feof', [FILEP], rffi.INT) def feof(fp): - validate_fd(c_fileno(fp)) - return _feof(fp) - -def is_valid_fp(fp): - return is_valid_fd(c_fileno(fp)) + with FdValidator(c_fileno(fp)): + return _feof(fp) pypy_decl = 'pypy_decl.h' udir.join(pypy_decl).write("/* Will be filled later */\n") diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -2,9 +2,10 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.pyobject import (PyObject, make_ref, make_typedescr, - decref) + decref, as_pyobj, incref) from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, cpython_struct, - PyObjectFields, cts, parse_dir, bootstrap_function, slot_function) + PyObjectFields, cts, parse_dir, bootstrap_function, slot_function, + Py_TPFLAGS_HEAPTYPE) from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError @@ -31,6 +32,10 @@ w_type = space.getattr(w_datetime, space.newtext("date")) datetimeAPI.c_DateType = rffi.cast( PyTypeObjectPtr, make_ref(space, w_type)) + # convenient place to modify this, needed since the make_typedescr attach + # links the "wrong" struct to W_DateTime_Date, which in turn is needed + # because datetime, with a tzinfo entry, inherits from date, without one + datetimeAPI.c_DateType.c_tp_basicsize = rffi.sizeof(PyObject.TO) w_type = space.getattr(w_datetime, space.newtext("datetime")) datetimeAPI.c_DateTimeType = rffi.cast( @@ -128,6 +133,7 @@ # W_DateTime_Date->tp_dealloc make_typedescr(W_DateTime_Date.typedef, basestruct=PyDateTime_DateTime.TO, + attach=type_attach, dealloc=date_dealloc, ) @@ -138,8 +144,10 @@ def type_attach(space, py_obj, w_obj, w_userdata=None): '''Fills a newly allocated py_obj from the w_obj - Can be called with a datetime, or a time ''' + if space.type(w_obj).name == 'date': + # No tzinfo + return py_datetime = rffi.cast(PyDateTime_Time, py_obj) w_tzinfo = space.getattr(w_obj, space.newtext('tzinfo')) if space.is_none(w_tzinfo): diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -5,7 +5,7 @@ from rpython.rlib.rarithmetic import widen from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, - cpython_struct, is_valid_fp) + cpython_struct) from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject @@ -155,22 +155,19 @@ BUF_SIZE = 8192 source = "" filename = rffi.charp2str(filename) - buf = lltype.malloc(rffi.CCHARP.TO, BUF_SIZE, flavor='raw') - if not is_valid_fp(fp): - lltype.free(buf, flavor='raw') - PyErr_SetFromErrno(space, space.w_IOError) - return None - try: + with rffi.scoped_alloc_buffer(BUF_SIZE) as buf: while True: - count = fread(buf, 1, BUF_SIZE, fp) + try: + count = fread(buf.raw, 1, BUF_SIZE, fp) + except OSError: + PyErr_SetFromErrno(space, space.w_IOError) + return count = rffi.cast(lltype.Signed, count) - source += rffi.charpsize2str(buf, count) + source += rffi.charpsize2str(buf.raw, count) if count < BUF_SIZE: if feof(fp): break PyErr_SetFromErrno(space, space.w_IOError) - finally: - lltype.free(buf, flavor='raw') return run_string(space, source, filename, start, w_globals, w_locals) # Undocumented function! diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -131,7 +131,6 @@ def wrap_inquirypred(space, w_self, w_args, func): func_inquiry = rffi.cast(inquiry, func) check_num_args(space, w_args, 0) - args_w = space.fixedview(w_args) res = generic_cpy_call(space, func_inquiry, w_self) res = rffi.cast(lltype.Signed, res) if res == -1: @@ -411,286 +410,334 @@ return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) -from rpython.rlib.nonconst import NonConstant +SLOT_FACTORIES = {} +def slot_factory(tp_name): + def decorate(func): + SLOT_FACTORIES[tp_name] = func + return func + return decorate -def build_slot_tp_function(space, typedef, name): + +SLOTS = {} + at specialize.memo() +def get_slot_tp_function(space, typedef, name, method_name): + """Return a description of the slot C function to use for the built-in + type for 'typedef'. The 'name' is the slot name. This is a memo + function that, after translation, returns one of a built-in finite set. + """ + key = (typedef, name) + try: + return SLOTS[key] + except KeyError: + slot_func = SLOT_FACTORIES[name](space, typedef, name, method_name) + api_func = slot_func.api_func if slot_func else None + SLOTS[key] = api_func + return api_func + + +def make_unary_slot(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - - handled = False - # unary functions - for tp_name, attr in [('tp_as_async.c_am_await', '__await__'), - ('tp_as_async.c_am_anext', '__anext__'), - ('tp_as_async.c_am_aiter', '__aiter__'), - ('tp_as_number.c_nb_int', '__int__'), - ('tp_as_number.c_nb_long', '__long__'), - ('tp_as_number.c_nb_float', '__float__'), - ('tp_as_number.c_nb_negative', '__neg__'), - ('tp_as_number.c_nb_positive', '__pos__'), - ('tp_as_number.c_nb_absolute', '__abs__'), - ('tp_as_number.c_nb_invert', '__invert__'), - ('tp_as_number.c_nb_index', '__index__'), - ('tp_str', '__str__'), - ('tp_repr', '__repr__'), - ('tp_iter', '__iter__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self): - return space.call_function(slot_fn, w_self) - handled = True - - for tp_name, attr in [('tp_hash', '__hash__'), - ('tp_as_sequence.c_sq_length', '__len__'), - ('tp_as_mapping.c_mp_length', '__len__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - @slot_function([PyObject], lltype.Signed, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_obj): - return space.int_w(space.call_function(slot_fn, w_obj)) - handled = True - - - # binary functions - for tp_name, attr in [('tp_as_number.c_nb_add', '__add__'), - ('tp_as_number.c_nb_subtract', '__sub__'), - ('tp_as_number.c_nb_multiply', '__mul__'), - ('tp_as_number.c_nb_divide', '__div__'), - ('tp_as_number.c_nb_remainder', '__mod__'), - ('tp_as_number.c_nb_divmod', '__divmod__'), - ('tp_as_number.c_nb_lshift', '__lshift__'), - ('tp_as_number.c_nb_rshift', '__rshift__'), - ('tp_as_number.c_nb_and', '__and__'), - ('tp_as_number.c_nb_xor', '__xor__'), - ('tp_as_number.c_nb_or', '__or__'), - ('tp_as_sequence.c_sq_concat', '__add__'), - ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'), - ('tp_as_mapping.c_mp_subscript', '__getitem__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg): - return space.call_function(slot_fn, w_self, w_arg) - handled = True - - # binary-with-Py_ssize_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_repeat', '__mul__'), - ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, Py_ssize_t], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg): - return space.call_function(slot_fn, w_self, space.newint(arg)) - handled = True - - # ternary functions - for tp_name, attr in [('tp_as_number.c_nb_power', '__pow__'), - ]: - if name == tp_name: - slot_fn = w_type.lookup(attr) - if slot_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, w_arg2): - return space.call_function(slot_fn, w_self, w_arg1, w_arg2) - handled = True - # ternary-with-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, w_arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, w_arg1, w_arg2) - else: - space.call_function(slot_del, w_self, w_arg1) - return 0 - handled = True - # ternary-Py_size_t-void returning-Py_size_t-type - for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'), - ]: - if name == tp_name: - slot_ass = w_type.lookup(attr) - if slot_ass is None: - return - slot_del = w_type.lookup('__delitem__') - if slot_del is None: - return - - @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_func(space, w_self, arg1, arg2): - if arg2: - w_arg2 = from_ref(space, rffi.cast(PyObject, arg2)) - space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2) - else: - space.call_function(slot_del, w_self, space.newint(arg1)) - return 0 - handled = True - if handled: - pass - elif name == 'tp_setattro': - setattr_fn = w_type.lookup('__setattr__') - delattr_fn = w_type.lookup('__delattr__') - if setattr_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1) - @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) - def slot_tp_setattro(space, w_self, w_name, w_value): - if w_value is not None: - space.call_function(setattr_fn, w_self, w_name, w_value) - else: - space.call_function(delattr_fn, w_self, w_name) - return 0 - slot_func = slot_tp_setattro - elif name == 'tp_getattro': - getattr_fn = w_type.lookup('__getattribute__') - if getattr_fn is None: - return - - @slot_function([PyObject, PyObject], PyObject) - @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) - def slot_tp_getattro(space, w_self, w_name): - return space.call_function(getattr_fn, w_self, w_name) - slot_func = slot_tp_getattro - - elif name == 'tp_call': - call_fn = w_type.lookup('__call__') - if call_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_call(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(call_fn, args) - slot_func = slot_tp_call - - elif name == 'tp_iternext': - iternext_fn = w_type.lookup('__next__') - if iternext_fn is None: - return - - @slot_function([PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_iternext(space, w_self): - try: - return space.call_function(iternext_fn, w_self) - except OperationError as e: - if not e.match(space, space.w_StopIteration): - raise - return None - slot_func = slot_tp_iternext - - elif name == 'tp_init': - init_fn = w_type.lookup('__init__') - if init_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_init(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - space.call_args(init_fn, args) - return 0 - slot_func = slot_tp_init - elif name == 'tp_new': - new_fn = w_type.lookup('__new__') - if new_fn is None: - return - - @slot_function([PyTypeObjectPtr, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_new(space, w_self, w_args, w_kwds): - args = Arguments(space, [w_self], - w_stararg=w_args, w_starstararg=w_kwds) - return space.call_args(space.get(new_fn, w_self), args) - slot_func = slot_tp_new - elif name == 'tp_as_buffer.c_bf_getbuffer': - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - buff_w = slot_from___buffer__(space, typedef, buff_fn) - elif typedef.buffer: - buff_w = slot_from_buffer_w(space, typedef, buff_fn) - else: - return - slot_func = buff_w - elif name == 'tp_descr_get': - get_fn = w_type.lookup('__get__') - if get_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], PyObject) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_get(space, w_self, w_obj, w_value): - if w_obj is None: - w_obj = space.w_None - return space.call_function(get_fn, w_self, w_obj, w_value) - slot_func = slot_tp_descr_get - elif name == 'tp_descr_set': - set_fn = w_type.lookup('__set__') - delete_fn = w_type.lookup('__delete__') - if set_fn is None and delete_fn is None: - return - - @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) - def slot_tp_descr_set(space, w_self, w_obj, w_value): - if w_value is not None: - if set_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __set__", typedef.name) - space.call_function(set_fn, w_self, w_obj, w_value) - else: - if delete_fn is None: - raise oefmt(space.w_TypeError, - "%s object has no __delete__", typedef.name) - space.call_function(delete_fn, w_self, w_obj) - return 0 - slot_func = slot_tp_descr_set - else: - # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce - # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length - # richcmpfunc(s) + slot_fn = w_type.lookup(attr) + if slot_fn is None: return + @slot_function([PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self): + return space.call_function(slot_fn, w_self) return slot_func +UNARY_SLOTS = [ + 'tp_as_async.c_am_await', + 'tp_as_async.c_am_anext', + 'tp_as_async.c_am_aiter', + 'tp_as_number.c_nb_int', + 'tp_as_number.c_nb_long', + 'tp_as_number.c_nb_float', + 'tp_as_number.c_nb_negative', + 'tp_as_number.c_nb_positive', + 'tp_as_number.c_nb_absolute', + 'tp_as_number.c_nb_invert', + 'tp_as_number.c_nb_index', + 'tp_as_number.c_nb_hex', + 'tp_as_number.c_nb_oct', + 'tp_str', + 'tp_repr', + 'tp_iter'] +for name in UNARY_SLOTS: + slot_factory(name)(make_unary_slot) + +def make_unary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + @slot_function([PyObject], lltype.Signed, error=-1) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_obj): + return space.int_w(space.call_function(slot_fn, w_obj)) + return slot_func + +UNARY_SLOTS_INT = [ + 'tp_hash', + 'tp_as_sequence.c_sq_length', + 'tp_as_mapping.c_mp_length',] +for name in UNARY_SLOTS_INT: + slot_factory(name)(make_unary_slot_int) + + +def make_binary_slot(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg): + return space.call_function(slot_fn, w_self, w_arg) + return slot_func + +BINARY_SLOTS = [ + 'tp_as_number.c_nb_add', + 'tp_as_number.c_nb_subtract', + 'tp_as_number.c_nb_multiply', + 'tp_as_number.c_nb_divide', + 'tp_as_number.c_nb_remainder', + 'tp_as_number.c_nb_divmod', + 'tp_as_number.c_nb_lshift', + 'tp_as_number.c_nb_rshift', + 'tp_as_number.c_nb_and', + 'tp_as_number.c_nb_xor', + 'tp_as_number.c_nb_or', + 'tp_as_sequence.c_sq_concat', + 'tp_as_sequence.c_sq_inplace_concat', + 'tp_as_mapping.c_mp_subscript',] +for name in BINARY_SLOTS: + slot_factory(name)(make_binary_slot) + + +def make_binary_slot_int(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, Py_ssize_t], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, arg): + return space.call_function(slot_fn, w_self, space.newint(arg)) + return slot_func + +BINARY_SLOTS_INT = [ + 'tp_as_sequence.c_sq_item', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_repeat', + 'tp_as_sequence.c_sq_inplace_repeat',] +for name in BINARY_SLOTS_INT: + slot_factory(name)(make_binary_slot_int) + + at slot_factory('tp_as_number.c_nb_power') +def make_nb_power(space, typedef, name, attr): + w_type = space.gettypeobject(typedef) + slot_fn = w_type.lookup(attr) + if slot_fn is None: + return + + @slot_function([PyObject, PyObject, PyObject], PyObject) + @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name)) + def slot_func(space, w_self, w_arg1, w_arg2): + return space.call_function(slot_fn, w_self, w_arg1, w_arg2) + return slot_func + From pypy.commits at gmail.com Thu Mar 1 06:41:57 2018 From: pypy.commits at gmail.com (amauryfa) Date: Thu, 01 Mar 2018 03:41:57 -0800 (PST) Subject: [pypy-commit] pypy py3.6: CPython Issue #14711: os.stat_float_times() has been deprecated. Message-ID: <5a97e705.8ec11c0a.8830a.9cd1@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93933:1297ffb4dd1a Date: 2018-02-26 01:57 +0100 http://bitbucket.org/pypy/pypy/changeset/1297ffb4dd1a/ Log: CPython Issue #14711: os.stat_float_times() has been deprecated. diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -586,6 +586,8 @@ future calls return ints. If newval is omitted, return the current setting. """ + space.warn(space.newtext("stat_float_times() is deprecated"), + space.w_DeprecationWarning) state = space.fromcache(StatState) if newval == -1: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -162,8 +162,12 @@ def test_stat_float_times(self): path = self.path posix = self.posix - current = posix.stat_float_times() - assert current is True + import warnings + with warnings.catch_warnings(record=True) as l: + warnings.simplefilter('always') + current = posix.stat_float_times() + assert current is True + assert "stat_float_times" in repr(l[0].message) try: posix.stat_float_times(True) st = posix.stat(path) @@ -180,6 +184,7 @@ finally: posix.stat_float_times(current) + def test_stat_result(self): st = self.posix.stat_result((0, 0, 0, 0, 0, 0, 0, 41, 42.1, 43)) assert st.st_atime == 41 From pypy.commits at gmail.com Thu Mar 1 16:11:39 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 01 Mar 2018 13:11:39 -0800 (PST) Subject: [pypy-commit] pypy py3.5: Cleanup duplication between setup_buffer_procs() and make_bf_getbuffer() by killing the former Message-ID: <5a986c8b.8ec11c0a.8830a.1724@mx.google.com> Author: Ronan Lamy Branch: py3.5 Changeset: r93934:235304efbb73 Date: 2018-03-01 21:10 +0000 http://bitbucket.org/pypy/pypy/changeset/235304efbb73/ Log: Cleanup duplication between setup_buffer_procs() and make_bf_getbuffer() by killing the former diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -189,33 +189,6 @@ decref(space, view.c_obj) return 0 -def fill_buffer(space, view, pybuf, py_obj): - view.c_buf = cts.cast('void *', pybuf.get_raw_address()) - view.c_obj = py_obj - if py_obj: - incref(space, py_obj) - view.c_len = pybuf.getlength() - view.c_itemsize = pybuf.getitemsize() - rffi.setintfield(view, 'c_readonly', int(pybuf.readonly)) - rffi.setintfield(view, 'c_ndim', pybuf.getndim()) - view.c_format = rffi.str2charp(pybuf.getformat()) - shape = pybuf.getshape() - if not shape: - view.c_shape = lltype.nullptr(Py_ssize_tP.TO) - else: - view.c_shape = cts.cast('Py_ssize_t*', view.c__shape) - for i, n in enumerate(shape): - view.c_shape[i] = n - strides = pybuf.getstrides() - if not strides: - view.c_strides = lltype.nullptr(Py_ssize_tP.TO) - else: - view.c_strides = cts.cast('Py_ssize_t*', view.c__strides) - for i, n in enumerate(strides): - view.c_strides[i] = n - view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) - view.c_internal = lltype.nullptr(rffi.VOIDP.TO) - DEFAULT_FMT = rffi.str2charp("B") @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -688,12 +688,46 @@ return space.call_args(space.get(new_fn, w_self), args) return slot_tp_new + at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], + rffi.INT_real, error=-1) +def bytes_getbuffer(space, w_str, view, flags): + from pypy.module.cpyext.bytesobject import PyBytes_AsString + from pypy.module.cpyext.buffer import PyBuffer_FillInfo + c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str)) + return PyBuffer_FillInfo(space, view, w_str, c_buf, + space.len_w(w_str), 1, flags) + +def slot_from_buffer_w(space, typedef): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, c_view, flags): + w_obj = w_self + if c_view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + c_view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + c_view.c_obj = make_ref(space, w_s) + c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(c_view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, c_view) + return ret + return 0 + return buff_w + @slot_factory('tp_as_buffer.c_bf_getbuffer') def make_bf_getbuffer(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - return slot_from___buffer__(space, typedef, buff_fn) + if space.is_w(w_type, space.w_bytes): + return bytes_getbuffer elif typedef.buffer: return slot_from_buffer_w(space, typedef) else: @@ -739,59 +773,6 @@ return slot_tp_descr_set -def slot_from___buffer__(space, typedef, buff_fn): - name = 'bf_getbuffer' - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name, typedef.name)) - def buff_w(space, w_self, c_view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if c_view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - c_view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - c_view.c_obj = make_ref(space, w_s) - c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(c_view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, c_view) - return ret - return 0 - return buff_w - -def slot_from_buffer_w(space, typedef): - name = 'bf_getbuffer' - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name, typedef.name)) - def buff_w(space, w_self, c_view, flags): - w_obj = w_self - if c_view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - c_view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - c_view.c_obj = make_ref(space, w_s) - c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(c_view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, c_view) - return ret - return 0 - return buff_w - missing_wrappers = ['wrap_indexargfunc', 'wrap_del'] for name in missing_wrappers: assert name not in globals() diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -519,33 +519,6 @@ realize=type_realize, dealloc=type_dealloc) - at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) -def bytes_getbuffer(space, w_str, view, flags): - from pypy.module.cpyext.bytesobject import PyBytes_AsString - from pypy.module.cpyext.buffer import PyBuffer_FillInfo - c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str)) - return PyBuffer_FillInfo(space, view, w_str, c_buf, - space.len_w(w_str), 1, flags) - - at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) -def bf_getbuffer(space, w_obj, view, flags): - from pypy.module.cpyext.buffer import fill_buffer - buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags)) - fill_buffer(space, view, buf, as_pyobj(space, w_obj)) - return 0 - -def setup_buffer_procs(space, w_type, pto): - bufspec = w_type.layout.typedef.buffer - if not bufspec: - return - c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - lltype.render_immortal(c_buf) - if space.is_w(w_type, space.w_bytes): - c_buf.c_bf_getbuffer = llslot(space, bytes_getbuffer) - else: - c_buf.c_bf_getbuffer = llslot(space, bf_getbuffer) - pto.c_tp_as_buffer = c_buf - @slot_function([PyObject], lltype.Void) def type_dealloc(space, obj): from pypy.module.cpyext.object import _dealloc @@ -604,8 +577,6 @@ pto.c_tp_itemsize = 1 elif space.is_w(w_type, space.w_tuple): pto.c_tp_itemsize = rffi.sizeof(PyObject) - # buffer protocol - setup_buffer_procs(space, w_type, pto) state = space.fromcache(State) pto.c_tp_free = state.C.PyObject_Free @@ -723,7 +694,6 @@ pto.c_tp_as_buffer = base.c_tp_as_buffer if base.c_tp_as_buffer: # inherit base.c_tp_as_buffer functions not inherited from w_type - # note: builtin types are handled in setup_buffer_procs pto_as = pto.c_tp_as_buffer base_as = base.c_tp_as_buffer if not pto_as.c_bf_getbuffer: From pypy.commits at gmail.com Thu Mar 1 16:41:35 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 01 Mar 2018 13:41:35 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: utf8-ization Message-ID: <5a98738f.d4abdf0a.71640.69ec@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5872:b41ddb578d82 Date: 2018-03-01 22:41 +0100 http://bitbucket.org/pypy/extradoc/changeset/b41ddb578d82/ Log: utf8-ization diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -20,7 +20,7 @@ Antonio Cuni 18.3/23.3 Ermina, exact dates to be confirmed Alexander Schremmer 18.3/20.3 Ermina Ronan Lamy 17.3/23.3 Ermina -Ren� Dudfield 18.3/24.3 Ermina +René Dudfield 18.3/24.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there From pypy.commits at gmail.com Thu Mar 1 19:24:01 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Mar 2018 16:24:01 -0800 (PST) Subject: [pypy-commit] pypy default: port winreg changes from 0684dbb33360 Message-ID: <5a9899a1.17b0df0a.cf321.5f6f@mx.google.com> Author: Matti Picus Branch: Changeset: r93935:645bfc512f05 Date: 2018-03-01 16:17 -0800 http://bitbucket.org/pypy/pypy/changeset/645bfc512f05/ Log: port winreg changes from 0684dbb33360 diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -169,7 +169,7 @@ The docs imply key must be in the HKEY_USER or HKEY_LOCAL_MACHINE tree""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegLoadKey(hkey, subkey, filename) + ret = rwinreg.RegLoadKeyA(hkey, subkey, filename) if ret != 0: raiseWindowsError(space, ret, 'RegLoadKey') @@ -188,7 +188,7 @@ The caller of this method must possess the SeBackupPrivilege security privilege. This function passes NULL for security_attributes to the API.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegSaveKey(hkey, filename, None) + ret = rwinreg.RegSaveKeyA(hkey, filename, None) if ret != 0: raiseWindowsError(space, ret, 'RegSaveKey') @@ -219,7 +219,7 @@ else: subkey = space.text_w(w_subkey) with rffi.scoped_str2charp(value) as dataptr: - ret = rwinreg.RegSetValue(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value)) + ret = rwinreg.RegSetValueA(hkey, subkey, rwinreg.REG_SZ, dataptr, len(value)) if ret != 0: raiseWindowsError(space, ret, 'RegSetValue') @@ -240,7 +240,7 @@ else: subkey = space.text_w(w_subkey) with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, subkey, None, bufsize_p) + ret = rwinreg.RegQueryValueA(hkey, subkey, None, bufsize_p) bufSize = intmask(bufsize_p[0]) if ret == rwinreg.ERROR_MORE_DATA: bufSize = 256 @@ -249,7 +249,7 @@ while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, subkey, buf, bufsize_p) + ret = rwinreg.RegQueryValueA(hkey, subkey, buf, bufsize_p) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry bufSize *= 2 @@ -423,7 +423,7 @@ hkey = hkey_w(w_hkey, space) buf, buflen = convert_to_regdata(space, w_value, typ) try: - ret = rwinreg.RegSetValueEx(hkey, value_name, 0, typ, buf, buflen) + ret = rwinreg.RegSetValueExA(hkey, value_name, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -441,7 +441,7 @@ subkey = space.text_w(w_subkey) null_dword = lltype.nullptr(rwin32.LPDWORD.TO) with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, null_dword, + ret = rwinreg.RegQueryValueExA(hkey, subkey, null_dword, null_dword, None, retDataSize) bufSize = intmask(retDataSize[0]) if ret == rwinreg.ERROR_MORE_DATA: @@ -453,7 +453,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, subkey, null_dword, + ret = rwinreg.RegQueryValueExA(hkey, subkey, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -484,7 +484,7 @@ If the function fails, an exception is raised.""" hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, subkey, rethkey) + ret = rwinreg.RegCreateKeyA(hkey, subkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'CreateKey') return W_HKEY(space, rethkey[0]) @@ -504,7 +504,7 @@ If the function fails, an exception is raised.""" hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, subkey, res, None, 0, + ret = rwinreg.RegCreateKeyExA(hkey, subkey, res, None, 0, sam, None, rethkey, lltype.nullptr(rwin32.LPDWORD.TO)) if ret != 0: @@ -524,7 +524,7 @@ If the method succeeds, the entire key, including all of its values, is removed. If the method fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteKey(hkey, subkey) + ret = rwinreg.RegDeleteKeyA(hkey, subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteKey') @@ -535,7 +535,7 @@ key is an already open key, or any one of the predefined HKEY_* constants. value is a string that identifies the value to remove.""" hkey = hkey_w(w_hkey, space) - ret = rwinreg.RegDeleteValue(hkey, subkey) + ret = rwinreg.RegDeleteValueA(hkey, subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteValue') @@ -553,7 +553,7 @@ If the function fails, an EnvironmentError exception is raised.""" hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, subkey, res, sam, rethkey) + ret = rwinreg.RegOpenKeyExA(hkey, subkey, res, sam, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegOpenKeyEx') return W_HKEY(space, rethkey[0]) @@ -578,7 +578,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyA( hkey, None, null_dword, null_dword, null_dword, null_dword, null_dword, null_dword, retValueSize, retDataSize, @@ -598,7 +598,7 @@ bufDataSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegEnumValue( + ret = rwinreg.RegEnumValueA( hkey, index, valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: @@ -643,7 +643,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, 257) as buf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: retValueSize[0] = r_uint(257) # includes NULL terminator - ret = rwinreg.RegEnumKeyEx(hkey, index, buf, retValueSize, + ret = rwinreg.RegEnumKeyExA(hkey, index, buf, retValueSize, null_dword, None, null_dword, lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: @@ -665,7 +665,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as nValues: with lltype.scoped_alloc(rwin32.PFILETIME.TO, 1) as ft: null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyA( hkey, None, null_dword, null_dword, nSubKeys, null_dword, null_dword, nValues, null_dword, null_dword, @@ -692,7 +692,7 @@ machine = space.text_or_none_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) + ret = rwinreg.RegConnectRegistryA(machine, hkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegConnectRegistry') return W_HKEY(space, rethkey[0]) diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,77 +47,107 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'A' +suffix = 'W' +def get_traits(suffix): + RegSetValue = external( + 'RegSetValue' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) -RegSetValue = external( - 'RegSetValue' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) + RegSetValueEx = external( + 'RegSetValueEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, + rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], + rffi.LONG) -RegSetValueEx = external( - 'RegSetValueEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, - rwin32.DWORD, rffi.CCHARP, rwin32.DWORD], - rffi.LONG) + RegQueryValue = external( + 'RegQueryValue' + suffix, + [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], + rffi.LONG) -RegQueryValue = external( - 'RegQueryValue' + suffix, - [HKEY, rffi.CCHARP, rffi.CCHARP, rwin32.PLONG], - rffi.LONG) + RegQueryValueEx = external( + 'RegQueryValueEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) -RegQueryValueEx = external( - 'RegQueryValueEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) + RegCreateKey = external( + 'RegCreateKey' + suffix, + [HKEY, rffi.CCHARP, PHKEY], + rffi.LONG) -RegCreateKey = external( - 'RegCreateKey' + suffix, - [HKEY, rffi.CCHARP, PHKEY], - rffi.LONG) + RegCreateKeyEx = external( + 'RegCreateKeyEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, + REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], + rffi.LONG) -RegCreateKeyEx = external( - 'RegCreateKeyEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, rffi.CCHARP, rwin32.DWORD, - REGSAM, rffi.VOIDP, PHKEY, rwin32.LPDWORD], - rffi.LONG) + RegDeleteValue = external( + 'RegDeleteValue' + suffix, + [HKEY, rffi.CCHARP], + rffi.LONG) -RegDeleteValue = external( - 'RegDeleteValue' + suffix, - [HKEY, rffi.CCHARP], - rffi.LONG) + RegDeleteKey = external( + 'RegDeleteKey' + suffix, + [HKEY, rffi.CCHARP], + rffi.LONG) -RegDeleteKey = external( - 'RegDeleteKey' + suffix, - [HKEY, rffi.CCHARP], - rffi.LONG) + RegOpenKeyEx = external( + 'RegOpenKeyEx' + suffix, + [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], + rffi.LONG) -RegOpenKeyEx = external( - 'RegOpenKeyEx' + suffix, - [HKEY, rffi.CCHARP, rwin32.DWORD, REGSAM, PHKEY], - rffi.LONG) + RegEnumValue = external( + 'RegEnumValue' + suffix, + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD], + rffi.LONG) -RegEnumValue = external( - 'RegEnumValue' + suffix, - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD], - rffi.LONG) + RegEnumKeyEx = external( + 'RegEnumKeyEx' + suffix, + [HKEY, rwin32.DWORD, rffi.CCHARP, + rwin32.LPDWORD, rwin32.LPDWORD, + rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) -RegEnumKeyEx = external( - 'RegEnumKeyEx' + suffix, - [HKEY, rwin32.DWORD, rffi.CCHARP, - rwin32.LPDWORD, rwin32.LPDWORD, - rffi.CCHARP, rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) + RegQueryInfoKey = external( + 'RegQueryInfoKey' + suffix, + [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, + rwin32.LPDWORD, rwin32.PFILETIME], + rffi.LONG) -RegQueryInfoKey = external( - 'RegQueryInfoKey' + suffix, - [HKEY, rffi.CCHARP, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.LPDWORD, rwin32.LPDWORD, - rwin32.LPDWORD, rwin32.PFILETIME], - rffi.LONG) + RegLoadKey = external( + 'RegLoadKey' + suffix, + [HKEY, rffi.CCHARP, rffi.CCHARP], + rffi.LONG) + + RegSaveKey = external( + 'RegSaveKey' + suffix, + [HKEY, rffi.CCHARP, rffi.VOIDP], + rffi.LONG) + + RegConnectRegistry = external( + 'RegConnectRegistry' + suffix, + [rffi.CCHARP, HKEY, PHKEY], + rffi.LONG) + + return (RegSetValue, RegSetValueEx, RegQueryValue, RegQueryValueEx, + RegCreateKey, RegCreateKeyEx, RegDeleteValue, RegDeleteKey, + RegOpenKeyEx, RegEnumValue, RegEnumKeyEx, RegQueryInfoKey, + RegLoadKey, RegSaveKey, RegConnectRegistry) + +RegSetValueW, RegSetValueExW, RegQueryValueW, RegQueryValueExW, \ + RegCreateKeyW, RegCreateKeyExW, RegDeleteValueW, RegDeleteKeyW, \ + RegOpenKeyExW, RegEnumValueW, RegEnumKeyExW, RegQueryInfoKeyW, \ + RegLoadKeyW, RegSaveKeyW, RegConnectRegistryW = get_traits('W') + +RegSetValueA, RegSetValueExA, RegQueryValueA, RegQueryValueExA, \ + RegCreateKeyA, RegCreateKeyExA, RegDeleteValueA, RegDeleteKeyA, \ + RegOpenKeyExA, RegEnumValueA, RegEnumKeyExA, RegQueryInfoKeyA, \ + RegLoadKeyA, RegSaveKeyA, RegConnectRegistryA = get_traits('A') RegCloseKey = external( 'RegCloseKey', @@ -129,21 +159,6 @@ [HKEY], rffi.LONG) -RegLoadKey = external( - 'RegLoadKey' + suffix, - [HKEY, rffi.CCHARP, rffi.CCHARP], - rffi.LONG) - -RegSaveKey = external( - 'RegSaveKey' + suffix, - [HKEY, rffi.CCHARP, rffi.VOIDP], - rffi.LONG) - -RegConnectRegistry = external( - 'RegConnectRegistry' + suffix, - [rffi.CCHARP, HKEY, PHKEY], - rffi.LONG) - _ExpandEnvironmentStringsW = external( 'ExpandEnvironmentStringsW', [rffi.CWCHARP, rffi.CWCHARP, rwin32.DWORD], From pypy.commits at gmail.com Thu Mar 1 19:24:03 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Mar 2018 16:24:03 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5a9899a3.97961c0a.8aed5.04f2@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93936:bd80880bbcdc Date: 2018-03-01 16:21 -0800 http://bitbucket.org/pypy/pypy/changeset/bd80880bbcdc/ Log: merge default into py3.5 From pypy.commits at gmail.com Thu Mar 1 19:24:05 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 01 Mar 2018 16:24:05 -0800 (PST) Subject: [pypy-commit] pypy py3.5: cleanup Message-ID: <5a9899a5.46b51c0a.6cdee.38b9@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93937:8e1fc4f86361 Date: 2018-03-01 16:22 -0800 http://bitbucket.org/pypy/pypy/changeset/8e1fc4f86361/ Log: cleanup diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -47,7 +47,7 @@ HKEY = rwin32.HANDLE PHKEY = rffi.CArrayPtr(HKEY) REGSAM = rwin32.DWORD -suffix = 'W' + def get_traits(suffix): RegSetValue = external( 'RegSetValue' + suffix, From pypy.commits at gmail.com Fri Mar 2 18:40:25 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 02 Mar 2018 15:40:25 -0800 (PST) Subject: [pypy-commit] pypy py3.5: test, fix for vs 2010 and above having both EWOULDBLOCK and WSAEWOULDBLOCK Message-ID: <5a99e0e9.c76d1c0a.6cf7b.1be4@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93938:c3f835dbfc65 Date: 2018-03-02 15:39 -0800 http://bitbucket.org/pypy/pypy/changeset/c3f835dbfc65/ Log: test, fix for vs 2010 and above having both EWOULDBLOCK and WSAEWOULDBLOCK diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -814,10 +814,10 @@ def test_recv_send_timeout(self): from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() + cli.settimeout(1.0) cli.connect(self.serv.getsockname()) fileno, addr = self.serv._accept() t = socket(fileno=fileno) - cli.settimeout(1.0) # test recv() timeout t.send(b'*') buf = cli.recv(100) diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -1080,6 +1080,10 @@ EINPROGRESS = cConfig.EINPROGRESS or cConfig.WSAEINPROGRESS EWOULDBLOCK = cConfig.EWOULDBLOCK or cConfig.WSAEWOULDBLOCK EAFNOSUPPORT = cConfig.EAFNOSUPPORT or cConfig.WSAEAFNOSUPPORT +# vs 2010 and above define both the constansts +WSAEINPROGRESS = cConfig.WSAEINPROGRESS or cConfig.EINPROGRESS +WSAEWOULDBLOCK = cConfig.WSAEWOULDBLOCK or cConfig.EWOULDBLOCK +WSAEAFNOSUPPORT = cConfig.WSAEAFNOSUPPORT or cConfig.EAFNOSUPPORT EISCONN = cConfig.EISCONN or cConfig.WSAEISCONN linux = cConfig.linux diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -707,7 +707,8 @@ address.unlock() errno = _c.geterrno() timeout = self.timeout - if timeout > 0.0 and res < 0 and errno == _c.EWOULDBLOCK: + if (timeout > 0.0 and res < 0 and + errno in (_c.EWOULDBLOCK, _c.WSAEWOULDBLOCK)): tv = rffi.make(_c.timeval) rffi.setintfield(tv, 'c_tv_sec', int(timeout)) rffi.setintfield(tv, 'c_tv_usec', @@ -733,7 +734,7 @@ return (self.getsockopt_int(_c.SOL_SOCKET, _c.SO_ERROR), False) elif n == 0: - return (_c.EWOULDBLOCK, True) + return (_c.WSAEWOULDBLOCK, True) else: return (_c.geterrno(), False) From pypy.commits at gmail.com Sat Mar 3 09:21:37 2018 From: pypy.commits at gmail.com (stevie_92) Date: Sat, 03 Mar 2018 06:21:37 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Added myself to winter sprint Message-ID: <5a9aaf71.177a1c0a.873fe.3352@mx.google.com> Author: Stefan Beyer Branch: extradoc Changeset: r5875:0e42fc53e29d Date: 2018-03-03 15:20 +0100 http://bitbucket.org/pypy/extradoc/changeset/0e42fc53e29d/ Log: Added myself to winter sprint diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -21,6 +21,7 @@ Alexander Schremmer 18.3/20.3 Ermina Ronan Lamy 17.3/23.3 Ermina René Dudfield 18.3/24.3 Ermina +Stefan Beyer 18.3/24.3 Ermina ==================== ============== ======================= **NOTE:** lodging is by default in Ermina. Based on past years, there From pypy.commits at gmail.com Sat Mar 3 11:09:27 2018 From: pypy.commits at gmail.com (amauryfa) Date: Sat, 03 Mar 2018 08:09:27 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Some ast validation errors are shown as TypeError Message-ID: <5a9ac8b7.08bb1c0a.a4914.3e9d@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93945:4f89f769905c Date: 2018-03-03 17:07 +0100 http://bitbucket.org/pypy/pypy/changeset/4f89f769905c/ Log: Some ast validation errors are shown as TypeError diff --git a/pypy/interpreter/astcompiler/test/test_validate.py b/pypy/interpreter/astcompiler/test/test_validate.py --- a/pypy/interpreter/astcompiler/test/test_validate.py +++ b/pypy/interpreter/astcompiler/test/test_validate.py @@ -397,7 +397,8 @@ def test_constant(self): node = ast.Constant(self.space.newlist([1]), 0, 0) - self.expr(node, "got an invalid type in Constant: list") + self.expr(node, "got an invalid type in Constant: list", + exc=validate.ValidationTypeError) def test_stdlib_validates(self): stdlib = os.path.join(os.path.dirname(ast.__file__), '../../../lib-python/3') diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py --- a/pypy/interpreter/astcompiler/validate.py +++ b/pypy/interpreter/astcompiler/validate.py @@ -11,12 +11,17 @@ class ValidationError(Exception): + # Will be seen as a ValueError def __init__(self, message): self.message = message def __str__(self): return self.message +class ValidationTypeError(ValidationError): + # Will be seen as a TypeError + pass + def expr_context_name(ctx): if not 1 <= ctx <= len(ast.expr_context_to_class): @@ -111,8 +116,8 @@ for w_item in space.unpackiterable(w_obj): validate_constant(space, w_item) return - raise ValidationError("got an invalid type in Constant: %s" % - space.type(w_obj).name) + raise ValidationTypeError("got an invalid type in Constant: %s" % + space.type(w_obj).name) class AstValidator(ast.ASTVisitor): diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py --- a/pypy/interpreter/pycompiler.py +++ b/pypy/interpreter/pycompiler.py @@ -138,6 +138,9 @@ def validate_ast(self, node): try: validate.validate_ast(self.space, node) + except validate.ValidationTypeError as e: + raise OperationError(self.space.w_TypeError, + self.space.newtext(e.message)) except validate.ValidationError as e: raise OperationError(self.space.w_ValueError, self.space.newtext(e.message)) From pypy.commits at gmail.com Sun Mar 4 13:49:23 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 04 Mar 2018 10:49:23 -0800 (PST) Subject: [pypy-commit] pypy rgil-cffi: Use cffi instead of ctypes for the untranslated versions of rgil functions Message-ID: <5a9c3fb3.15741c0a.ff2dd.c92c@mx.google.com> Author: Ronan Lamy Branch: rgil-cffi Changeset: r93950:ecf414f04231 Date: 2018-03-04 18:48 +0000 http://bitbucket.org/pypy/pypy/changeset/ecf414f04231/ Log: Use cffi instead of ctypes for the untranslated versions of rgil functions diff --git a/rpython/rlib/rgil.py b/rpython/rlib/rgil.py --- a/rpython/rlib/rgil.py +++ b/rpython/rlib/rgil.py @@ -17,26 +17,47 @@ llexternal = rffi.llexternal +from cffi import FFI +ffi = FFI() + +cdef = """ + void RPyGilAllocate(void); + long RPyGilYieldThread(void); + void RPyGilRelease(void); + void RPyGilAcquire(void); + long *RPyFetchFastGil(void); +""" +ffi.cdef(cdef) +lib = ffi.verify(""" + #include "src/thread.h" + #define RPY_WITH_GIL + """, + sources=[str(translator_c_dir/ 'src' / 'thread.c')], + include_dirs=[cdir]) + + _gil_allocate = llexternal('RPyGilAllocate', [], lltype.Void, _nowrapper=True, sandboxsafe=True, - compilation_info=eci) + compilation_info=eci, _callable=lib.RPyGilAllocate) _gil_yield_thread = llexternal('RPyGilYieldThread', [], lltype.Signed, _nowrapper=True, sandboxsafe=True, - compilation_info=eci) + compilation_info=eci, + _callable=lib.RPyGilYieldThread) -_gil_release = llexternal('RPyGilRelease', [], lltype.Void, - _nowrapper=True, sandboxsafe=True, - compilation_info=eci) +_gil_release = llexternal('RPyGilRelease', [], lltype.Void, + _nowrapper=True, sandboxsafe=True, + compilation_info=eci, _callable=lib.RPyGilRelease) -_gil_acquire = llexternal('RPyGilAcquire', [], lltype.Void, - _nowrapper=True, sandboxsafe=True, - compilation_info=eci) +_gil_acquire = llexternal('RPyGilAcquire', [], lltype.Void, + _nowrapper=True, sandboxsafe=True, + compilation_info=eci, _callable=lib.RPyGilAcquire) gil_fetch_fastgil = llexternal('RPyFetchFastGil', [], llmemory.Address, _nowrapper=True, sandboxsafe=True, - compilation_info=eci) + compilation_info=eci, + _callable=lib.RPyFetchFastGil) # ____________________________________________________________ From pypy.commits at gmail.com Sun Mar 4 15:54:15 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Mar 2018 12:54:15 -0800 (PST) Subject: [pypy-commit] pypy default: use externals directory for 3rd party packages, setup via get_externals.py Message-ID: <5a9c5cf7.85cf1c0a.d6b68.c2b0@mx.google.com> Author: Matti Picus Branch: Changeset: r93951:e8813036ddc6 Date: 2018-03-04 12:32 -0800 http://bitbucket.org/pypy/pypy/changeset/e8813036ddc6/ Log: use externals directory for 3rd party packages, setup via get_externals.py diff --git a/pypy/tool/build_cffi_imports.py b/pypy/tool/build_cffi_imports.py --- a/pypy/tool/build_cffi_imports.py +++ b/pypy/tool/build_cffi_imports.py @@ -29,6 +29,11 @@ if status != 0: status, stdout, stderr = run_subprocess(str(pypy_c), ['-m', 'ensurepip']) failures = [] + env = os.environ.copy() + if sys.platform == 'win32': + env['INCLUDE'] = r'..\externals\include;' + env.get('INCLUDE', '') + env['LIB'] = r'..\externals\lib;' + env.get('LIB', '') + env['PATH'] = r'..\externals\bin;' + env.get('PATH', '') for key, module in sorted(cffi_build_scripts.items()): if module is None or getattr(options, 'no_' + key, False): continue @@ -40,7 +45,8 @@ cwd = None print('*', ' '.join(args), file=sys.stderr) try: - status, stdout, stderr = run_subprocess(str(pypy_c), args, cwd=cwd) + status, stdout, stderr = run_subprocess(str(pypy_c), args, + cwd=cwd, env=env) if status != 0: print(stdout, stderr, file=sys.stderr) failures.append((key, module)) From pypy.commits at gmail.com Sun Mar 4 16:26:01 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Mar 2018 13:26:01 -0800 (PST) Subject: [pypy-commit] pypy default: test, fix for os.move, os.replace mixing unicode and ascii file names on win32 Message-ID: <5a9c6469.03b01c0a.97c4d.b7be@mx.google.com> Author: Matti Picus Branch: Changeset: r93952:8b47f49b6df4 Date: 2018-03-04 13:25 -0800 http://bitbucket.org/pypy/pypy/changeset/8b47f49b6df4/ Log: test, fix for os.move, os.replace mixing unicode and ascii file names on win32 diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1175,6 +1175,19 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + def test_rename(self): + os = self.posix + with open(self.path, "w") as f: + f.write("this is a rename test") + unicode_name = str(self.udir) + u'/test\u03be.txt' + os.rename(self.path, unicode_name) + with open(unicode_name) as f: + assert f.read() == 'this is a rename test' + os.rename(unicode_name, self.path) + with open(self.path) as f: + assert f.read() == 'this is a rename test' + + class AppTestEnvironment(object): def setup_class(cls): diff --git a/rpython/rlib/_os_support.py b/rpython/rlib/_os_support.py --- a/rpython/rlib/_os_support.py +++ b/rpython/rlib/_os_support.py @@ -99,6 +99,13 @@ return unicode_traits else: return string_traits + + @specialize.argtype(0, 1) + def _preferred_traits2(path1, path2): + if _prefer_unicode(path1) or _prefer_unicode(path2): + return unicode_traits + else: + return string_traits else: @specialize.argtype(0) def _prefer_unicode(path): diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -8,7 +8,7 @@ from rpython.rlib import debug, jit, rstring, rthread, types from rpython.rlib._os_support import ( _CYGWIN, _MACRO_ON_POSIX, UNDERSCORE_ON_WIN32, _WIN32, - _prefer_unicode, _preferred_traits) + _prefer_unicode, _preferred_traits, _preferred_traits2) from rpython.rlib.objectmodel import ( specialize, enforceargs, register_replacement_for, NOT_CONSTANT) from rpython.rlib.rarithmetic import intmask, widen @@ -1257,7 +1257,7 @@ handle_posix_error('rename', c_rename(_as_bytes0(path1), _as_bytes0(path2))) else: - traits = _preferred_traits(path1) + traits = _preferred_traits2(path1, path2) win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) @@ -1267,7 +1267,7 @@ @specialize.argtype(0, 1) def replace(path1, path2): if _WIN32: - traits = _preferred_traits(path1) + traits = _preferred_traits2(path1, path2) win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) From pypy.commits at gmail.com Sun Mar 4 18:57:06 2018 From: pypy.commits at gmail.com (mjacob) Date: Sun, 04 Mar 2018 15:57:06 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: Add revdb improvements to the list of Leysin 2018 sprint tasks. Message-ID: <5a9c87d2.81b5df0a.f3ce.ece9@mx.google.com> Author: Manuel Jacob Branch: extradoc Changeset: r5876:70a8d16b4a7d Date: 2018-03-05 00:57 +0100 http://bitbucket.org/pypy/extradoc/changeset/70a8d16b4a7d/ Log: Add revdb improvements to the list of Leysin 2018 sprint tasks. diff --git a/planning/sprint-leysin-2018-notes.rst b/planning/sprint-leysin-2018-notes.rst --- a/planning/sprint-leysin-2018-notes.rst +++ b/planning/sprint-leysin-2018-notes.rst @@ -14,3 +14,4 @@ - update www.pypy.org, speed.pypy.org (web devs needed) - go over projects at https://bitbucket.org/pypy, delete or document dead projects - document the different rpython decorators like enforceargs, signature, and then interp-level unwrap_spec +- revdb: bring it up to date, improve usability, other improvements From pypy.commits at gmail.com Sun Mar 4 23:14:12 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 04 Mar 2018 20:14:12 -0800 (PST) Subject: [pypy-commit] pypy default: support non-win32, unbreak the world Message-ID: <5a9cc414.cf061c0a.97435.464e@mx.google.com> Author: Matti Picus Branch: Changeset: r93953:3033dc0b120a Date: 2018-03-04 20:13 -0800 http://bitbucket.org/pypy/pypy/changeset/3033dc0b120a/ Log: support non-win32, unbreak the world diff --git a/rpython/rlib/_os_support.py b/rpython/rlib/_os_support.py --- a/rpython/rlib/_os_support.py +++ b/rpython/rlib/_os_support.py @@ -114,3 +114,7 @@ @specialize.argtype(0) def _preferred_traits(path): return string_traits + + @specialize.argtype(0, 1) + def _preferred_traits2(path1, path2): + return string_traits From pypy.commits at gmail.com Mon Mar 5 02:53:07 2018 From: pypy.commits at gmail.com (Raemi) Date: Sun, 04 Mar 2018 23:53:07 -0800 (PST) Subject: [pypy-commit] extradoc extradoc: confirm my arrival date Message-ID: <5a9cf763.46a4df0a.23d10.6ac8@mx.google.com> Author: Remi Meier Branch: extradoc Changeset: r5877:bae0c54dd461 Date: 2018-03-05 08:52 +0100 http://bitbucket.org/pypy/extradoc/changeset/bae0c54dd461/ Log: confirm my arrival date diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -12,7 +12,7 @@ Armin Rigo private Matti Picus 17.3/22.3 Ermina Manuel Jacob 17.3/24.3 Ermina -Remi Meier 1[78].3/24.3 Ermina +Remi Meier 18.3/24.3 Ermina Joannah Nanjekye 17.3/25.3 separate room Ermina Jean-Daniel 17.3/21.3 own booking Ermina Arianna 17.3/21.3 own booking Ermina From pypy.commits at gmail.com Mon Mar 5 03:52:38 2018 From: pypy.commits at gmail.com (amauryfa) Date: Mon, 05 Mar 2018 00:52:38 -0800 (PST) Subject: [pypy-commit] pypy py3.6: More checks for forbidden 'await' Message-ID: <5a9d0556.17361c0a.62746.a3e4@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93956:e8e5e733ff62 Date: 2018-03-05 00:19 +0100 http://bitbucket.org/pypy/pypy/changeset/e8e5e733ff62/ Log: More checks for forbidden 'await' diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -283,6 +283,10 @@ def _get_code_flags(self): return 0 + def _check_async_function(self): + """Returns true if 'await' is allowed.""" + return False + def _handle_body(self, body): """Compile a list of statements, handling doc strings if needed.""" if body: @@ -1087,6 +1091,8 @@ self.emit_op(ops.YIELD_FROM) def visit_Await(self, aw): + if not self._check_async_function(): + self.error("'await' outside async function", aw) self.update_position(aw.lineno) aw.value.walkabout(self) self.emit_op(ops.GET_AWAITABLE) @@ -1752,6 +1758,9 @@ for i in range(start, len(func.body)): func.body[i].walkabout(self) + def _check_async_function(self): + return True + class LambdaCodeGenerator(AbstractFunctionCodeGenerator): def _compile(self, lam): @@ -1780,6 +1789,9 @@ def _end_comp(self): self.emit_op(ops.RETURN_VALUE) + def _check_async_function(self): + return True + class GenExpCodeGenerator(ComprehensionCodeGenerator): diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -1163,6 +1163,14 @@ """ self.simple_test(source, "None", None) + def test_await_in_nested(self): + source = """if 1: + async def foo(): + def bar(): + [i for i in await items] + """ + e = py.test.raises(SyntaxError, self.simple_test, source, "None", None) + def test_load_classderef(self): source = """if 1: def f(): From pypy.commits at gmail.com Mon Mar 5 03:52:34 2018 From: pypy.commits at gmail.com (amauryfa) Date: Mon, 05 Mar 2018 00:52:34 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Legacy __aiter__ protocol now emits a DeprecationWarning. Message-ID: <5a9d0552.d2d0df0a.10931.9871@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93954:256b3eca58cb Date: 2018-03-04 10:05 +0100 http://bitbucket.org/pypy/pypy/changeset/256b3eca58cb/ Log: Legacy __aiter__ protocol now emits a DeprecationWarning. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1609,7 +1609,7 @@ "__aiter__ should return an asynchronous " "iterator, not awaitable" % space.type(w_obj).name), - space.w_PendingDeprecationWarning) + space.w_DeprecationWarning) self.pushvalue(w_awaitable) def GET_ANEXT(self, oparg, next_instr): From pypy.commits at gmail.com Mon Mar 5 03:52:40 2018 From: pypy.commits at gmail.com (amauryfa) Date: Mon, 05 Mar 2018 00:52:40 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Using 'await' in other contexts raises a DeprecationWarning Message-ID: <5a9d0558.17361c0a.62746.a3e7@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93957:3da474cc3b78 Date: 2018-03-05 09:04 +0100 http://bitbucket.org/pypy/pypy/changeset/3da474cc3b78/ Log: Using 'await' in other contexts raises a DeprecationWarning diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -125,7 +125,7 @@ def check_forbidden_name(self, name, node): try: - misc.check_forbidden_name(name) + misc.check_forbidden_name(self.space, name) except misc.ForbiddenNameAssignment as e: self.error("cannot assign to %s" % (e.name,), node) @@ -135,7 +135,7 @@ def set_context(self, expr, ctx): """Set the context of an expression to Store or Del if possible.""" try: - expr.set_context(ctx) + expr.set_context(self.space, ctx) except ast.UnacceptableExpressionContext as e: self.error_ast(e.msg, e.node) except misc.ForbiddenNameAssignment as e: diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py --- a/pypy/interpreter/astcompiler/asthelpers.py +++ b/pypy/interpreter/astcompiler/asthelpers.py @@ -15,7 +15,7 @@ def as_node_list(self, space): raise AssertionError("only for expressions") - def set_context(self, ctx): + def set_context(self, space, ctx): raise AssertionError("should only be on expressions") @@ -27,7 +27,7 @@ def as_node_list(self, space): return None - def set_context(self, ctx): + def set_context(self, space, ctx): d = self._description if d is None: d = "%r" % (self,) @@ -43,32 +43,32 @@ def as_node_list(self, space): return self.elts - def set_context(self, ctx): + def set_context(self, space, ctx): if self.elts: for elt in self.elts: - elt.set_context(ctx) + elt.set_context(space, ctx) self.ctx = ctx class __extend__(ast.Attribute): - def set_context(self, ctx): + def set_context(self, space, ctx): if ctx == ast.Store: - misc.check_forbidden_name(self.attr, self) + misc.check_forbidden_name(space, self.attr, self) self.ctx = ctx class __extend__(ast.Subscript): - def set_context(self, ctx): + def set_context(self, space, ctx): self.ctx = ctx class __extend__(ast.Name): - def set_context(self, ctx): + def set_context(self, space, ctx): if ctx == ast.Store: - misc.check_forbidden_name(self.id, self) + misc.check_forbidden_name(space, self.id, self) self.ctx = ctx @@ -79,14 +79,14 @@ def as_node_list(self, space): return self.elts - def set_context(self, ctx): + def set_context(self, space, ctx): if self.elts: for elt in self.elts: - elt.set_context(ctx) + elt.set_context(space, ctx) self.ctx = ctx else: # Assignment to () raises an error. - ast.expr.set_context(self, ctx) + ast.expr.set_context(self, space, ctx) class __extend__(ast.Lambda): @@ -141,9 +141,9 @@ _description = "starred expression" - def set_context(self, ctx): + def set_context(self, space, ctx): self.ctx = ctx - self.value.set_context(ctx) + self.value.set_context(space, ctx) class __extend__(ast.IfExp): diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py --- a/pypy/interpreter/astcompiler/misc.py +++ b/pypy/interpreter/astcompiler/misc.py @@ -68,10 +68,14 @@ self.node = node -def check_forbidden_name(name, node=None): +def check_forbidden_name(space, name, node=None): """Raise an error if the name cannot be assigned to.""" if name in ("None", "__debug__"): raise ForbiddenNameAssignment(name, node) + if name in ("async", "await"): + space.warn(space.newtext( + "'async' and 'await' will become reserved keywords" + " in Python 3.7"), space.w_DeprecationWarning) # XXX Warn about using True and False diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -1300,6 +1300,14 @@ # compiling the produced AST previously triggered a crash compile(ast, '', 'exec') + def test_await_warning(self): + import warnings + source = "def f(): await = 5" + with warnings.catch_warnings(record=True) as l: + warnings.simplefilter("always") + compile(source, '', 'exec') + assert isinstance(l[0].message, DeprecationWarning) + class TestOptimizations: def count_instructions(self, source): From pypy.commits at gmail.com Mon Mar 5 03:52:36 2018 From: pypy.commits at gmail.com (amauryfa) Date: Mon, 05 Mar 2018 00:52:36 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Move coroutine tests to test_coroutine.py Message-ID: <5a9d0554.51d81c0a.bba85.1bfe@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93955:df41ee03b376 Date: 2018-03-04 14:52 +0100 http://bitbucket.org/pypy/pypy/changeset/df41ee03b376/ Log: Move coroutine tests to test_coroutine.py diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -1466,55 +1466,3 @@ return f'ab{x}cd' """ code, blocks = generate_function_code(source, self.space) - -class AppTestCoroutine: - def w_run_async(self, coro): - buffer = [] - result = None - while True: - try: - buffer.append(coro.send(None)) - except StopIteration as ex: - result = ex.args[0] if ex.args else None - break - return buffer, result - - def test_async_generator(self): - """ - async def f(i): - return i - - async def run_list(): - return [await c for c in [f(1), f(41)]] - - assert self.run_async(run_list()) == ([], [1, 41]) - """ - - def test_async_genexpr(self): - """ - async def f(it): - for i in it: - yield i - - async def run_gen(): - gen = (i + 1 async for i in f([10, 20])) - return [g + 100 async for g in gen] - - assert self.run_async(run_gen()) == ([], [111, 121]) - """ - - def test_anext_tuple(self): - """ - async def foo(): - try: - yield (1,) - except ZeroDivisionError: - yield (2,) - - async def run(): - it = foo().__aiter__() - return await it.__anext__() - - assert self.run_async(run()) == ([], (1,)) - """ - diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py --- a/pypy/interpreter/test/test_coroutine.py +++ b/pypy/interpreter/test/test_coroutine.py @@ -499,3 +499,53 @@ else: assert False, "didn't raise" """ + + def w_run_async(self, coro): + buffer = [] + result = None + while True: + try: + buffer.append(coro.send(None)) + except StopIteration as ex: + result = ex.args[0] if ex.args else None + break + return buffer, result + + def test_async_generator(self): + """ + async def f(i): + return i + + async def run_list(): + return [await c for c in [f(1), f(41)]] + + assert self.run_async(run_list()) == ([], [1, 41]) + """ + + def test_async_genexpr(self): + """ + async def f(it): + for i in it: + yield i + + async def run_gen(): + gen = (i + 1 async for i in f([10, 20])) + return [g + 100 async for g in gen] + + assert self.run_async(run_gen()) == ([], [111, 121]) + """ + + def test_anext_tuple(self): + """ + async def foo(): + try: + yield (1,) + except ZeroDivisionError: + yield (2,) + + async def run(): + it = foo().__aiter__() + return await it.__anext__() + + assert self.run_async(run()) == ([], (1,)) + """ From pypy.commits at gmail.com Mon Mar 5 03:52:42 2018 From: pypy.commits at gmail.com (amauryfa) Date: Mon, 05 Mar 2018 00:52:42 -0800 (PST) Subject: [pypy-commit] pypy py3.6: Remember the __cause__ when an exception is raised in __await__. Message-ID: <5a9d055a.6980df0a.a70a7.d141@mx.google.com> Author: Amaury Forgeot d'Arc Branch: py3.6 Changeset: r93958:ab182b33a9c1 Date: 2018-03-05 09:26 +0100 http://bitbucket.org/pypy/pypy/changeset/ab182b33a9c1/ Log: Remember the __cause__ when an exception is raised in __await__. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1596,14 +1596,15 @@ try: w_awaitable = get_awaitable_iter(space, w_iter) except OperationError as e: - # yay! get_awaitable_iter() carefully builds a useful - # error message, but here we're eating *all errors* - # to replace it with a generic one. if e.async(space): raise - raise oefmt(space.w_TypeError, + new_error = oefmt(space.w_TypeError, "'async for' received an invalid object " "from __aiter__: %T", w_iter) + e.normalize_exception(space) + new_error.normalize_exception(space) + new_error.set_cause(space, e.get_w_value(space)) + raise new_error space.warn(space.newtext( "'%s' implements legacy __aiter__ protocol; " "__aiter__ should return an asynchronous " @@ -1628,14 +1629,15 @@ try: w_awaitable = get_awaitable_iter(space, w_next_iter) except OperationError as e: - # yay! get_awaitable_iter() carefully builds a useful - # error message, but here we're eating *all errors* - # to replace it with a generic one. if e.async(space): raise - raise oefmt(space.w_TypeError, + new_error = oefmt(space.w_TypeError, "'async for' received an invalid object " "from __anext__: %T", w_next_iter) + e.normalize_exception(space) + new_error.normalize_exception(space) + new_error.set_cause(space, e.get_w_value(space)) + raise new_error self.pushvalue(w_awaitable) def FORMAT_VALUE(self, oparg, next_instr): diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py --- a/pypy/interpreter/test/test_coroutine.py +++ b/pypy/interpreter/test/test_coroutine.py @@ -75,6 +75,24 @@ assert next(cr.__await__()) == 20 """ + def test_for_error_cause(self): """ + class F: + def __aiter__(self): + return self + def __anext__(self): + return self + def __await__(self): + 1 / 0 + + async def main(): + async for _ in F(): + pass + + c = raises(TypeError, main().send, None) + assert 'an invalid object from __anext__' in c.value.args[0], c.value + assert isinstance(c.value.__cause__, ZeroDivisionError) + """ + def test_set_coroutine_wrapper(self): """ import sys async def f(): From pypy.commits at gmail.com Mon Mar 5 17:02:15 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 05 Mar 2018 14:02:15 -0800 (PST) Subject: [pypy-commit] pypy default: reset ro to rw and unlink temp files Message-ID: <5a9dbe67.1b88df0a.4bda7.cdf9@mx.google.com> Author: Matti Picus Branch: Changeset: r93959:d9e0d0815c04 Date: 2018-03-06 00:01 +0200 http://bitbucket.org/pypy/pypy/changeset/d9e0d0815c04/ Log: reset ro to rw and unlink temp files diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -971,9 +971,12 @@ if sys.platform == 'win32': os.chmod(self.path, 0400) assert (os.stat(self.path).st_mode & 0600) == 0400 + os.chmod(self.path, 0700) else: os.chmod(self.path, 0200) assert (os.stat(self.path).st_mode & 0777) == 0200 + os.chmod(self.path, 0700) + os.unlink(self.path) if hasattr(os, 'fchmod'): def test_fchmod(self): @@ -983,6 +986,7 @@ assert (os.fstat(f.fileno()).st_mode & 0777) == 0200 f.close() assert (os.stat(self.path).st_mode & 0777) == 0200 + os.unlink(self.path) if hasattr(os, 'mkfifo'): def test_mkfifo(self): @@ -1186,6 +1190,7 @@ os.rename(unicode_name, self.path) with open(self.path) as f: assert f.read() == 'this is a rename test' + os.unlink(self.path) From pypy.commits at gmail.com Wed Mar 7 10:27:54 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 07 Mar 2018 07:27:54 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: Backed out changeset 89f271b2dd38 Message-ID: <5aa004fa.01da1c0a.97901.390a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93960:1d03f9c2353f Date: 2018-03-07 13:34 +0100 http://bitbucket.org/pypy/pypy/changeset/1d03f9c2353f/ Log: Backed out changeset 89f271b2dd38 back out this change for now. it leads to weird crashes and isn't the main point of this branch anyway. diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -33,6 +33,13 @@ # maybe should be delegated to the optimization classes? +def tag_box(box, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import Const + if isinstance(box, Const): + return memo.getconst(box) + else: + return liveboxes_from_env[box] # has to exist + def decode_box(resumestorage, tagged, liveboxes, cpu): from rpython.jit.metainterp.resume import untag, TAGCONST, TAGINT, TAGBOX from rpython.jit.metainterp.resume import NULLREF, TAG_CONST_OFFSET, tagged_eq @@ -53,12 +60,11 @@ raise AssertionError("unreachable") return box -def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, adder): +def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): from rpython.jit.metainterp.history import ConstInt - available_boxes = {} for box in liveboxes: - if box is not None: + if box is not None and box in liveboxes_from_env: available_boxes[box] = None # class knowledge is stored as bits, true meaning the class is known, false @@ -92,16 +98,16 @@ numb_state.append_int(len(triples_struct)) for box1, descr, box2 in triples_struct: descr_index = descr.descr_index - numb_state.append_short(adder._gettagged(box1)) + numb_state.append_short(tag_box(box1, liveboxes_from_env, memo)) numb_state.append_int(descr_index) - numb_state.append_short(adder._gettagged(box2)) + numb_state.append_short(tag_box(box2, liveboxes_from_env, memo)) numb_state.append_int(len(triples_array)) for box1, index, descr, box2 in triples_array: descr_index = descr.descr_index - numb_state.append_short(adder._gettagged(box1)) + numb_state.append_short(tag_box(box1, liveboxes_from_env, memo)) numb_state.append_int(index) numb_state.append_int(descr_index) - numb_state.append_short(adder._gettagged(box2)) + numb_state.append_short(tag_box(box2, liveboxes_from_env, memo)) else: numb_state.append_int(0) numb_state.append_int(0) @@ -112,8 +118,8 @@ numb_state.append_int(len(tuples_loopinvariant)) for constarg0, box in tuples_loopinvariant: numb_state.append_short( - adder._gettagged(ConstInt(constarg0))) - numb_state.append_short(adder._gettagged(box)) + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) else: numb_state.append_int(0) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -464,7 +464,7 @@ numb_state.patch(1, len(liveboxes)) - self._add_optimizer_sections(numb_state, liveboxes) + self._add_optimizer_sections(numb_state, liveboxes, liveboxes_from_env) storage.rd_numb = numb_state.create_numbering() storage.rd_consts = self.memo.consts return liveboxes[:] @@ -584,11 +584,11 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] - def _add_optimizer_sections(self, numb_state, liveboxes): + def _add_optimizer_sections(self, numb_state, liveboxes, liveboxes_from_env): # add extra information about things the optimizer learned from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge serialize_optimizer_knowledge( - self.optimizer, numb_state, liveboxes, self) + self.optimizer, numb_state, liveboxes, liveboxes_from_env, self.memo) class AbstractVirtualInfo(object): kind = REF diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -60,7 +60,7 @@ numb_state.append_int(1) # size of resume block liveboxes = [InputArgInt(), box2, box1, box3] - serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, None) + serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) assert unpack_numbering(numb_state.create_numbering()) == [ 1, 0b010000, 0, 0, 0] @@ -100,7 +100,7 @@ numb_state.append_int(1) # size of resume block liveboxes = [box for (box, _) in boxes_known_classes] - serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, None) + serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) @@ -221,31 +221,6 @@ self.check_resops(getfield_gc_i=4) # 3x a.x, 1x a.n self.check_resops(getfield_gc_r=1) # in main loop - def test_bridge_field_read_virtual(self): - myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n', 'a']) - class A(object): - pass - class Virt(object): - def __init__(self, n1): - self.n1 = n1 - - def f(y, n): - a = A() - a.n = n - res = 0 - while y > 0: - myjitdriver.jit_merge_point(y=y, n=n, res=res, a=a) - v = Virt(a.n) - if y > n: - res += 1 - res += v.n1 + a.n - y -= 1 - return res - res = self.meta_interp(f, [32, 16]) - assert res == f(32, 16) - self.check_trace_count(3) - self.check_resops(getfield_gc_i=1) # 1x a.x - def test_bridge_field_read_constants(self): myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) class A(object): From pypy.commits at gmail.com Wed Mar 7 10:27:56 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 07 Mar 2018 07:27:56 -0800 (PST) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: document branch Message-ID: <5aa004fc.b4b0df0a.29449.e8ab@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93961:68fc67129db2 Date: 2018-03-07 13:36 +0100 http://bitbucket.org/pypy/pypy/changeset/68fc67129db2/ Log: document branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,9 @@ .. branch: refactor-slots Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. From pypy.commits at gmail.com Wed Mar 7 22:23:26 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 07 Mar 2018 19:23:26 -0800 (PST) Subject: [pypy-commit] pypy default: update test file to match upstream Message-ID: <5aa0acae.c8ce1c0a.e7297.4fb6@mx.google.com> Author: Matti Picus Branch: Changeset: r93962:6a54c39fe8ab Date: 2018-03-08 05:22 +0200 http://bitbucket.org/pypy/pypy/changeset/6a54c39fe8ab/ Log: update test file to match upstream diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -398,6 +398,7 @@ # x.close_lib() py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') + x.close_lib() def test_no_len_on_nonarray(): p = new_primitive_type("int") From pypy.commits at gmail.com Thu Mar 8 13:56:24 2018 From: pypy.commits at gmail.com (mattip) Date: Thu, 08 Mar 2018 10:56:24 -0800 (PST) Subject: [pypy-commit] pypy.org extradoc: update homebrew link Message-ID: <5aa18758.1b88df0a.4bda7.1723@mx.google.com> Author: Matti Picus Branch: extradoc Changeset: r916:949018b0e502 Date: 2018-03-08 20:55 +0200 http://bitbucket.org/pypy/pypy.org/changeset/949018b0e502/ Log: update homebrew link diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -104,7 +104,7 @@
  • use Squeaky's portable Linux binaries.
  • or download PyPy from your release vendor (usually an outdated -version): Ubuntu (PPA), Debian, Homebrew, MacPorts, +version): Ubuntu (PPA), Debian, Homebrew, MacPorts, Fedora, Gentoo and Arch are known to package PyPy, with various degrees of being up-to-date.
  • or translate your own PyPy.
    • diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -72,7 +72,7 @@ .. _`Debian`: http://packages.debian.org/sid/pypy .. _`Fedora`: http://fedoraproject.org/wiki/Features/PyPyStack .. _`Gentoo`: http://packages.gentoo.org/package/dev-python/pypy -.. _`Homebrew`: https://github.com/mxcl/homebrew/blob/master/Library/Formula/pypy.rb +.. _`Homebrew`: https://github.com/Homebrew/homebrew-core/blob/master/Formula/pypy.rb .. _`Arch`: https://wiki.archlinux.org/index.php/PyPy .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux From pypy.commits at gmail.com Fri Mar 9 03:00:35 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 09 Mar 2018 00:00:35 -0800 (PST) Subject: [pypy-commit] pypy default: merge heads Message-ID: <5aa23f23.c83f1c0a.3390c.342b@mx.google.com> Author: Armin Rigo Branch: Changeset: r93964:42f9add4e266 Date: 2018-03-09 08:59 +0100 http://bitbucket.org/pypy/pypy/changeset/42f9add4e266/ Log: merge heads From pypy.commits at gmail.com Fri Mar 9 03:00:33 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 09 Mar 2018 00:00:33 -0800 (PST) Subject: [pypy-commit] pypy default: update to cffi/99940f1f5402 Message-ID: <5aa23f21.f2b8df0a.37528.3086@mx.google.com> Author: Armin Rigo Branch: Changeset: r93963:93c05c65dcaa Date: 2018-03-09 08:54 +0100 http://bitbucket.org/pypy/pypy/changeset/93c05c65dcaa/ Log: update to cffi/99940f1f5402 diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -53,8 +53,7 @@ self.libhandle = rffi.cast(DLLHANDLE, 0) if not libhandle: - raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed", - self.libname) + return self.may_unregister_rpython_finalizer(self.ffi.space) # Clear the dict to force further accesses to do cdlopen_fetch() diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -90,7 +90,6 @@ w_ctype.convert_from_object(rffi.cast(rffi.CCHARP, cdata), w_value) def close_lib(self): - self.check_closed() self._finalize_() diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -398,6 +398,7 @@ # x.close_lib() py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') + x.close_lib() def test_no_len_on_nonarray(): p = new_primitive_type("int") diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -114,12 +114,10 @@ from re_python_pysrc import ffi lib = ffi.dlopen(self.extmod) ffi.dlclose(lib) - e = raises(ffi.error, ffi.dlclose, lib) - assert str(e.value) == ( - "library '%s' is already closed" % (self.extmod,)) e = raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (self.extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(self): self.fix_path() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -508,9 +508,6 @@ ffi.cdef("int foobar(void); int foobaz;") lib = ffi.dlopen(lib_m) ffi.dlclose(lib) - e = py.test.raises(ValueError, ffi.dlclose, lib) - assert str(e.value).startswith("library '") - assert str(e.value).endswith("' has already been closed") e = py.test.raises(ValueError, getattr, lib, 'foobar') assert str(e.value).startswith("library '") assert str(e.value).endswith("' has already been closed") @@ -520,3 +517,4 @@ e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) assert str(e.value).startswith("library '") assert str(e.value).endswith("' has already been closed") + ffi.dlclose(lib) # does not raise diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py @@ -120,12 +120,10 @@ str_extmod = extmod.encode('utf-8') else: str_extmod = extmod - e = py.test.raises(ffi.error, ffi.dlclose, lib) - assert str(e.value).startswith( - "library '%s' is already closed" % (str_extmod,)) e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (str_extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(): from re_python_pysrc import ffi From pypy.commits at gmail.com Fri Mar 9 07:59:37 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Mar 2018 04:59:37 -0800 (PST) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5aa28539.4b981c0a.ebce1.5d58@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93966:66f30ba67227 Date: 2018-03-09 14:58 +0200 http://bitbucket.org/pypy/pypy/changeset/66f30ba67227/ Log: merge default into py3.5 diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -3,7 +3,7 @@ import importlib.machinery -def get_hashed_dir(cfile): +def _get_hashed_filename(cfile): with open(cfile,'r') as fid: content = fid.read() # from cffi's Verifier() @@ -23,10 +23,28 @@ username = os.environ['USERNAME'] #windows except KeyError: username = os.getuid() - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % ( username, k1, k2) - if not os.path.exists(output_dir): + +def get_hashed_dir(cfile): + hashed_fn = _get_hashed_filename(cfile) + try: + with open(hashed_fn) as f: + dirname = f.read(1024) + except IOError: + dirname = '' + tmpdir = tempfile.gettempdir() + if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname + or not os.path.isdir(os.path.join(tmpdir, dirname))): + dirname = binascii.hexlify(os.urandom(8)) + if not isinstance(dirname, str): # Python 3 + dirname = dirname.decode('ascii') + dirname = 'testcapi_' + dirname + output_dir = os.path.join(tmpdir, dirname) + try: os.mkdir(output_dir) + except OSError: + pass return output_dir @@ -35,13 +53,12 @@ return suffixes[0] if suffixes else None -def compile_shared(csource, modulename, output_dir=None): +def compile_shared(csource, modulename, output_dir): """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module, and import it. """ thisdir = os.path.dirname(__file__) - if output_dir is None: - output_dir = tempfile.mkdtemp() + assert output_dir is not None from distutils.ccompiler import new_compiler @@ -85,4 +102,16 @@ # Now import the newly created library, it will replace the original # module in sys.modules fp, filename, description = imp.find_module(modulename, path=[output_dir]) - imp.load_module(modulename, fp, filename, description) + with fp: + imp.load_module(modulename, fp, filename, description) + + # If everything went fine up to now, write the name of this new + # directory to 'hashed_fn', for future processes (and to avoid a + # growing number of temporary directories that are not completely + # obvious to clean up on Windows) + hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource)) + try: + with open(hashed_fn, 'w') as f: + f.write(os.path.basename(output_dir)) + except IOError: + pass diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -53,8 +53,7 @@ self.libhandle = rffi.cast(DLLHANDLE, 0) if not libhandle: - raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed", - self.libname) + return self.may_unregister_rpython_finalizer(self.ffi.space) # Clear the dict to force further accesses to do cdlopen_fetch() diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -90,7 +90,6 @@ w_ctype.convert_from_object(rffi.cast(rffi.CCHARP, cdata), w_value) def close_lib(self): - self.check_closed() self._finalize_() diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -398,6 +398,7 @@ # x.close_lib() py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') + x.close_lib() def test_no_len_on_nonarray(): p = new_primitive_type("int") diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -114,12 +114,10 @@ from re_python_pysrc import ffi lib = ffi.dlopen(self.extmod) ffi.dlclose(lib) - e = raises(ffi.error, ffi.dlclose, lib) - assert str(e.value) == ( - "library '%s' is already closed" % (self.extmod,)) e = raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (self.extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(self): self.fix_path() diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1087,9 +1087,12 @@ if sys.platform == 'win32': os.chmod(my_path, 0o400) assert (os.stat(my_path).st_mode & 0o600) == 0o400 + os.chmod(self.path, 0700) else: os.chmod(my_path, 0o200) assert (os.stat(my_path).st_mode & 0o777) == 0o200 + os.chmod(self.path, 0700) + os.unlink(self.path) if hasattr(os, 'fchmod'): def test_fchmod(self): @@ -1100,6 +1103,7 @@ assert (os.fstat(f.fileno()).st_mode & 0o777) == 0o200 f.close() assert (os.stat(my_path).st_mode & 0o777) == 0o200 + os.unlink(self.path) if hasattr(os, 'mkfifo'): def test_mkfifo(self): @@ -1401,6 +1405,20 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + def test_rename(self): + os = self.posix + with open(self.path, "w") as f: + f.write("this is a rename test") + unicode_name = str(self.udir) + u'/test\u03be.txt' + os.rename(self.path, unicode_name) + with open(unicode_name) as f: + assert f.read() == 'this is a rename test' + os.rename(unicode_name, self.path) + with open(self.path) as f: + assert f.read() == 'this is a rename test' + os.unlink(self.path) + + def test_device_encoding(self): import sys encoding = self.posix.device_encoding(sys.stdout.fileno()) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -508,9 +508,6 @@ ffi.cdef("int foobar(void); int foobaz;") lib = ffi.dlopen(lib_m) ffi.dlclose(lib) - e = py.test.raises(ValueError, ffi.dlclose, lib) - assert str(e.value).startswith("library '") - assert str(e.value).endswith("' has already been closed") e = py.test.raises(ValueError, getattr, lib, 'foobar') assert str(e.value).startswith("library '") assert str(e.value).endswith("' has already been closed") @@ -520,3 +517,4 @@ e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) assert str(e.value).startswith("library '") assert str(e.value).endswith("' has already been closed") + ffi.dlclose(lib) # does not raise diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py @@ -120,12 +120,10 @@ str_extmod = extmod.encode('utf-8') else: str_extmod = extmod - e = py.test.raises(ffi.error, ffi.dlclose, lib) - assert str(e.value).startswith( - "library '%s' is already closed" % (str_extmod,)) e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (str_extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(): from re_python_pysrc import ffi diff --git a/rpython/rlib/_os_support.py b/rpython/rlib/_os_support.py --- a/rpython/rlib/_os_support.py +++ b/rpython/rlib/_os_support.py @@ -99,6 +99,13 @@ return unicode_traits else: return string_traits + + @specialize.argtype(0, 1) + def _preferred_traits2(path1, path2): + if _prefer_unicode(path1) or _prefer_unicode(path2): + return unicode_traits + else: + return string_traits else: @specialize.argtype(0) def _prefer_unicode(path): @@ -107,3 +114,7 @@ @specialize.argtype(0) def _preferred_traits(path): return string_traits + + @specialize.argtype(0, 1) + def _preferred_traits2(path1, path2): + return string_traits diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -8,7 +8,7 @@ from rpython.rlib import debug, jit, rstring, rthread, types from rpython.rlib._os_support import ( _CYGWIN, _MACRO_ON_POSIX, UNDERSCORE_ON_WIN32, _WIN32, - _prefer_unicode, _preferred_traits) + _prefer_unicode, _preferred_traits, _preferred_traits2) from rpython.rlib.objectmodel import ( specialize, enforceargs, register_replacement_for, NOT_CONSTANT) from rpython.rlib.rarithmetic import intmask, widen @@ -1257,7 +1257,7 @@ handle_posix_error('rename', c_rename(_as_bytes0(path1), _as_bytes0(path2))) else: - traits = _preferred_traits(path1) + traits = _preferred_traits2(path1, path2) win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) @@ -1267,7 +1267,7 @@ @specialize.argtype(0, 1) def replace(path1, path2): if _WIN32: - traits = _preferred_traits(path1) + traits = _preferred_traits2(path1, path2) win32traits = make_win32_traits(traits) path1 = traits.as_str0(path1) path2 = traits.as_str0(path2) diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -171,17 +171,17 @@ def __init__(self, cc=None, x64=False, ver0=None): self.x64 = x64 + patch_os_env(self.externals) + self.c_environ = os.environ.copy() if cc is None: msvc_compiler_environ, self.vsver = find_msvc_env(x64, ver0=ver0) Platform.__init__(self, 'cl.exe') if msvc_compiler_environ: + self.c_environ.update(msvc_compiler_environ) if x64: self.externals_branch = 'win34_%d' % self.vsver else: self.externals_branch = 'win32_%d' % self.vsver - patch_env(msvc_compiler_environ, self.externals) - self.c_environ = os.environ.copy() - self.c_environ.update(msvc_compiler_environ) else: self.cc = cc @@ -543,17 +543,18 @@ # These are the external libraries, created and maintained by get_externals.py # The buildbot runs get_externals before building -def patch_env(env, externals = Platform.externals): +def patch_os_env(externals = Platform.externals): #print 'adding %s to PATH, INCLUDE, LIB' % basepath binpath = externals + r'\bin' path = os.environ['PATH'] - if binpath not in path.split(';'): + if binpath not in path: path = binpath + ';' + path - # make sure externals is in current path for tests and translating - os.environ['PATH'] = path - env['PATH'] = binpath + ';' + env.get('PATH', '') - env['INCLUDE'] = externals + r'\include;' + env.get('INCLUDE', '') - env['LIB'] = externals + r'\lib;' + env.get('LIB', '') + # make sure externals is in current path for tests and translating + os.environ['PATH'] = path + if externals not in os.environ.get('INCLUDE', ''): + os.environ['INCLUDE'] = externals + r'\include;' + os.environ.get('INCLUDE', '') + if externals not in os.environ.get('LIB', ''): + os.environ['LIB'] = externals + r'\lib;' + os.environ.get('LIB', '') return None class WinDefinition(posix.Definition): From pypy.commits at gmail.com Fri Mar 9 07:59:35 2018 From: pypy.commits at gmail.com (mattip) Date: Fri, 09 Mar 2018 04:59:35 -0800 (PST) Subject: [pypy-commit] pypy default: allow calling MsvcPlatform more than once in the process Message-ID: <5aa28537.49da1c0a.4f8d9.83f9@mx.google.com> Author: Matti Picus Branch: Changeset: r93965:63d0a5cf5da2 Date: 2018-03-09 13:48 +0200 http://bitbucket.org/pypy/pypy/changeset/63d0a5cf5da2/ Log: allow calling MsvcPlatform more than once in the process diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -162,17 +162,17 @@ def __init__(self, cc=None, x64=False, ver0=None): self.x64 = x64 + patch_os_env(self.externals) + self.c_environ = os.environ.copy() if cc is None: msvc_compiler_environ, self.vsver = find_msvc_env(x64, ver0=ver0) Platform.__init__(self, 'cl.exe') if msvc_compiler_environ: + self.c_environ.update(msvc_compiler_environ) if x64: self.externals_branch = 'win34_%d' % self.vsver else: self.externals_branch = 'win32_%d' % self.vsver - patch_env(msvc_compiler_environ, self.externals) - self.c_environ = os.environ.copy() - self.c_environ.update(msvc_compiler_environ) else: self.cc = cc @@ -533,17 +533,18 @@ # These are the external libraries, created and maintained by get_externals.py # The buildbot runs get_externals before building -def patch_env(env, externals = Platform.externals): +def patch_os_env(externals = Platform.externals): #print 'adding %s to PATH, INCLUDE, LIB' % basepath binpath = externals + r'\bin' path = os.environ['PATH'] - if binpath not in path.split(';'): + if binpath not in path: path = binpath + ';' + path - # make sure externals is in current path for tests and translating - os.environ['PATH'] = path - env['PATH'] = binpath + ';' + env.get('PATH', '') - env['INCLUDE'] = externals + r'\include;' + env.get('INCLUDE', '') - env['LIB'] = externals + r'\lib;' + env.get('LIB', '') + # make sure externals is in current path for tests and translating + os.environ['PATH'] = path + if externals not in os.environ.get('INCLUDE', ''): + os.environ['INCLUDE'] = externals + r'\include;' + os.environ.get('INCLUDE', '') + if externals not in os.environ.get('LIB', ''): + os.environ['LIB'] = externals + r'\lib;' + os.environ.get('LIB', '') return None class WinDefinition(posix.Definition): From pypy.commits at gmail.com Sat Mar 10 17:21:58 2018 From: pypy.commits at gmail.com (mattip) Date: Sat, 10 Mar 2018 14:21:58 -0800 (PST) Subject: [pypy-commit] pypy py3.5: fix tests, remaining test_rename failure succeeds on cpython3 Message-ID: <5aa45a86.eca9df0a.a7ba6.0e99@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93967:19f63e72f7ce Date: 2018-03-11 00:09 +0200 http://bitbucket.org/pypy/pypy/changeset/19f63e72f7ce/ Log: fix tests, remaining test_rename failure succeeds on cpython3 diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1087,12 +1087,11 @@ if sys.platform == 'win32': os.chmod(my_path, 0o400) assert (os.stat(my_path).st_mode & 0o600) == 0o400 - os.chmod(self.path, 0700) + os.chmod(self.path, 0o700) else: os.chmod(my_path, 0o200) assert (os.stat(my_path).st_mode & 0o777) == 0o200 - os.chmod(self.path, 0700) - os.unlink(self.path) + os.chmod(self.path, 0o700) if hasattr(os, 'fchmod'): def test_fchmod(self): @@ -1103,7 +1102,6 @@ assert (os.fstat(f.fileno()).st_mode & 0o777) == 0o200 f.close() assert (os.stat(my_path).st_mode & 0o777) == 0o200 - os.unlink(self.path) if hasattr(os, 'mkfifo'): def test_mkfifo(self): @@ -1407,16 +1405,17 @@ def test_rename(self): os = self.posix - with open(self.path, "w") as f: + fname = self.path2 + 'rename.txt' + with open(fname, "w") as f: f.write("this is a rename test") unicode_name = str(self.udir) + u'/test\u03be.txt' - os.rename(self.path, unicode_name) + os.rename(fname, unicode_name) with open(unicode_name) as f: assert f.read() == 'this is a rename test' - os.rename(unicode_name, self.path) - with open(self.path) as f: + os.rename(unicode_name, fname) + with open(fname) as f: assert f.read() == 'this is a rename test' - os.unlink(self.path) + os.unlink(fname) def test_device_encoding(self): From pypy.commits at gmail.com Sun Mar 11 16:19:49 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Mar 2018 13:19:49 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: make sure batfile is defined Message-ID: <5aa58f65.abb2df0a.9edef.738a@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93968:a445edc093be Date: 2018-03-11 21:49 +0200 http://bitbucket.org/pypy/pypy/changeset/a445edc093be/ Log: make sure batfile is defined diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -223,6 +223,7 @@ that fails it falls back to the VS90COMNTOOLS env var. """ vsbase = VS_BASE % version + batfile = 'vcvarsall.bat' try: productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, "productdir") @@ -233,7 +234,6 @@ if not productdir or not os.path.isdir(productdir): toolskey = "VS%0.f0COMNTOOLS" % version toolsdir = os.environ.get(toolskey, None) - batfile = 'vcvarsall.bat' if toolsdir and os.path.isdir(toolsdir): if os.path.exists(os.path.join(toolsdir, 'VsDevCmd.bat')): From pypy.commits at gmail.com Sun Mar 11 16:19:51 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 11 Mar 2018 13:19:51 -0700 (PDT) Subject: [pypy-commit] pypy default: skip unicode file name on non-win32 Message-ID: <5aa58f67.c7581c0a.96ec9.8870@mx.google.com> Author: Matti Picus Branch: Changeset: r93969:c462956c2800 Date: 2018-03-11 22:18 +0200 http://bitbucket.org/pypy/pypy/changeset/c462956c2800/ Log: skip unicode file name on non-win32 diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1179,6 +1179,7 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + @py.test.mark.skipif("sys.platform != 'win32'") def test_rename(self): os = self.posix with open(self.path, "w") as f: From pypy.commits at gmail.com Mon Mar 12 08:58:45 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 05:58:45 -0700 (PDT) Subject: [pypy-commit] pypy call-loopinvariant-into-bridges: xfail this test, potential improvement for later Message-ID: <5aa67985.c76d1c0a.b87ce.f09d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: call-loopinvariant-into-bridges Changeset: r93970:76c222aca477 Date: 2018-03-12 13:08 +0100 http://bitbucket.org/pypy/pypy/changeset/76c222aca477/ Log: xfail this test, potential improvement for later diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -360,6 +363,7 @@ self.check_trace_count(3) self.check_resops(call_r=1) + @pytest.mark.xfail() def test_bridge_call_loopinvariant_2(self): class A(object): pass From pypy.commits at gmail.com Mon Mar 12 08:58:48 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 05:58:48 -0700 (PDT) Subject: [pypy-commit] pypy default: merge call-loopinvariant-into-bridges: speed up branchy code that does a lot of inlining Message-ID: <5aa67988.c9351c0a.6c959.fb75@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r93971:ed869ecba520 Date: 2018-03-12 13:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ed869ecba520/ Log: merge call-loopinvariant-into-bridges: speed up branchy code that does a lot of inlining diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,9 @@ .. branch: refactor-slots Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # # ( ) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # # ( ) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# +# ( ) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -27,6 +30,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +65,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +105,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) dct = {box: cls for box, known_class in boxes_known_classes @@ -321,3 +326,74 @@ self.check_trace_count(3) self.check_resops(guard_value=1) self.check_resops(getarrayitem_gc_i=5) + + def test_bridge_call_loopinvariant(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + a = get() + a = get() + res += a.x + if y > n: + res += 1 + res += get().x + a.x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) + + @pytest.mark.xfail() + def test_bridge_call_loopinvariant_2(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + if y > n: + res += get().x + res += 1 + res += get().x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -40,7 +40,7 @@ class FakeOptimizer(object): metainterp_sd = None - optheap = None + optheap = optrewrite = None def __init__(self, trace=None): self.trace = trace From pypy.commits at gmail.com Mon Mar 12 12:52:34 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:34 -0700 (PDT) Subject: [pypy-commit] pypy parser-tuning: a branch to try to improve some things in the (python) parser Message-ID: <5aa6b052.5cb9df0a.e319.ea74@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: parser-tuning Changeset: r93972:9578739334e0 Date: 2018-03-12 14:47 +0100 http://bitbucket.org/pypy/pypy/changeset/9578739334e0/ Log: a branch to try to improve some things in the (python) parser From pypy.commits at gmail.com Mon Mar 12 12:52:39 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:39 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: create Nonterminal._children list lazily Message-ID: <5aa6b057.85b81c0a.aad07.a1db@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93974:d15189dd835a Date: 2018-03-12 14:54 +0100 http://bitbucket.org/pypy/pypy/changeset/d15189dd835a/ Log: create Nonterminal._children list lazily diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -127,21 +127,27 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type, children): + def __init__(self, type): Node.__init__(self, type) - self._children = children + self._children = None def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) def get_child(self, i): + assert self._children is not None return self._children[i] def num_children(self): + if self._children is None: + return 0 return len(self._children) def append_child(self, child): - self._children.append(child) + if self._children is None: + self._children = [child] + else: + self._children.append(child) class Nonterminal1(AbstractNonterminal): @@ -209,7 +215,7 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start, []) + current_node = Nonterminal(start) self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0, current_node) def add_token(self, token_type, value, lineno, column, line): @@ -284,7 +290,7 @@ def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - new_node = Nonterminal(node_type, []) + new_node = Nonterminal(node_type) self.stack.state = next_state self.stack = self.stack.push(next_dfa, 0, new_node) diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -55,8 +55,7 @@ n = parser.Terminal(tp, value, 0, 0) else: tp = gram.symbol_ids[data[0]] - children = [] - n = parser.Nonterminal(tp, children) + n = parser.Nonterminal(tp) new_indent = count_indent(line) if new_indent >= last_indent: if new_indent == last_indent and node_stack: From pypy.commits at gmail.com Mon Mar 12 12:52:41 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:41 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: test Nonterminal1 optimization Message-ID: <5aa6b059.26acdf0a.1c8c9.e71c@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93975:89c59a311709 Date: 2018-03-12 16:59 +0100 http://bitbucket.org/pypy/pypy/changeset/89c59a311709/ Log: test Nonterminal1 optimization diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -290,3 +290,28 @@ NEWLINE ENDMARKER""" assert tree_from_string(expected, gram) == p.parse("hi 42 end") + + + def test_optimized_terminal(self): + gram = """foo: bar baz 'end' NEWLINE ENDMARKER +bar: NAME +baz: NUMBER +""" + p, gram = self.parser_for(gram, False) + expected = """ + foo + bar + NAME "a_name" + baz + NUMBER "42" + NAME "end" + NEWLINE + ENDMARKER""" + input = "a_name 42 end" + tree = p.parse(input) + assert tree_from_string(expected, gram) == tree + assert isinstance(tree, parser.Nonterminal) + assert isinstance(tree.get_child(0), parser.Nonterminal1) + assert isinstance(tree.get_child(1), parser.Nonterminal1) + + From pypy.commits at gmail.com Mon Mar 12 12:52:48 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:48 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: Nonterminal again always has children Message-ID: <5aa6b060.92181c0a.d9c3a.4472@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93978:8cafe63e849a Date: 2018-03-12 17:09 +0100 http://bitbucket.org/pypy/pypy/changeset/8cafe63e849a/ Log: Nonterminal again always has children diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -133,9 +133,11 @@ class Nonterminal(AbstractNonterminal): __slots__ = ("_children", ) - def __init__(self, type): + def __init__(self, type, children=None): Node.__init__(self, type) - self._children = None + if children is None: + children = [] + self._children = children def __repr__(self): return "Nonterminal(type=%s, children=%r)" % (self.type, self._children) @@ -145,15 +147,10 @@ return self._children[i] def num_children(self): - if self._children is None: - return 0 return len(self._children) def append_child(self, child): - if self._children is None: - self._children = [child] - else: - self._children.append(child) + self._children.append(child) class Nonterminal1(AbstractNonterminal): @@ -211,8 +208,8 @@ if node is None: self.node = Nonterminal1(self.dfa.symbol_id, child) elif isinstance(node, Nonterminal1): - self.node = Nonterminal(self.dfa.symbol_id) - self.node._children = [node._child, child] + newnode = self.node = Nonterminal( + self.dfa.symbol_id, [node._child, child]) else: self.node.append_child(child) From pypy.commits at gmail.com Mon Mar 12 12:52:44 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:44 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: add a target for benchmarking just the parser Message-ID: <5aa6b05c.35b8df0a.429f5.f84a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93976:73fdbc94d5d4 Date: 2018-03-12 16:59 +0100 http://bitbucket.org/pypy/pypy/changeset/73fdbc94d5d4/ Log: add a target for benchmarking just the parser diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -132,7 +132,11 @@ w_message = space.str(e.get_w_value(space)) raise error.SyntaxError(space.text_w(w_message)) raise + if enc is not None: + compile_info.encoding = enc + return self._parse(textsrc, compile_info) + def _parse(self, textsrc, compile_info): flags = compile_info.flags # The tokenizer is very picky about how it wants its input. @@ -188,6 +192,4 @@ finally: # Avoid hanging onto the tree. self.root = None - if enc is not None: - compile_info.encoding = enc return tree diff --git a/pypy/interpreter/pyparser/targetparse.py b/pypy/interpreter/pyparser/targetparse.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/pyparser/targetparse.py @@ -0,0 +1,39 @@ +import sys +import os +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +print ROOT +sys.path.insert(0, str(ROOT)) +import time +from pypy.interpreter.pyparser import pyparse + + + +with file("../../../rpython/rlib/unicodedata/unicodedb_5_2_0.py") as f: + s = f.read() + +class FakeSpace(object): + pass + +fakespace = FakeSpace() + +def bench(title): + a = time.clock() + info = pyparse.CompileInfo("", "exec") + parser = pyparse.PythonParser(fakespace) + tree = parser._parse(s, info) + b = time.clock() + print title, (b-a) + + +def entry_point(argv): + bench("foo") + + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None + +if __name__ == '__main__': + entry_point(sys.argv) From pypy.commits at gmail.com Mon Mar 12 12:52:46 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:46 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: lazify the creation of the Nonterminal Message-ID: <5aa6b05e.51bbdf0a.2eb3e.0052@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93977:6440ef7f3d66 Date: 2018-03-12 17:01 +0100 http://bitbucket.org/pypy/pypy/changeset/6440ef7f3d66/ Log: lazify the creation of the Nonterminal this makes it necessary to turn the DFA into an object that knows its id diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -147,8 +147,10 @@ for label, next in state.arcs.iteritems(): arcs.append((self.make_label(gram, label), dfa.index(next))) states.append((arcs, state.is_final)) - gram.dfas.append((states, self.make_first(gram, name))) - assert len(gram.dfas) - 1 == gram.symbol_ids[name] - 256 + symbol_id = gram.symbol_ids[name] + dfa = parser.DFA(symbol_id, states, self.make_first(gram, name)) + gram.dfas.append(dfa) + assert len(gram.dfas) - 1 == symbol_id - 256 gram.start = gram.symbol_ids[self.start_symbol] return gram diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -41,6 +41,12 @@ pass return True +class DFA(object): + def __init__(self, symbol_id, states, first): + self.symbol_id = symbol_id + self.states = states + self.first = first + class Node(object): @@ -188,18 +194,28 @@ class StackEntry(object): - def __init__(self, next, dfa, state, node): + def __init__(self, next, dfa, state): self.next = next self.dfa = dfa self.state = state - self.node = node + self.node = None - def push(self, dfa, state, node): - return StackEntry(self, dfa, state, node) + def push(self, dfa, state): + return StackEntry(self, dfa, state) def pop(self): return self.next + def node_append_child(self, child): + node = self.node + if node is None: + self.node = Nonterminal1(self.dfa.symbol_id, child) + elif isinstance(node, Nonterminal1): + self.node = Nonterminal(self.dfa.symbol_id) + self.node._children = [node._child, child] + else: + self.node.append_child(child) + class Parser(object): @@ -215,8 +231,7 @@ if start == -1: start = self.grammar.start self.root = None - current_node = Nonterminal(start) - self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0, current_node) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) @@ -224,7 +239,7 @@ while True: dfa = self.stack.dfa state_index = self.stack.state - states, first = dfa + states = dfa.states arcs, is_accepting = states[state_index] for i, next_state in arcs: sym_id = self.grammar.labels[i] @@ -242,12 +257,12 @@ return True dfa = self.stack.dfa state_index = self.stack.state - state = dfa[0][state_index] + state = dfa.states[state_index] return False elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa[1]: + if label_index in sub_node_dfa.first: self.push(sub_node_dfa, next_state, sym_id, lineno, column) break @@ -285,26 +300,21 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" new_node = Terminal(token_type, value, lineno, column) - self.stack.node.append_child(new_node) + self.stack.node_append_child(new_node) self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - new_node = Nonterminal(node_type) - self.stack.state = next_state - self.stack = self.stack.push(next_dfa, 0, new_node) + self.stack = self.stack.push(next_dfa, 0) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" top = self.stack self.stack = top.pop() node = top.node + assert node is not None if self.stack: - # we are now done with node, so we can store it more efficiently if - # it has just one child - if node.num_children() == 1: - node = Nonterminal1(node.type, node.get_child(0)) - self.stack.node.append_child(node) + self.stack.node_append_child(node) else: self.root = node diff --git a/pypy/interpreter/pyparser/test/test_metaparser.py b/pypy/interpreter/pyparser/test/test_metaparser.py --- a/pypy/interpreter/pyparser/test/test_metaparser.py +++ b/pypy/interpreter/pyparser/test/test_metaparser.py @@ -34,8 +34,8 @@ assert len(g.dfas) == 1 eval_sym = g.symbol_ids["eval"] assert g.start == eval_sym - states, first = g.dfas[eval_sym - 256] - assert states == [([(1, 1)], False), ([], True)] + dfa = g.dfas[eval_sym - 256] + assert dfa.states == [([(1, 1)], False), ([], True)] assert g.labels[0] == 0 def test_load_python_grammars(self): @@ -51,7 +51,7 @@ def test_items(self): g = self.gram_for("foo: NAME STRING OP '+'") assert len(g.dfas) == 1 - states = g.dfas[g.symbol_ids["foo"] - 256][0] + states = g.dfas[g.symbol_ids["foo"] - 256].states last = states[0][0][0][1] for state in states[1:-1]: assert last < state[0][0][1] diff --git a/pypy/module/parser/pyparser.py b/pypy/module/parser/pyparser.py --- a/pypy/module/parser/pyparser.py +++ b/pypy/module/parser/pyparser.py @@ -152,7 +152,7 @@ # Raise an exception now and be done with it. raise parser_error(space, w_tuple, "Illegal syntax-tree; cannot start with terminal symbol.") - node = pyparse.parser.Nonterminal(type, []) + node = pyparse.parser.Nonterminal(type) build_node_children(space, w_tuple, node, node_state) return node @@ -171,7 +171,7 @@ strn = space.text_w(w_obj) child = pyparse.parser.Terminal(type, strn, node_state.lineno, 0) else: - child = pyparse.parser.Nonterminal(type, []) + child = pyparse.parser.Nonterminal(type) node.append_child(child) if type >= 256: # Nonterminal node build_node_children(space, w_elem, child, node_state) @@ -187,8 +187,7 @@ raise parse_error(space, "Unrecognized node type %d." % type) dfa = parser.grammar.dfas[type] # Run the DFA for this nonterminal - states, first = dfa - arcs, is_accepting = states[0] + arcs, is_accepting = dfa.states[0] for pos in range(tree.num_children()): ch = tree.get_child(pos) for i, next_state in arcs: @@ -198,7 +197,7 @@ if ch.type >= 256: validate_node(space, ch) # Update the state, and move on to the next child. - arcs, is_accepting = states[next_state] + arcs, is_accepting = dfa.states[next_state] break else: raise parse_error(space, "Illegal node") From pypy.commits at gmail.com Mon Mar 12 12:52:36 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:36 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: use a chained stack instead of list-of-tuples Message-ID: <5aa6b054.c89edf0a.57982.e379@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93973:dde9199be301 Date: 2018-03-12 14:48 +0100 http://bitbucket.org/pypy/pypy/changeset/dde9199be301/ Log: use a chained stack instead of list-of-tuples diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -181,12 +181,25 @@ return "ParserError(%s, %r)" % (self.token_type, self.value) +class StackEntry(object): + def __init__(self, next, dfa, state, node): + self.next = next + self.dfa = dfa + self.state = state + self.node = node + + def push(self, dfa, state, node): + return StackEntry(self, dfa, state, node) + + def pop(self): + return self.next + + class Parser(object): def __init__(self, grammar): self.grammar = grammar self.root = None - self.stack = None def prepare(self, start=-1): """Setup the parser for parsing. @@ -197,14 +210,14 @@ start = self.grammar.start self.root = None current_node = Nonterminal(start, []) - self.stack = [] - self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) + self.stack = StackEntry(None, self.grammar.dfas[start - 256], 0, current_node) def add_token(self, token_type, value, lineno, column, line): label_index = self.classify(token_type, value, lineno, column, line) sym_id = 0 # for the annotator while True: - dfa, state_index, node = self.stack[-1] + dfa = self.stack.dfa + state_index = self.stack.state states, first = dfa arcs, is_accepting = states[state_index] for i, next_state in arcs: @@ -217,10 +230,12 @@ # the stack. while state[1] and not state[0]: self.pop() - if not self.stack: + if self.stack is None: + assert self.stack is None # Parsing is done. return True - dfa, state_index, node = self.stack[-1] + dfa = self.stack.dfa + state_index = self.stack.state state = dfa[0][state_index] return False elif sym_id >= 256: @@ -235,7 +250,8 @@ # state is accepting, it's invalid input. if is_accepting: self.pop() - if not self.stack: + if self.stack is None: + assert self.stack is None raise ParseError("too much input", token_type, value, lineno, column, line) else: @@ -262,26 +278,27 @@ def shift(self, next_state, token_type, value, lineno, column): """Shift a non-terminal and prepare for the next state.""" - dfa, state, node = self.stack[-1] new_node = Terminal(token_type, value, lineno, column) - node.append_child(new_node) - self.stack[-1] = (dfa, next_state, node) + self.stack.node.append_child(new_node) + self.stack.state = next_state def push(self, next_dfa, next_state, node_type, lineno, column): """Push a terminal and adjust the current state.""" - dfa, state, node = self.stack[-1] new_node = Nonterminal(node_type, []) - self.stack[-1] = (dfa, next_state, node) - self.stack.append((next_dfa, 0, new_node)) + + self.stack.state = next_state + self.stack = self.stack.push(next_dfa, 0, new_node) def pop(self): """Pop an entry off the stack and make its node a child of the last.""" - dfa, state, node = self.stack.pop() + top = self.stack + self.stack = top.pop() + node = top.node if self.stack: # we are now done with node, so we can store it more efficiently if # it has just one child if node.num_children() == 1: node = Nonterminal1(node.type, node.get_child(0)) - self.stack[-1][2].append_child(node) + self.stack.node.append_child(node) else: self.root = node From pypy.commits at gmail.com Mon Mar 12 12:52:50 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 12 Mar 2018 09:52:50 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: store the first set as 32-char string, as opposed to a dict Message-ID: <5aa6b062.919bdf0a.f1976.d7e2@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93979:2221844101cc Date: 2018-03-12 17:40 +0100 http://bitbucket.org/pypy/pypy/changeset/2221844101cc/ Log: store the first set as 32-char string, as opposed to a dict diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -1,6 +1,7 @@ """ A CPython inspired RPython parser. """ +from rpython.rlib.objectmodel import not_rpython class Grammar(object): @@ -45,8 +46,23 @@ def __init__(self, symbol_id, states, first): self.symbol_id = symbol_id self.states = states - self.first = first + self.first = self._first_to_string(first) + def could_match_token(self, label_index): + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + return bool(ord(self.first[label_index >> 3]) & bit) + + @staticmethod + @not_rpython + def _first_to_string(first): + l = sorted(first.keys()) + b = bytearray(32) + for label_index in l: + pos = label_index >> 3 + bit = 1 << (label_index & 0b111) + b[pos] |= bit + return str(b) class Node(object): @@ -259,7 +275,7 @@ elif sym_id >= 256: sub_node_dfa = self.grammar.dfas[sym_id - 256] # Check if this token can start a child node. - if label_index in sub_node_dfa.first: + if sub_node_dfa.could_match_token(label_index): self.push(sub_node_dfa, next_state, sym_id, lineno, column) break diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -7,6 +7,12 @@ from pypy.interpreter.pyparser.test.test_metaparser import MyGrammar +def test_char_set(): + first = {5: None, 9: None, 100: None, 255:None} + p = parser.DFA(None, None, first) + for i in range(256): + assert p.could_match_token(i) == (i in first) + class SimpleParser(parser.Parser): def parse(self, input): From pypy.commits at gmail.com Tue Mar 13 18:34:42 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 13 Mar 2018 15:34:42 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: Did not get visa :-( Message-ID: <5aa85202.bb86df0a.de80e.992f@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5878:eefa229420b4 Date: 2018-03-13 23:34 +0100 http://bitbucket.org/pypy/extradoc/changeset/eefa229420b4/ Log: Did not get visa :-( diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -13,7 +13,7 @@ Matti Picus 17.3/22.3 Ermina Manuel Jacob 17.3/24.3 Ermina Remi Meier 18.3/24.3 Ermina -Joannah Nanjekye 17.3/25.3 separate room Ermina +Joannah Nanjekye 17.3/25.3 CANCELLED Jean-Daniel 17.3/21.3 own booking Ermina Arianna 17.3/21.3 own booking Ermina Floris Bruynooghe 17.3/21.3 Ermina From pypy.commits at gmail.com Wed Mar 14 03:48:37 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 14 Mar 2018 00:48:37 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: add benchmark item to sprint goals Message-ID: <5aa8d3d5.8ee81c0a.326ec.3e40@mx.google.com> Author: mattip Branch: extradoc Changeset: r5879:c532b161d30f Date: 2018-03-14 07:48 +0000 http://bitbucket.org/pypy/extradoc/changeset/c532b161d30f/ Log: add benchmark item to sprint goals diff --git a/planning/sprint-leysin-2018-notes.rst b/planning/sprint-leysin-2018-notes.rst --- a/planning/sprint-leysin-2018-notes.rst +++ b/planning/sprint-leysin-2018-notes.rst @@ -11,7 +11,8 @@ - General 3.5 and 3.6 improvements - JIT topics: guard-compatible, and the subsequent research project to save and reuse traces across processes - finish unicode-utf8 -- update www.pypy.org, speed.pypy.org (web devs needed) +- update www.pypy.org (web devs needed) - go over projects at https://bitbucket.org/pypy, delete or document dead projects - document the different rpython decorators like enforceargs, signature, and then interp-level unwrap_spec - revdb: bring it up to date, improve usability, other improvements +- update ``prof`` and ``performance`` to better handle warmup, get benchmarks running on https://speed.python.org, kill http://speed.pypy.org \ No newline at end of file From pypy.commits at gmail.com Thu Mar 15 03:36:54 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 15 Mar 2018 00:36:54 -0700 (PDT) Subject: [pypy-commit] cffi default: Issue #362 Message-ID: <5aaa2296.d3badf0a.70b3d.1a83@mx.google.com> Author: Armin Rigo Branch: Changeset: r3115:e5f8ac3b8e6b Date: 2018-03-15 08:36 +0100 http://bitbucket.org/cffi/cffi/changeset/e5f8ac3b8e6b/ Log: Issue #362 Py_Finalize() will free any threadstate around, so in that case we must not call PyThreadState_Delete() any more on them from cffi_thread_shutdown(). diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -7490,6 +7490,9 @@ init_cffi_tls(); if (PyErr_Occurred()) INITERROR; + init_cffi_tls_delete(); + if (PyErr_Occurred()) + INITERROR; if (init_ffi_lib(m) < 0) INITERROR; diff --git a/c/ffi_obj.c b/c/ffi_obj.c --- a/c/ffi_obj.c +++ b/c/ffi_obj.c @@ -941,16 +941,6 @@ #define ffi_memmove b_memmove /* ffi_memmove() => b_memmove() from _cffi_backend.c */ -#ifdef WITH_THREAD -# include "pythread.h" -#else -typedef void *PyThread_type_lock; -# define PyThread_allocate_lock() ((void *)-1) -# define PyThread_free_lock(lock) ((void)(lock)) -# define PyThread_acquire_lock(lock, _) ((void)(lock)) -# define PyThread_release_lock(lock) ((void)(lock)) -#endif - PyDoc_STRVAR(ffi_init_once_doc, "init_once(function, tag): run function() once. More precisely,\n" "'function()' is called the first time we see a given 'tag'.\n" diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h --- a/c/misc_thread_common.h +++ b/c/misc_thread_common.h @@ -1,6 +1,7 @@ #ifndef WITH_THREAD # error "xxx no-thread configuration not tested, please report if you need that" #endif +#include "pythread.h" struct cffi_tls_s { @@ -24,12 +25,79 @@ static struct cffi_tls_s *get_cffi_tls(void); /* in misc_thread_posix.h or misc_win32.h */ + +/* issue #362: Py_Finalize() will free any threadstate around, so in + * that case we must not call PyThreadState_Delete() any more on them + * from cffi_thread_shutdown(). The following mess is to give a + * thread-safe way to know that Py_Finalize() started. + */ +#define TLS_DEL_LOCK() PyThread_acquire_lock(cffi_tls_delete_lock, WAIT_LOCK) +#define TLS_DEL_UNLOCK() PyThread_release_lock(cffi_tls_delete_lock) +static PyThread_type_lock cffi_tls_delete_lock = NULL; +static int cffi_tls_delete; +static PyObject *old_exitfunc; + +static PyObject *cffi_tls_shutdown(PyObject *self, PyObject *args) +{ + /* the lock here will wait until any parallel cffi_thread_shutdown() + is done. Future cffi_thread_shutdown() won't touch their + PyThreadState any more, which are all supposed to be freed anyway + very soon after the present cffi_tls_shutdown() function is called. + */ + TLS_DEL_LOCK(); + cffi_tls_delete = 0; /* Py_Finalize() called */ + TLS_DEL_UNLOCK(); + + PyObject *ofn = old_exitfunc; + if (ofn == NULL) + { + Py_INCREF(Py_None); + return Py_None; + } + else + { + old_exitfunc = NULL; + return PyObject_CallFunction(ofn, ""); + } +} + +static void init_cffi_tls_delete(void) +{ + static PyMethodDef mdef = { + "cffi_tls_shutdown", cffi_tls_shutdown, METH_NOARGS, + }; + PyObject *shutdown_fn; + + cffi_tls_delete_lock = PyThread_allocate_lock(); + if (cffi_tls_delete_lock == NULL) + { + PyErr_SetString(PyExc_SystemError, + "can't allocate cffi_tls_delete_lock"); + return; + } + + shutdown_fn = PyCFunction_New(&mdef, NULL); + if (shutdown_fn == NULL) + return; + + old_exitfunc = PySys_GetObject("exitfunc"); + if (PySys_SetObject("exitfunc", shutdown_fn) == 0) + cffi_tls_delete = 1; /* all ready */ + Py_DECREF(shutdown_fn); +} + static void cffi_thread_shutdown(void *p) { struct cffi_tls_s *tls = (struct cffi_tls_s *)p; if (tls->local_thread_state != NULL) { - PyThreadState_Delete(tls->local_thread_state); + /* + * issue #362: see comments above + */ + TLS_DEL_LOCK(); + if (cffi_tls_delete) + PyThreadState_Delete(tls->local_thread_state); + TLS_DEL_UNLOCK(); } free(tls); } From pypy.commits at gmail.com Fri Mar 16 08:20:29 2018 From: pypy.commits at gmail.com (cfbolz) Date: Fri, 16 Mar 2018 05:20:29 -0700 (PDT) Subject: [pypy-commit] pypy pyparser-improvements: improve error messages of ParseError Message-ID: <5aabb68d.6291df0a.967b1.8103@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: pyparser-improvements Changeset: r93980:1c51dd151fee Date: 2018-03-16 13:18 +0100 http://bitbucket.org/pypy/pypy/changeset/1c51dd151fee/ Log: improve error messages of ParseError only covers very simple cases, but at least if tells you about forgotten ':' in the line starting a block, which is the syntax error I always make. diff --git a/pypy/interpreter/pyparser/metaparser.py b/pypy/interpreter/pyparser/metaparser.py --- a/pypy/interpreter/pyparser/metaparser.py +++ b/pypy/interpreter/pyparser/metaparser.py @@ -164,6 +164,13 @@ else: gram.labels.append(gram.symbol_ids[label]) gram.symbol_to_label[label] = label_index + first = self.first[label] + if len(first) == 1: + first, = first + if not first[0].isupper(): + first = first.strip("\"'") + assert label_index not in gram.token_to_error_string + gram.token_to_error_string[label_index] = first return label_index elif label.isupper(): token_index = gram.TOKENS[label] @@ -185,7 +192,7 @@ else: gram.labels.append(gram.KEYWORD_TOKEN) gram.keyword_ids[value] = label_index - return label_index + result = label_index else: try: token_index = gram.OPERATOR_MAP[value] @@ -196,7 +203,10 @@ else: gram.labels.append(token_index) gram.token_ids[token_index] = label_index - return label_index + result = label_index + assert result not in gram.token_to_error_string + gram.token_to_error_string[result] = value + return result def make_first(self, gram, name): original_firsts = self.first[name] diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py --- a/pypy/interpreter/pyparser/parser.py +++ b/pypy/interpreter/pyparser/parser.py @@ -17,6 +17,7 @@ self.symbol_names = {} self.symbol_to_label = {} self.keyword_ids = {} + self.token_to_error_string = {} self.dfas = [] self.labels = [0] self.token_ids = {} @@ -193,7 +194,7 @@ class ParseError(Exception): def __init__(self, msg, token_type, value, lineno, column, line, - expected=-1): + expected=-1, expected_str=None): self.msg = msg self.token_type = token_type self.value = value @@ -201,6 +202,7 @@ self.column = column self.line = line self.expected = expected + self.expected_str = expected_str def __str__(self): return "ParserError(%s, %r)" % (self.token_type, self.value) @@ -293,10 +295,13 @@ # error. if len(arcs) == 1: expected = sym_id + expected_str = self.grammar.token_to_error_string.get( + arcs[0][0], None) else: expected = -1 + expected_str = None raise ParseError("bad input", token_type, value, lineno, - column, line, expected) + column, line, expected, expected_str) def classify(self, token_type, value, lineno, column, line): """Find the label for a token.""" diff --git a/pypy/interpreter/pyparser/pyparse.py b/pypy/interpreter/pyparser/pyparse.py --- a/pypy/interpreter/pyparser/pyparse.py +++ b/pypy/interpreter/pyparser/pyparse.py @@ -185,6 +185,9 @@ else: new_err = error.SyntaxError msg = "invalid syntax" + if e.expected_str is not None: + msg += " (expected '%s')" % e.expected_str + raise new_err(msg, e.lineno, e.column, e.line, compile_info.filename) else: diff --git a/pypy/interpreter/pyparser/test/test_parser.py b/pypy/interpreter/pyparser/test/test_parser.py --- a/pypy/interpreter/pyparser/test/test_parser.py +++ b/pypy/interpreter/pyparser/test/test_parser.py @@ -321,3 +321,12 @@ assert isinstance(tree.get_child(1), parser.Nonterminal1) + def test_error_string(self): + p, gram = self.parser_for( + "foo: 'if' NUMBER '+' NUMBER" + ) + info = py.test.raises(parser.ParseError, p.parse, "if 42") + info.value.expected_str is None + info = py.test.raises(parser.ParseError, p.parse, "if 42 42") + info.value.expected_str == '+' + diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -165,3 +165,11 @@ for linefeed in ["\r\n","\r"]: tree = self.parse(fmt % linefeed) assert expected_tree == tree + + def test_error_forgotten_chars(self): + info = py.test.raises(SyntaxError, self.parse, "if 1\n print 4") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "for i in range(10)\n print i") + assert "(expected ':')" in info.value.msg + info = py.test.raises(SyntaxError, self.parse, "def f:\n print 1") + assert "(expected '(')" in info.value.msg From pypy.commits at gmail.com Sat Mar 17 11:24:48 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 17 Mar 2018 08:24:48 -0700 (PDT) Subject: [pypy-commit] pypy default: (mattip, arigo) Message-ID: <5aad3340.8481df0a.4c19b.d676@mx.google.com> Author: Armin Rigo Branch: Changeset: r93981:60377e65ccee Date: 2018-03-17 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/60377e65ccee/ Log: (mattip, arigo) Randomly fix a failure on narrow unicode builds diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,3 +1,4 @@ +import sys import pytest try: from hypothesis import given, strategies as st, example, settings @@ -44,5 +45,7 @@ @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): + if s == u'\ufacf' and sys.maxunicode == 65535: + pytest.skip('chr(0xfacf) normalizes to chr(0x2284a), which is too big') norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) From pypy.commits at gmail.com Sat Mar 17 12:00:43 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 17 Mar 2018 09:00:43 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: (mattip, arigo) Message-ID: <5aad3bab.88c0df0a.62094.1a59@mx.google.com> Author: Armin Rigo Branch: unicode-utf8 Changeset: r93982:b73752662f75 Date: 2018-03-17 17:00 +0100 http://bitbucket.org/pypy/pypy/changeset/b73752662f75/ Log: (mattip, arigo) Fix for test_unicodeobject.py in cpyext diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -178,7 +178,7 @@ array = rffi.cast(rffi.CWCHARP, PyUnicode_AS_DATA(space, word)) array2 = PyUnicode_AS_UNICODE(space, word) array3 = PyUnicode_AsUnicode(space, word) - for (i, char) in enumerate(space.unicode_w(word)): + for (i, char) in enumerate(space.utf8_w(word)): assert array[i] == char assert array2[i] == char assert array3[i] == char @@ -216,12 +216,12 @@ def test_fromstring(self, space): s = rffi.str2charp(u'sp\x09m'.encode("utf-8")) w_res = PyUnicode_FromString(space, s) - assert space.unicode_w(w_res) == u'sp\x09m' + assert space.utf8_w(w_res) == u'sp\x09m'.encode("utf-8") res = PyUnicode_FromStringAndSize(space, s, 4) w_res = from_ref(space, res) Py_DecRef(space, res) - assert space.unicode_w(w_res) == u'sp\x09m' + assert space.utf8_w(w_res) == u'sp\x09m'.encode("utf-8") rffi.free_charp(s) def test_unicode_resize(self, space): @@ -256,17 +256,18 @@ u = rffi.str2charp(u'sp\x134m'.encode("utf-8")) w_u = PyUnicode_DecodeUTF8(space, u, 5, None) assert space.type(w_u) is space.w_unicode - assert space.unicode_w(w_u) == u'sp\x134m' + assert space.utf8_w(w_u) == u'sp\x134m'.encode("utf-8") w_u = PyUnicode_DecodeUTF8(space, u, 2, None) assert space.type(w_u) is space.w_unicode - assert space.unicode_w(w_u) == 'sp' + assert space.utf8_w(w_u) == 'sp' rffi.free_charp(u) def test_encode_utf8(self, space): u = rffi.unicode2wcharp(u'sp\x09m') w_s = PyUnicode_EncodeUTF8(space, u, 4, None) - assert space.unicode_w(w_s) == u'sp\x09m'.encode('utf-8') + assert space.type(w_s) is space.w_bytes + assert space.bytes_w(w_s) == u'sp\x09m'.encode('utf-8') rffi.free_wcharp(u) def test_encode_decimal(self, space): @@ -364,18 +365,19 @@ def test_fromobject(self, space): w_u = space.wrap(u'a') assert PyUnicode_FromObject(space, w_u) is w_u - assert space.unicode_w( + assert space.utf8_w( PyUnicode_FromObject(space, space.wrap('test'))) == 'test' def test_decode(self, space): b_text = rffi.str2charp('caf\x82xx') b_encoding = rffi.str2charp('cp437') - assert space.unicode_w( - PyUnicode_Decode(space, b_text, 4, b_encoding, None)) == u'caf\xe9' + assert (space.utf8_w( + PyUnicode_Decode(space, b_text, 4, b_encoding, None)) == + u'caf\xe9'.encode("utf-8")) w_text = PyUnicode_FromEncodedObject(space, space.wrap("test"), b_encoding, None) assert space.isinstance_w(w_text, space.w_unicode) - assert space.unicode_w(w_text) == "test" + assert space.utf8_w(w_text) == "test" with raises_w(space, TypeError): PyUnicode_FromEncodedObject(space, space.wrap(u"test"), @@ -391,8 +393,9 @@ u_text = u'abcdefg' s_text = space.str_w(PyUnicode_AsEncodedString(space, space.wrap(u_text), null_charp, null_charp)) b_text = rffi.str2charp(s_text) - assert space.unicode_w(PyUnicode_Decode( - space, b_text, len(s_text), null_charp, null_charp)) == u_text + assert (space.utf8_w(PyUnicode_Decode( + space, b_text, len(s_text), null_charp, null_charp)) == + u_text.encode("utf-8")) with raises_w(space, TypeError): PyUnicode_FromEncodedObject( space, space.wrap(u_text), null_charp, None) @@ -509,7 +512,7 @@ def test_concat(self, space): w_res = PyUnicode_Concat(space, space.wrap(u'a'), space.wrap(u'b')) - assert space.unicode_w(w_res) == u'ab' + assert space.utf8_w(w_res) == 'ab' def test_copy(self, space): w_x = space.wrap(u"abcd\u0660") @@ -580,30 +583,30 @@ w_format = space.wrap(u'hi %s') w_args = space.wrap((u'test',)) w_formated = PyUnicode_Format(space, w_format, w_args) - assert (space.unicode_w(w_formated) == - space.unicode_w(space.mod(w_format, w_args))) + assert (space.utf8_w(w_formated) == + space.utf8_w(space.mod(w_format, w_args))) def test_join(self, space): w_sep = space.wrap(u'') w_seq = space.wrap([u'a', u'b']) w_joined = PyUnicode_Join(space, w_sep, w_seq) - assert space.unicode_w(w_joined) == u'ab' + assert space.utf8_w(w_joined) == u'ab'.encode("utf-8") def test_fromordinal(self, space): w_char = PyUnicode_FromOrdinal(space, 65) - assert space.unicode_w(w_char) == u'A' + assert space.utf8_w(w_char) == 'A' w_char = PyUnicode_FromOrdinal(space, 0) - assert space.unicode_w(w_char) == u'\0' + assert space.utf8_w(w_char) == '\0' w_char = PyUnicode_FromOrdinal(space, 0xFFFF) - assert space.unicode_w(w_char) == u'\uFFFF' + assert space.utf8_w(w_char) == u'\uFFFF'.encode("utf-8") def test_replace(self, space): w_str = space.wrap(u"abababab") w_substr = space.wrap(u"a") w_replstr = space.wrap(u"z") - assert u"zbzbabab" == space.unicode_w( + assert "zbzbabab" == space.utf8_w( PyUnicode_Replace(space, w_str, w_substr, w_replstr, 2)) - assert u"zbzbzbzb" == space.unicode_w( + assert "zbzbzbzb" == space.utf8_w( PyUnicode_Replace(space, w_str, w_substr, w_replstr, -1)) def test_tailmatch(self, space): From pypy.commits at gmail.com Sat Mar 17 12:13:43 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 17 Mar 2018 09:13:43 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: (mattip, arigo) Message-ID: <5aad3eb7.c89edf0a.38bac.a192@mx.google.com> Author: Armin Rigo Branch: unicode-utf8 Changeset: r93983:eda16cfcfb5e Date: 2018-03-17 17:13 +0100 http://bitbucket.org/pypy/pypy/changeset/eda16cfcfb5e/ Log: (mattip, arigo) more of the same diff --git a/pypy/module/cpyext/test/test_codecs.py b/pypy/module/cpyext/test/test_codecs.py --- a/pypy/module/cpyext/test/test_codecs.py +++ b/pypy/module/cpyext/test/test_codecs.py @@ -11,5 +11,5 @@ w_encoded = space.call_method(w_encoder, 'encode', space.wrap(u'späm')) w_decoder = PyCodec_IncrementalDecoder(space, utf8, None) w_decoded = space.call_method(w_decoder, 'decode', w_encoded) - assert space.unicode_w(w_decoded) == u'späm' + assert space.utf8_w(w_decoded) == u'späm'.encode("utf-8") rffi.free_charp(utf8) diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -131,7 +131,7 @@ finally: rffi.free_charp(buf) w_a = space.getitem(w_globals, space.wrap("a")) - assert space.unicode_w(w_a) == u'caf\xe9' + assert space.utf8_w(w_a) == u'caf\xe9'.encode("utf-8") lltype.free(flags, flavor='raw') def test_run_file(self, space): From pypy.commits at gmail.com Sat Mar 17 12:21:51 2018 From: pypy.commits at gmail.com (arigo) Date: Sat, 17 Mar 2018 09:21:51 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: (mattip, arigo) Message-ID: <5aad409f.49c7df0a.3322d.8cb8@mx.google.com> Author: Armin Rigo Branch: unicode-utf8 Changeset: r93984:ead419b87760 Date: 2018-03-17 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/ead419b87760/ Log: (mattip, arigo) Improve error message for now diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -1034,7 +1034,9 @@ return space.newutf8(rutf8.unichr_as_utf8(code), 1) except ValueError: raise oefmt(space.w_ValueError, - "character is not in range [U+0000; U+10ffff]") + "array contains a 32-bit integer that is outside " + "the range [U+0000; U+10ffff] of valid unicode " + "characters") assert 0, "unreachable" # interface From pypy.commits at gmail.com Sun Mar 18 05:30:10 2018 From: pypy.commits at gmail.com (mjacob) Date: Sun, 18 Mar 2018 02:30:10 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: For the records: my train was cancelled and I arrive one day later. Message-ID: <5aae31a2.099bdf0a.a8077.079d@mx.google.com> Author: Manuel Jacob Branch: extradoc Changeset: r5880:40134a2a819d Date: 2018-03-18 10:29 +0100 http://bitbucket.org/pypy/extradoc/changeset/40134a2a819d/ Log: For the records: my train was cancelled and I arrive one day later. diff --git a/sprintinfo/leysin-winter-2018/people.txt b/sprintinfo/leysin-winter-2018/people.txt --- a/sprintinfo/leysin-winter-2018/people.txt +++ b/sprintinfo/leysin-winter-2018/people.txt @@ -11,7 +11,7 @@ ==================== ============== ======================= Armin Rigo private Matti Picus 17.3/22.3 Ermina -Manuel Jacob 17.3/24.3 Ermina +Manuel Jacob 18.3/24.3 Ermina Remi Meier 18.3/24.3 Ermina Joannah Nanjekye 17.3/25.3 CANCELLED Jean-Daniel 17.3/21.3 own booking Ermina From pypy.commits at gmail.com Sun Mar 18 07:21:10 2018 From: pypy.commits at gmail.com (rlamy) Date: Sun, 18 Mar 2018 04:21:10 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: merge from (old) default Message-ID: <5aae4ba6.c83f1c0a.112b6.406a@mx.google.com> Author: Ronan Lamy Branch: unicode-utf8 Changeset: r93985:542b2a7958bf Date: 2018-03-18 11:20 +0000 http://bitbucket.org/pypy/pypy/changeset/542b2a7958bf/ Log: merge from (old) default diff --git a/extra_tests/test_json.py b/extra_tests/test_json.py --- a/extra_tests/test_json.py +++ b/extra_tests/test_json.py @@ -1,5 +1,6 @@ import pytest import json +from hypothesis import given, strategies def is_(x, y): return type(x) is type(y) and x == y @@ -18,3 +19,15 @@ def test_issue2191(): assert is_(json.dumps(u"xxx", ensure_ascii=False), u'"xxx"') + +jsondata = strategies.recursive( + strategies.none() | + strategies.booleans() | + strategies.floats(allow_nan=False) | + strategies.text(), + lambda children: strategies.lists(children) | + strategies.dictionaries(strategies.text(), children)) + + at given(jsondata) +def test_roundtrip(d): + assert json.loads(json.dumps(d)) == d diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -2,11 +2,10 @@ from pypy.interpreter.error import OperationError, oefmt from rpython.rlib.objectmodel import specialize +from rpython.rlib.rstring import StringBuilder from rpython.rlib import rutf8 from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rlib.rstring import StringBuilder from rpython.rtyper.lltypesystem import rffi -from pypy.module._codecs import interp_codecs from pypy.module.unicodedata import unicodedb @specialize.memo() @@ -64,6 +63,7 @@ # These functions take and return unwrapped rpython strings def decode_unicode_escape(space, string): + from pypy.module._codecs import interp_codecs state = space.fromcache(interp_codecs.CodecState) unicodedata_handler = state.get_unicodedata_handler(space) result_utf8, consumed, length = str_decode_unicode_escape( @@ -1268,6 +1268,41 @@ return unicode_encode_utf_32_helper(s, errors, errorhandler, allow_surrogates, "little") +def py3k_str_decode_utf_32(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "native", 'utf-32-' + BYTEORDER2) + return result, length + +def py3k_str_decode_utf_32_be(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "big", 'utf-32-be') + return result, length + +def py3k_str_decode_utf_32_le(s, size, errors, final=True, + errorhandler=None): + result, length, byteorder = str_decode_utf_32_helper( + s, size, errors, final, errorhandler, "little", 'utf-32-le') + return result, length + +def py3k_unicode_encode_utf_32(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "native", + 'utf-32-' + BYTEORDER2) + +def py3k_unicode_encode_utf_32_be(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "big", + 'utf-32-be') + +def py3k_unicode_encode_utf_32_le(s, size, errors, + errorhandler=None, allow_surrogates=True): + return unicode_encode_utf_32_helper(s, size, errors, errorhandler, + allow_surrogates, "little", + 'utf-32-le') # ____________________________________________________________ # unicode-internal diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -1,9 +1,12 @@ from rpython.rlib import jit, rutf8 from rpython.rlib.objectmodel import we_are_translated, not_rpython -from rpython.rlib.rstring import StringBuilder +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder +from rpython.rlib import runicode +from rpython.rlib.runicode import code_to_unichr, MAXUNICODE from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.interpreter import unicodehelper class VersionTag(object): @@ -379,12 +382,15 @@ raise oefmt(space.w_TypeError, "handler must be callable") # ____________________________________________________________ -# delegation to runicode +# delegation to runicode/unicodehelper -from rpython.rlib import runicode +def _find_implementation(impl_name): + func = getattr(unicodehelper, impl_name) + return func def make_encoder_wrapper(name): rname = "utf8_encode_%s" % (name.replace("_encode", ""), ) + func = _find_implementation(rname) @unwrap_spec(errors='text_or_none') def wrap_encoder(space, w_arg, errors="strict"): from pypy.interpreter import unicodehelper @@ -393,7 +399,6 @@ if errors is None: errors = 'strict' state = space.fromcache(CodecState) - func = getattr(unicodehelper, rname) utf8len = w_arg._length # XXX deal with func() returning length or not result = func(w_arg._utf8, errors, state.encode_error_handler) @@ -403,7 +408,7 @@ def make_decoder_wrapper(name): rname = "str_decode_%s" % (name.replace("_decode", ""), ) - assert hasattr(runicode, rname) + func = _find_implementation(rname) @unwrap_spec(string='bufferstr', errors='text_or_none', w_final=WrappedDefault(False)) def wrap_decoder(space, string, errors="strict", w_final=None): @@ -413,7 +418,6 @@ errors = 'strict' final = space.is_true(w_final) state = space.fromcache(CodecState) - func = getattr(unicodehelper, rname) result, consumed, length = func(string, errors, final, state.decode_error_handler) return space.newtuple([space.newutf8(result, length), diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -18,6 +18,7 @@ from pypy.module.cpyext.bytesobject import PyString_Check from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.module._codecs.interp_codecs import CodecState +from pypy.interpreter import unicodehelper from pypy.objspace.std import unicodeobject import sys @@ -622,7 +623,7 @@ else: errors = None - result, _, length, byteorder = str_decode_utf_32_helper( + result, _, length, byteorder = unicodehelper.str_decode_utf_32_helper( string, errors, final=True, errorhandler=None, byteorder=byteorder) if pbyteorder is not None: pbyteorder[0] = rffi.cast(rffi.INT, byteorder) diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -712,7 +712,7 @@ # ____________________________________________________________ -# utf-32 +# utf-32 (not used in PyPy any more) def str_decode_utf_32(s, size, errors, final=True, errorhandler=None): From pypy.commits at gmail.com Sun Mar 18 09:05:38 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 06:05:38 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5aae6422.14b8df0a.6931.df8c@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r93986:19f8ede600f5 Date: 2018-03-18 12:33 +0100 http://bitbucket.org/pypy/pypy/changeset/19f8ede600f5/ Log: merge default into branch diff too long, truncating to 2000 out of 95489 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -44,3 +44,10 @@ d72f9800a42b46a8056951b1da2426d2c2d8d502 release-pypy3.5-v5.9.0 03d614975835870da65ff0481e1edad68ebbcb8d release-pypy2.7-v5.9.0 84a2f3e6a7f88f2fe698e473998755b3bd1a12e2 release-pypy2.7-v5.9.0 +0e7ea4fe15e82d5124e805e2e4a37cae1a402d4b release-pypy2.7-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +a91df6163fb76df245091f741dbf6a23ddc72374 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +0000000000000000000000000000000000000000 release-pypy3.5-v5.10.0 +09f9160b643e3f02ccb8c843b2fbb4e5cbf54082 release-pypy3.5-v5.10.0 +3f6eaa010fce78cc7973bdc1dfdb95970f08fed2 release-pypy3.5-v5.10.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -30,7 +30,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2017 +PyPy Copyright holders 2003-2018 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -339,8 +339,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -363,11 +365,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -388,6 +392,7 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr Heinrich-Heine University, Germany diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt --- a/extra_tests/requirements.txt +++ b/extra_tests/requirements.txt @@ -1,2 +1,3 @@ pytest hypothesis +vmprof diff --git a/pypy/module/test_lib_pypy/pyrepl/__init__.py b/extra_tests/test_pyrepl/__init__.py rename from pypy/module/test_lib_pypy/pyrepl/__init__.py rename to extra_tests/test_pyrepl/__init__.py --- a/pypy/module/test_lib_pypy/pyrepl/__init__.py +++ b/extra_tests/test_pyrepl/__init__.py @@ -1,3 +1,1 @@ -import sys -import lib_pypy.pyrepl -sys.modules['pyrepl'] = sys.modules['lib_pypy.pyrepl'] + diff --git a/pypy/module/test_lib_pypy/pyrepl/infrastructure.py b/extra_tests/test_pyrepl/infrastructure.py rename from pypy/module/test_lib_pypy/pyrepl/infrastructure.py rename to extra_tests/test_pyrepl/infrastructure.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_basic.py b/extra_tests/test_pyrepl/test_basic.py rename from pypy/module/test_lib_pypy/pyrepl/test_basic.py rename to extra_tests/test_pyrepl/test_basic.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_bugs.py b/extra_tests/test_pyrepl/test_bugs.py rename from pypy/module/test_lib_pypy/pyrepl/test_bugs.py rename to extra_tests/test_pyrepl/test_bugs.py diff --git a/extra_tests/test_pyrepl/test_functional.py b/extra_tests/test_pyrepl/test_functional.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_pyrepl/test_functional.py @@ -0,0 +1,28 @@ +# Copyright 2000-2007 Michael Hudson-Doyle +# Maciek Fijalkowski +# License: MIT +# some functional tests, to see if this is really working + +import pytest +import sys + + + at pytest.fixture() +def child(): + try: + import pexpect + except ImportError: + pytest.skip("no pexpect module") + except SyntaxError: + pytest.skip('pexpect wont work on py3k') + child = pexpect.spawn(sys.executable, ['-S'], timeout=10) + child.logfile = sys.stdout + child.sendline('from pyrepl.python_reader import main') + child.sendline('main()') + return child + + +def test_basic(child): + child.sendline('a = 3') + child.sendline('a') + child.expect('3') diff --git a/pypy/module/test_lib_pypy/pyrepl/test_keymap.py b/extra_tests/test_pyrepl/test_keymap.py rename from pypy/module/test_lib_pypy/pyrepl/test_keymap.py rename to extra_tests/test_pyrepl/test_keymap.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_reader.py b/extra_tests/test_pyrepl/test_reader.py rename from pypy/module/test_lib_pypy/pyrepl/test_reader.py rename to extra_tests/test_pyrepl/test_reader.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_readline.py b/extra_tests/test_pyrepl/test_readline.py rename from pypy/module/test_lib_pypy/pyrepl/test_readline.py rename to extra_tests/test_pyrepl/test_readline.py diff --git a/pypy/module/test_lib_pypy/pyrepl/test_wishes.py b/extra_tests/test_pyrepl/test_wishes.py rename from pypy/module/test_lib_pypy/pyrepl/test_wishes.py rename to extra_tests/test_pyrepl/test_wishes.py diff --git a/extra_tests/test_vmprof_greenlet.py b/extra_tests/test_vmprof_greenlet.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_vmprof_greenlet.py @@ -0,0 +1,28 @@ +import time +import pytest +import greenlet +vmprof = pytest.importorskip('vmprof') + +def count_samples(filename): + stats = vmprof.read_profile(filename) + return len(stats.profiles) + +def cpuburn(duration): + end = time.time() + duration + while time.time() < end: + pass + +def test_sampling_inside_callback(tmpdir): + # see also test_sampling_inside_callback inside + # pypy/module/_continuation/test/test_stacklet.py + # + G = greenlet.greenlet(cpuburn) + fname = tmpdir.join('log.vmprof') + with fname.open('w+b') as f: + vmprof.enable(f.fileno(), 1/250.0) + G.switch(0.1) + vmprof.disable() + + samples = count_samples(str(fname)) + # 0.1 seconds at 250Hz should be 25 samples + assert 23 < samples < 27 diff --git a/get_externals.py b/get_externals.py new file mode 100644 --- /dev/null +++ b/get_externals.py @@ -0,0 +1,69 @@ +'''Get external dependencies for building PyPy +they will end up in the platform.host().basepath, something like repo-root/external +''' + +from __future__ import print_function + +import argparse +import os +import zipfile +from subprocess import Popen, PIPE +from rpython.translator.platform import host + +def runcmd(cmd, verbose): + stdout = stderr = '' + report = False + try: + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.wait() != 0 or verbose: + report = True + except Exception as e: + stderr = str(e) + '\n' + stderr + report = True + if report: + print('running "%s" returned\n%s\n%s' % (' '.join(cmd), stdout, stderr)) + if stderr: + raise RuntimeError(stderr) + +def checkout_repo(dest='externals', org='pypy', branch='default', verbose=False): + url = 'https://bitbucket.org/{}/externals'.format(org) + if not os.path.exists(dest): + cmd = ['hg','clone',url,dest] + runcmd(cmd, verbose) + cmd = ['hg','-R', dest, 'update',branch] + runcmd(cmd, verbose) + +def extract_zip(externals_dir, zip_path): + with zipfile.ZipFile(os.fspath(zip_path)) as zf: + zf.extractall(os.fspath(externals_dir)) + return externals_dir / zf.namelist()[0].split('/')[0] + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument('-v', '--verbose', action='store_true') + p.add_argument('-O', '--organization', + help='Organization owning the deps repos', default='pypy') + p.add_argument('-e', '--externals', default=host.externals, + help='directory in which to store dependencies', + ) + p.add_argument('-b', '--branch', default=host.externals_branch, + help='branch to check out', + ) + p.add_argument('-p', '--platform', default=None, + help='someday support cross-compilation, ignore for now', + ) + return p.parse_args() + + +def main(): + args = parse_args() + checkout_repo( + dest=args.externals, + org=args.organization, + branch=args.branch, + verbose=args.verbose, + ) + +if __name__ == '__main__': + main() diff --git a/lib-python/2.7/subprocess.py b/lib-python/2.7/subprocess.py --- a/lib-python/2.7/subprocess.py +++ b/lib-python/2.7/subprocess.py @@ -1296,7 +1296,7 @@ 'copyfile' in caller.f_globals): dest_dir = sys.pypy_resolvedirof(target_executable) src_dir = sys.pypy_resolvedirof(sys.executable) - for libname in ['libpypy-c.so', 'libpypy-c.dylib']: + for libname in ['libpypy-c.so', 'libpypy-c.dylib', 'libpypy-c.dll']: dest_library = os.path.join(dest_dir, libname) src_library = os.path.join(src_dir, libname) if os.path.exists(src_library): diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py --- a/lib_pypy/_ctypes/array.py +++ b/lib_pypy/_ctypes/array.py @@ -12,7 +12,8 @@ if cls == (_CData,): # this is the Array class defined below res._ffiarray = None return res - if not hasattr(res, '_length_') or not isinstance(res._length_, int): + if not hasattr(res, '_length_') or not isinstance(res._length_, + (int, long)): raise AttributeError( "class must define a '_length_' attribute, " "which must be a positive integer") diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -2,7 +2,7 @@ import tempfile, binascii -def get_hashed_dir(cfile): +def _get_hashed_filename(cfile): with open(cfile,'r') as fid: content = fid.read() # from cffi's Verifier() @@ -21,10 +21,28 @@ username = os.environ['USERNAME'] #windows except KeyError: username = os.getuid() - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % ( username, k1, k2) - if not os.path.exists(output_dir): + +def get_hashed_dir(cfile): + hashed_fn = _get_hashed_filename(cfile) + try: + with open(hashed_fn) as f: + dirname = f.read(1024) + except IOError: + dirname = '' + tmpdir = tempfile.gettempdir() + if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname + or not os.path.isdir(os.path.join(tmpdir, dirname))): + dirname = binascii.hexlify(os.urandom(8)) + if not isinstance(dirname, str): # Python 3 + dirname = dirname.decode('ascii') + dirname = 'testcapi_' + dirname + output_dir = os.path.join(tmpdir, dirname) + try: os.mkdir(output_dir) + except OSError: + pass return output_dir @@ -34,13 +52,12 @@ return ext -def compile_shared(csource, modulename, output_dir=None): +def compile_shared(csource, modulename, output_dir): """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module, and import it. """ thisdir = os.path.dirname(__file__) - if output_dir is None: - output_dir = tempfile.mkdtemp() + assert output_dir is not None from distutils.ccompiler import new_compiler @@ -85,4 +102,16 @@ # Now import the newly created library, it will replace the original # module in sys.modules fp, filename, description = imp.find_module(modulename, path=[output_dir]) - imp.load_module(modulename, fp, filename, description) + with fp: + imp.load_module(modulename, fp, filename, description) + + # If everything went fine up to now, write the name of this new + # directory to 'hashed_fn', for future processes (and to avoid a + # growing number of temporary directories that are not completely + # obvious to clean up on Windows) + hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource)) + try: + with open(hashed_fn, 'w') as f: + f.write(os.path.basename(output_dir)) + except IOError: + pass diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -153,9 +153,10 @@ factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) + res = factory(database, timeout, detect_types, isolation_level, + check_same_thread, factory, cached_statements) add_memory_pressure(100 * 1024) - return factory(database, timeout, detect_types, isolation_level, - check_same_thread, factory, cached_statements) + return res def _unicode_text_factory(x): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,11 +1,12 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.2 +Version: 1.11.5 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: python-cffi at googlegroups.com License: MIT +Description-Content-Type: UNKNOWN Description: CFFI ==== @@ -27,5 +28,7 @@ Classifier: Programming Language :: Python :: 3.2 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.2" -__version_info__ = (1, 11, 2) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -7,6 +7,16 @@ we can learn about Py_DEBUG from pyconfig.h, but it is unclear if the same works for the other two macros. Py_DEBUG implies them, but not the other way around. + + Issue #350 is still open: on Windows, the code here causes it to link + with PYTHON36.DLL (for example) instead of PYTHON3.DLL. A fix was + attempted in 164e526a5515 and 14ce6985e1c3, but reverted: virtualenv + does not make PYTHON3.DLL available, and so the "correctly" compiled + version would not run inside a virtualenv. We will re-apply the fix + after virtualenv has been fixed for some time. For explanation, see + issue #355. For a workaround if you want PYTHON3.DLL and don't worry + about virtualenv, see issue #350. See also 'py_limited_api' in + setuptools_ext.py. */ #if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) # include diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -247,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.2" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -295,8 +295,9 @@ base_module_name = self.module_name.split('.')[-1] if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) - prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding),)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -835,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -883,7 +888,7 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") @@ -895,7 +900,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, @@ -1271,17 +1277,18 @@ _generate_cpy_extern_python_plus_c_ctx = \ _generate_cpy_extern_python_ctx - def _string_literal(self, s): - def _char_repr(c): - # escape with a '\' the characters '\', '"' or (for trigraphs) '?' - if c in '\\"?': return '\\' + c - if ' ' <= c < '\x7F': return c - if c == '\n': return '\\n' - return '\\%03o' % ord(c) - lines = [] - for line in s.splitlines(True) or ['']: - lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) - return ' \\\n'.join(lines) + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + for line in s.splitlines(True): + prnt(('// ' + line).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (ord(c),) + prnt(printed_line) # ---------- # emitting the opcodes for individual types diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -143,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -164,6 +169,17 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -301,7 +301,6 @@ return suffixes def _ensure_dir(filename): - try: - os.makedirs(os.path.dirname(filename)) - except OSError: - pass + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/lib_pypy/datetime.py b/lib_pypy/datetime.py --- a/lib_pypy/datetime.py +++ b/lib_pypy/datetime.py @@ -17,10 +17,13 @@ """ from __future__ import division -import time as _time +import time as _timemodule import math as _math import struct as _struct +# for cpyext, use these as base classes +from __pypy__._pypydatetime import dateinterop, deltainterop, timeinterop + _SENTINEL = object() def _cmp(x, y): @@ -179,7 +182,7 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): wday = (_ymd2ord(y, m, d) + 6) % 7 dnum = _days_before_month(y, m) + d - return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) + return _timemodule.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) def _format_time(hh, mm, ss, us): # Skip trailing microseconds when us==0. @@ -247,7 +250,7 @@ else: push(ch) newformat = "".join(newformat) - return _time.strftime(newformat, timetuple) + return _timemodule.strftime(newformat, timetuple) # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): @@ -433,7 +436,7 @@ raise TypeError("unsupported type for timedelta %s component: %s" % (tag, type(num))) -class timedelta(object): +class timedelta(deltainterop): """Represent the difference between two datetime objects. Supported operators: @@ -489,7 +492,7 @@ if not -_MAX_DELTA_DAYS <= d <= _MAX_DELTA_DAYS: raise OverflowError("days=%d; must have magnitude <= %d" % (d, _MAX_DELTA_DAYS)) - self = object.__new__(cls) + self = deltainterop.__new__(cls) self._days = d self._seconds = s self._microseconds = us @@ -667,7 +670,7 @@ timedelta.max = timedelta(_MAX_DELTA_DAYS, 24*3600-1, 1000000-1) timedelta.resolution = timedelta(microseconds=1) -class date(object): +class date(dateinterop): """Concrete date type. Constructors: @@ -707,12 +710,12 @@ if month is None and isinstance(year, bytes) and len(year) == 4 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year) self._hashcode = -1 return self year, month, day = _check_date_fields(year, month, day) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -724,13 +727,13 @@ @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." - y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) + y, m, d, hh, mm, ss, weekday, jday, dst = _timemodule.localtime(t) return cls(y, m, d) @classmethod def today(cls): "Construct a date from time.time()." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t) @classmethod @@ -1061,7 +1064,7 @@ _tzinfo_class = tzinfo -class time(object): +class time(timeinterop): """Time with time zone. Constructors: @@ -1097,14 +1100,14 @@ """ if isinstance(hour, bytes) and len(hour) == 6 and ord(hour[0]) < 24: # Pickle support - self = object.__new__(cls) + self = timeinterop.__new__(cls) self.__setstate(hour, minute or None) self._hashcode = -1 return self hour, minute, second, microsecond = _check_time_fields( hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = timeinterop.__new__(cls) self._hour = hour self._minute = minute self._second = second @@ -1408,15 +1411,20 @@ if isinstance(year, bytes) and len(year) == 10 and \ 1 <= ord(year[2]) <= 12: # Pickle support - self = object.__new__(cls) + self = dateinterop.__new__(cls) self.__setstate(year, month) self._hashcode = -1 return self - year, month, day = _check_date_fields(year, month, day) - hour, minute, second, microsecond = _check_time_fields( - hour, minute, second, microsecond) + elif isinstance(year, tuple) and len(year) == 7: + # Used by internal functions where the arguments are guaranteed to + # be valid. + year, month, day, hour, minute, second, microsecond = year + else: + year, month, day = _check_date_fields(year, month, day) + hour, minute, second, microsecond = _check_time_fields( + hour, minute, second, microsecond) _check_tzinfo_arg(tzinfo) - self = object.__new__(cls) + self = dateinterop.__new__(cls) self._year = year self._month = month self._day = day @@ -1461,7 +1469,7 @@ A timezone info object may be passed in as well. """ _check_tzinfo_arg(tz) - converter = _time.localtime if tz is None else _time.gmtime + converter = _timemodule.localtime if tz is None else _timemodule.gmtime self = cls._from_timestamp(converter, timestamp, tz) if tz is not None: self = tz.fromutc(self) @@ -1470,7 +1478,7 @@ @classmethod def utcfromtimestamp(cls, t): "Construct a UTC datetime from a POSIX timestamp (like time.time())." - return cls._from_timestamp(_time.gmtime, t, None) + return cls._from_timestamp(_timemodule.gmtime, t, None) @classmethod def _from_timestamp(cls, converter, timestamp, tzinfo): @@ -1488,18 +1496,18 @@ us = 0 y, m, d, hh, mm, ss, weekday, jday, dst = converter(timestamp) ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us, tzinfo) + return cls((y, m, d, hh, mm, ss, us), tzinfo=tzinfo) @classmethod def now(cls, tz=None): "Construct a datetime from time.time() and optional time zone info." - t = _time.time() + t = _timemodule.time() return cls.fromtimestamp(t, tz) @classmethod def utcnow(cls): "Construct a UTC datetime from time.time()." - t = _time.time() + t = _timemodule.time() return cls.utcfromtimestamp(t) @classmethod @@ -1797,7 +1805,7 @@ return diff and 1 or 0 def _add_timedelta(self, other, factor): - y, m, d, hh, mm, ss, us = _normalize_datetime( + result = _normalize_datetime( self._year, self._month, self._day + other.days * factor, @@ -1805,7 +1813,7 @@ self._minute, self._second + other.seconds * factor, self._microsecond + other.microseconds * factor) - return datetime(y, m, d, hh, mm, ss, us, tzinfo=self._tzinfo) + return datetime(result, tzinfo=self._tzinfo) def __add__(self, other): "Add a datetime and a timedelta." diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py --- a/lib_pypy/dbm.py +++ b/lib_pypy/dbm.py @@ -157,7 +157,14 @@ def open(filename, flag='r', mode=0666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") openflag = 0 diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info --- a/lib_pypy/greenlet.egg-info +++ b/lib_pypy/greenlet.egg-info @@ -1,6 +1,6 @@ Metadata-Version: 1.0 Name: greenlet -Version: 0.4.12 +Version: 0.4.13 Summary: Lightweight in-process concurrent programming Home-page: https://github.com/python-greenlet/greenlet Author: Ralf Schmitt (for CPython), PyPy team diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py --- a/lib_pypy/greenlet.py +++ b/lib_pypy/greenlet.py @@ -1,7 +1,7 @@ import sys import _continuation -__version__ = "0.4.12" +__version__ = "0.4.13" # ____________________________________________________________ # Exceptions diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -59,7 +59,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2017, The PyPy Project' +copyright = u'2018, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -217,6 +217,7 @@ Alejandro J. Cura Vladimir Kryachko Gabriel + Thomas Hisch Mark Williams Kunal Grover Nathan Taylor @@ -306,8 +307,10 @@ Stanisław Halik Julien Phalip Roman Podoliaka + Steve Papanik Eli Stevens Boglarka Vezer + gabrielg PavloKapyshin Tomer Chachamu Christopher Groskopf @@ -330,11 +333,13 @@ Konrad Delong Dinu Gherman pizi + Tomáš Pružina James Robert Armin Ronacher Diana Popa Mads Kiilerich Brett Cannon + Caleb Hattingh aliceinwire Zooko Wilcox-O Hearn James Lan @@ -355,4 +360,5 @@ Jason Madden Yaroslav Fedevych Even Wiik Thomassen + m at funkyhat.org Stefan Marr diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -545,6 +545,20 @@ ``del foo.bar`` where ``foo`` is a module (or class) that contains the function ``bar``, is significantly slower than CPython. +* Various built-in functions in CPython accept only positional arguments + and not keyword arguments. That can be considered a long-running + historical detail: newer functions tend to accept keyword arguments + and older function are occasionally fixed to do so as well. In PyPy, + most built-in functions accept keyword arguments (``help()`` shows the + argument names). But don't rely on it too much because future + versions of PyPy may have to rename the arguments if CPython starts + accepting them too. + +* PyPy3: ``distutils`` has been enhanced to allow finding ``VsDevCmd.bat`` in the + directory pointed to by the ``VS%0.f0COMNTOOLS`` (typically ``VS140COMNTOOLS``) + environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** + that value. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/gc_info.rst b/pypy/doc/gc_info.rst --- a/pypy/doc/gc_info.rst +++ b/pypy/doc/gc_info.rst @@ -1,17 +1,137 @@ -Garbage collector configuration -=============================== +Garbage collector documentation and configuration +================================================= + + +Incminimark +----------- + +PyPy's default garbage collector is called incminimark - it's an incremental, +generational moving collector. Here we hope to explain a bit how it works +and how it can be tuned to suit the workload. + +Incminimark first allocates objects in so called *nursery* - place for young +objects, where allocation is very cheap, being just a pointer bump. The nursery +size is a very crucial variable - depending on your workload (one or many +processes) and cache sizes you might want to experiment with it via +*PYPY_GC_NURSERY* environment variable. When the nursery is full, there is +performed a minor collection. Freed objects are no longer referencable and +just die, just by not being referenced any more; on the other hand, objects +found to still be alive must survive and are copied from the nursery +to the old generation. Either to arenas, which are collections +of objects of the same size, or directly allocated with malloc if they're +larger. (A third category, the very large objects, are initially allocated +outside the nursery and never move.) + +Since Incminimark is an incremental GC, the major collection is incremental, +meaning there should not be any pauses longer than 1ms. + + +Fragmentation +------------- + +Before we discuss issues of "fragmentation", we need a bit of precision. +There are two kinds of related but distinct issues: + +* If the program allocates a lot of memory, and then frees it all by + dropping all references to it, then we might expect to see the RSS + to drop. (RSS = Resident Set Size on Linux, as seen by "top"; it is an + approximation of the actual memory usage from the OS's point of view.) + This might not occur: the RSS may remain at its highest value. This + issue is more precisely caused by the process not returning "free" + memory to the OS. We call this case "unreturned memory". + +* After doing the above, if the RSS didn't go down, then at least future + allocations should not cause the RSS to grow more. That is, the process + should reuse unreturned memory as long as it has got some left. If this + does not occur, the RSS grows even larger and we have real fragmentation + issues. + + +gc.get_stats +------------ + +There is a special function in the ``gc`` module called +``get_stats(memory_pressure=False)``. + +``memory_pressure`` controls whether or not to report memory pressure from +objects allocated outside of the GC, which requires walking the entire heap, +so it's disabled by default due to its cost. Enable it when debugging +mysterious memory disappearance. + +Example call looks like that:: + + >>> gc.get_stats(True) + Total memory consumed: + GC used: 4.2MB (peak: 4.2MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler used: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.2MB + + Total memory allocated: + GC allocated: 4.5MB (peak: 4.5MB) + in arenas: 763.7kB + rawmalloced: 383.1kB + nursery: 3.1MB + raw assembler allocated: 0.0kB + memory pressure: 0.0kB + ----------------------------- + Total: 4.5MB + +In this particular case, which is just at startup, GC consumes relatively +little memory and there is even less unused, but allocated memory. In case +there is a lot of unreturned memory or actual fragmentation, the "allocated" +can be much higher than "used". Generally speaking, "peak" will more closely +resemble the actual memory consumed as reported by RSS. Indeed, returning +memory to the OS is a hard and not solved problem. In PyPy, it occurs only if +an arena is entirely free---a contiguous block of 64 pages of 4 or 8 KB each. +It is also rare for the "rawmalloced" category, at least for common system +implementations of ``malloc()``. + +The details of various fields: + +* GC in arenas - small old objects held in arenas. If the amount "allocated" + is much higher than the amount "used", we have unreturned memory. It is + possible but unlikely that we have internal fragmentation here. However, + this unreturned memory cannot be reused for any ``malloc()``, including the + memory from the "rawmalloced" section. + +* GC rawmalloced - large objects allocated with malloc. This is gives the + current (first block of text) and peak (second block of text) memory + allocated with ``malloc()``. The amount of unreturned memory or + fragmentation caused by ``malloc()`` cannot easily be reported. Usually + you can guess there is some if the RSS is much larger than the total + memory reported for "GC allocated", but do keep in mind that this total + does not include malloc'ed memory not known to PyPy's GC at all. If you + guess there is some, consider using `jemalloc`_ as opposed to system malloc. + +.. _`jemalloc`: http://jemalloc.net/ + +* nursery - amount of memory allocated for nursery, fixed at startup, + controlled via an environment variable + +* raw assembler allocated - amount of assembler memory that JIT feels + responsible for + +* memory pressure, if asked for - amount of memory we think got allocated + via external malloc (eg loading cert store in SSL contexts) that is kept + alive by GC objects, but not accounted in the GC + .. _minimark-environment-variables: -Minimark --------- +Environment variables +--------------------- PyPy's default ``incminimark`` garbage collector is configurable through several environment variables: ``PYPY_GC_NURSERY`` The nursery size. - Defaults to 1/2 of your cache or ``4M``. + Defaults to 1/2 of your last-level cache, or ``4M`` if unknown. Small values (like 1 or 1KB) are useful for debugging. ``PYPY_GC_NURSERY_DEBUG`` diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -62,7 +62,7 @@ * go to pypy/tool/release and run ``force-builds.py `` The following JIT binaries should be built, however, we need more buildbots - windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel, + windows, linux-32, linux-64, osx64, armhf-raspberrian, armel, freebsd64 * wait for builds to complete, make sure there are no failures diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,8 @@ .. toctree:: + release-v5.10.1.rst + release-v5.10.0.rst release-v5.9.0.rst release-v5.8.0.rst release-v5.7.1.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-pypy2-5.10.0.rst whatsnew-pypy2-5.9.0.rst whatsnew-pypy2-5.8.0.rst whatsnew-pypy2-5.7.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -1,26 +1,41 @@ Potential Project List ====================== -Google Summer of Code 2017 --------------------------- +Getting involved +---------------- -PyPy is generally open to new ideas for Google Summer of Code. We are happy to accept good ideas around the PyPy ecosystem. If you need more information about the ideas we propose for this year please join us on irc, channel #pypy (freenode). If you are unsure, but still think that you can make a valuable contribution to PyPy, dont hesitate to contact us on #pypy or on our mailing list. - +We are happy to discuss ideas around the PyPy ecosystem. +If you are interested in palying with RPython or PyPy, or have a new idea not +mentioned here please join us on irc, channel #pypy (freenode). If you are unsure, +but still think that you can make a valuable contribution to PyPy, dont +hesitate to contact us on #pypy or on our mailing list. Here are some ideas +to get you thinking: * **Optimize PyPy Memory Usage**: Sometimes PyPy consumes more memory than CPython. - Two examples: 1) PyPy seems to allocate and keep alive more strings when importing a big Python modules. - 2) The base interpreter size (cold VM started from a console) of PyPy is bigger than the one of CPython. - The general procedure of this project is: Run both CPython and PyPy of the same Python version and - compare the memory usage (using Massif or other tools). + Two examples: 1) PyPy seems to allocate and keep alive more strings when + importing a big Python modules. 2) The base interpreter size (cold VM started + from a console) of PyPy is bigger than the one of CPython. The general + procedure of this project is: Run both CPython and PyPy of the same Python + version and compare the memory usage (using Massif or other tools). If PyPy consumes a lot more memory then find and resolve the issue. -* **VMProf + memory profiler**: vmprof by now has a memory profiler that can be used already. We want extend it with more features and resolve some current limitations. +* **VMProf + memory profiler**: vmprof is a statistical memory profiler. We + want extend it with new features and resolve some current limitations. -* **VMProf visualisations**: vmprof just shows a flame graph of the statistical profile and some more information about specific call sites. It would be very interesting to experiment with different information (such as memory, or even information generated by our jit compiler). +* **VMProf visualisations**: vmprof shows a flame graph of the statistical + profile and some more information about specific call sites. It would be + very interesting to experiment with different information (such as memory, + or even information generated by our jit compiler). -* **Explicit typing in RPython**: PyPy wants to have better ways to specify the signature and class attribute types in RPython. See more information about this topic below on this page. +* **Explicit typing in RPython**: PyPy wants to have better ways to specify + the signature and class attribute types in RPython. See more information + about this topic below on this page. -* **Virtual Reality (VR) visualisations for vmprof**: This is a very open topic with lots of freedom to explore data visualisation for profiles. No VR hardware would be needed for this project. Either universities provide such hardware or in any other case we potentially can lend the VR hardware setup. +* **Virtual Reality (VR) visualisations for vmprof**: This is a very open + topic with lots of freedom to explore data visualisation for profiles. No + VR hardware would be needed for this project. Either universities provide + such hardware or in any other case we potentially can lend the VR hardware + setup. Simple tasks for newcomers -------------------------- @@ -34,6 +49,11 @@ * Implement AF_XXX packet types of sockets: https://bitbucket.org/pypy/pypy/issue/1942/support-for-af_xxx-sockets +* Help with documentation. One task would be to document rpython configuration + options currently listed only on :doc:`this site ` also on the + RPython_ documentation site. + +.. _RPython: http://rpython.readthedocs.io Mid-to-large tasks ------------------ @@ -201,7 +221,9 @@ Introduce new benchmarks ------------------------ -We're usually happy to introduce new benchmarks. Please consult us +Our benchmark runner_ is showing its age. We should merge with the `CPython site`_ + +Additionally, we're usually happy to introduce new benchmarks. Please consult us before, but in general something that's real-world python code and is not already represented is welcome. We need at least a standalone script that can run without parameters. Example ideas (benchmarks need @@ -209,6 +231,8 @@ * `hg` +.. _runner: http://speed.pypy.org +.. _`CPython site`: https://speed.python.org/ ====================================== Make more python modules pypy-friendly @@ -238,15 +262,6 @@ using more pypy-friendly technologies, e.g. cffi. Here is a partial list of good work that needs to be finished: -**matplotlib** https://github.com/matplotlib/matplotlib - - Status: using the matplotlib branch of PyPy and the tkagg-cffi branch of - matplotlib from https://github.com/mattip/matplotlib/tree/tkagg-cffi, the - tkagg backend can function. - - TODO: the matplotlib branch passes numpy arrays by value (copying all the - data), this proof-of-concept needs help to become completely compliant - **wxPython** https://bitbucket.org/amauryfa/wxpython-cffi Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi diff --git a/pypy/doc/release-v5.10.0.rst b/pypy/doc/release-v5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.0.rst @@ -0,0 +1,100 @@ +====================================== +PyPy2.7 and PyPy3.5 v5.10 dual release +====================================== + +The PyPy team is proud to release both PyPy2.7 v5.10 (an interpreter supporting +Python 2.7 syntax), and a final PyPy3.5 v5.10 (an interpreter for Python +3.5 syntax). The two releases are both based on much the same codebase, thus +the dual release. + +This release is an incremental release with very few new features, the main +feature being the final PyPy3.5 release that works on linux and OS X with beta +windows support. It also includes fixes for `vmprof`_ cooperation with greenlets. + +Compared to 5.9, the 5.10 release contains mostly bugfixes and small improvements. +We have in the pipeline big new features coming for PyPy 6.0 that did not make +the release cut and should be available within the next couple months. + +As always, this release is 100% compatible with the previous one and fixed +several issues and bugs raised by the growing community of PyPy users. +As always, we strongly recommend updating. + +There are quite a few important changes that are in the pipeline that did not +make it into the 5.10 release. Most important are speed improvements to cpyext +(which will make numpy and pandas a bit faster) and utf8 branch that changes +internal representation of unicode to utf8, which should help especially the +Python 3.5 version of PyPy. + +This release concludes the Mozilla Open Source `grant`_ for having a compatible +PyPy 3.5 release and we're very grateful for that. Of course, we will continue +to improve PyPy 3.5 and probably move to 3.6 during the course of 2018. + +You can download the v5.10 releases here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. + +We would also like to thank our contributors and +encourage new people to join the project. PyPy has many +layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation +improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ +with making RPython's JIT even better. + +.. _vmprof: http://vmprof.readthedocs.io +.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly +.. _`help`: project-ideas.html + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +The PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +* improve ssl handling on windows for pypy3 (makes pip work) +* improve unicode handling in various error reporters +* fix vmprof cooperation with greenlets +* fix some things in cpyext +* test and document the cmp(nan, nan) == 0 behaviour +* don't crash when calling sleep with inf or nan +* fix bugs in _io module +* inspect.isbuiltin() now returns True for functions implemented in C +* allow the sequences future-import, docstring, future-import for CPython bug compatibility +* Issue #2699: non-ascii messages in warnings +* posix.lockf +* fixes for FreeBSD platform +* add .debug files, so builds contain debugging info, instead of being stripped +* improvements to cppyy +* issue #2677 copy pure c PyBuffer_{From,To}Contiguous from cpython +* issue #2682, split firstword on any whitespace in sqlite3 +* ctypes: allow ptr[0] = foo when ptr is a pointer to struct +* matplotlib will work with tkagg backend once `matplotlib pr #9356`_ is merged +* improvements to utf32 surrogate handling +* cffi version bump to 1.11.2 + +.. _`matplotlib pr #9356`: https://github.com/matplotlib/matplotlib/pull/9356 diff --git a/pypy/doc/release-v5.10.1.rst b/pypy/doc/release-v5.10.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-v5.10.1.rst @@ -0,0 +1,63 @@ +=========== +PyPy 5.10.1 +=========== + +We have released a bugfix PyPy3.5-v5.10.1 +due to the following issues: + + * Fix ``time.sleep(float('nan')`` which would hang on windows + + * Fix missing ``errno`` constants on windows + + * Fix issue 2718_ for the REPL on linux + + * Fix an overflow in converting 3 secs to nanosecs (issue 2717_ ) + + * Flag kwarg to ``os.setxattr`` had no effect + + * Fix the winreg module for unicode entries in the registry on windows + +Note that many of these fixes are for our new beta verison of PyPy3.5 on +windows. There may be more unicode problems in the windows beta version +especially around the subject of directory- and file-names with non-ascii +characters. + +Our downloads are available now. On macos, we recommend you wait for the +Homebrew_ package. + +Thanks to those who reported the issues. + +.. _2718: https://bitbucket.org/pypy/pypy/issues/2718 +.. _2717: https://bitbucket.org/pypy/pypy/issues/2717 +.. _Homebrew: http://brewformulas.org/Pypy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison) +due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy 3.5 release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux, + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + +.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org +.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html + +Please update, and continue to help us make PyPy better. + +Cheers + +The PyPy Team + diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -81,6 +81,7 @@ 'Yasir Suhail':['yasirs'], 'Squeaky': ['squeaky'], "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'], + "Dodan Mihai": ['mihai.dodan at gmail.com'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,43 +2,58 @@ What's new in PyPy2.7 5.10+ =========================== -.. this is a revision shortly after release-pypy2.7-v5.9.0 -.. startrev:d56dadcef996 +.. this is a revision shortly after release-pypy2.7-v5.10.0 +.. startrev: 6b024edd9d12 +.. branch: cpyext-avoid-roundtrip -.. branch: cppyy-packaging +Big refactoring of some cpyext code, which avoids a lot of nonsense when +calling C from Python and vice-versa: the result is a big speedup in +function/method calls, up to 6 times faster. -Cleanup and improve cppyy packaging +.. branch: cpyext-datetime2 -.. branch: docs-osx-brew-openssl +Support ``tzinfo`` field on C-API datetime objects, fixes latest pandas HEAD -.. branch: keep-debug-symbols -Add a smartstrip tool, which can optionally keep the debug symbols in a -separate file, instead of just stripping them away. Use it in packaging +.. branch: mapdict-size-limit -.. branch: bsd-patches +Fix a corner case of mapdict: When an instance is used like a dict (using +``setattr`` and ``getattr``, or ``.__dict__``) and a lot of attributes are +added, then the performance using mapdict is linear in the number of +attributes. This is now fixed (by switching to a regular dict after 80 +attributes). -Fix failures on FreeBSD, contributed by David Naylor as patches on the issue -tracker (issues 2694, 2695, 2696, 2697) -.. branch: run-extra-tests +.. branch: cpyext-faster-arg-passing -Run extra_tests/ in buildbot +When using cpyext, improve the speed of passing certain objects from PyPy to C +code, most notably None, True, False, types, all instances of C-defined types. +Before, a dict lookup was needed every time such an object crossed over, now it +is just a field read. -.. branch: vmprof-0.4.10 -Upgrade the _vmprof backend to vmprof 0.4.10 +.. branch: 2634_datetime_timedelta_performance -.. branch: fix-vmprof-stacklet-switch +Improve datetime + timedelta performance. -Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) +.. branch: memory-accounting -.. branch: win32-vcvars +Improve way to describe memory -.. branch rdict-fast-hash +.. branch: msvc14 -Make it possible to declare that the hash function of an r_dict is fast in RPython. +Allow compilaiton with Visual Studio 2017 compiler suite on windows + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. .. branch: unicode-utf8-re .. branch: utf8-io diff --git a/pypy/doc/whatsnew-pypy2-5.10.0.rst b/pypy/doc/whatsnew-pypy2-5.10.0.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-5.10.0.rst @@ -0,0 +1,46 @@ +========================== +What's new in PyPy2.7 5.10 +========================== + +.. this is a revision shortly after release-pypy2.7-v5.9.0 +.. startrev:d56dadcef996 + + +.. branch: cppyy-packaging + +Cleanup and improve cppyy packaging + +.. branch: docs-osx-brew-openssl + +.. branch: keep-debug-symbols + +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches + +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests + +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 + +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +.. branch: fix-vmprof-stacklet-switch-2 +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + +.. branch: rdict-fast-hash + +Make it possible to declare that the hash function of an r_dict is fast in RPython. + +.. branch: unicode-utf8-re +.. branch: utf8-io +Utf8 handling for unicode + diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -39,10 +39,24 @@ .. _Microsoft Visual C++ Compiler for Python 2.7: https://www.microsoft.com/en-us/download/details.aspx?id=44266 +Installing "Build Tools for Visual Studio 2017" (for Python 3) +-------------------------------------------------------------- + +As documented in the CPython Wiki_, CPython now recommends Visual C++ version +14.0. A compact version of the compiler suite can be obtained from Microsoft_ +downloads, search the page for "Build Tools for Visual Studio 2017". + +You will also need to install the the `Windows SDK`_ in order to use the +`mt.exe` mainfest compiler. + +.. _Wiki: https://wiki.python.org/moin/WindowsCompilers +.. _Microsoft: https://www.visualstudio.com/downloads +.. _`Windows SDK`: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk + Translating PyPy with Visual Studio ----------------------------------- -We routinely test translation using v9, also known as Visual Studio 2008. +We routinely test translation of PyPy 2.7 using v9 and PyPy 3 with vc14. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -82,8 +96,8 @@ .. _build instructions: http://pypy.org/download.html#building-from-source -Setting Up Visual Studio for building SSL in Python3 ----------------------------------------------------- +Setting Up Visual Studio 9.0 for building SSL in Python3 +-------------------------------------------------------- On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after translation. However ``distutils`` does not support the Micorosft-provided Visual C @@ -132,243 +146,14 @@ Installing external packages ---------------------------- -On Windows, there is no standard place where to download, build and -install third-party libraries. We recommend installing them in the parent -directory of the pypy checkout. For example, if you installed pypy in -``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You must then set the -INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. +We uses a `repository` parallel to pypy to hold binary compiled versions of the +build dependencies for windows. As part of the `rpython` setup stage, environment +variables will be set to use these dependencies. The repository has a README +file on how to replicate, and a branch for each supported platform. You may run + the `get_externals.py` utility to checkout the proper branch for your platform +and PyPy version. - -Abridged method (using Visual Studio 2008) ------------------------------------------- - -Download the versions of all the external packages from -https://bitbucket.org/pypy/pypy/downloads/local_59.zip -(for post-5.8 builds) with sha256 checksum -``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911`` -https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip -(to reproduce 5.8 builds) with sha256 checksum -``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or -https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip -(for 2.4 release and later) or -https://bitbucket.org/pypy/pypy/downloads/local.zip -(for pre-2.4 versions) -Then expand it into the base directory (base_dir) and modify your environment -to reflect this:: - - set PATH=\bin;%PATH% - set INCLUDE=\include;%INCLUDE% - set LIB=\lib;%LIB% - -Now you should be good to go. If you choose this method, you do not need -to download/build anything else. - -Nonabridged method (building from scratch) ------------------------------------------- - -If you want to, you can rebuild everything from scratch by continuing. - - -The Boehm garbage collector -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This library is needed if you plan to use the ``--gc=boehm`` translation -option (this is the default at some optimization levels like ``-O1``, -but unneeded for high-performance translations like ``-O2``). -You may get it at -http://hboehm.info/gc/gc_source/gc-7.1.tar.gz - -Versions 7.0 and 7.1 are known to work; the 6.x series won't work with -RPython. Unpack this folder in the base directory. -The default GC_abort(...) function in misc.c will try to open a MessageBox. -You may want to disable this with the following patch:: - - --- a/misc.c Sun Apr 20 14:08:27 2014 +0300 - +++ b/misc.c Sun Apr 20 14:08:37 2014 +0300 - @@ -1058,7 +1058,7 @@ - #ifndef PCR - void GC_abort(const char *msg) - { - -# if defined(MSWIN32) - +# if 0 && defined(MSWIN32) - (void) MessageBoxA(NULL, msg, "Fatal error in gc", MB_ICONERROR|MB_OK); - # else - GC_err_printf("%s\n", msg); - -Then open a command prompt:: - - cd gc-7.1 - nmake -f NT_THREADS_MAKEFILE - copy Release\gc.dll - - -The zlib compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in -the base directory. Then compile:: - - cd zlib-1.2.11 - nmake -f win32\Makefile.msc - copy zlib.lib - copy zlib.h zconf.h - copy zlib1.dll # (needed for tests via ll2ctypes) - - -The bz2 compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Get the same version of bz2 used by python and compile as a static library:: - - svn export http://svn.python.org/projects/external/bzip2-1.0.6 - cd bzip2-1.0.6 - nmake -f makefile.msc - copy libbz2.lib - copy bzlib.h - - -The sqlite3 database library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi -wrapper is compiled when the module is imported for the first time. -The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility. - - -The expat XML parser -~~~~~~~~~~~~~~~~~~~~ - -CPython compiles expat from source as part of the build. PyPy uses the same -code base, but expects to link to a static lib of expat. Here are instructions -to reproduce the static lib in version 2.2.4. - -Download the source code of expat: https://github.com/libexpat/libexpat. -``git checkout`` the proper tag, in this case ``R_2_2_4``. Run -``vcvars.bat`` to set up the visual compiler tools, and CD into the source -directory. Create a file ``stdbool.h`` with the content - -.. code-block:: c - - #pragma once - - #define false 0 - #define true 1 - - #define bool int - -and put it in a place on the ``INCLUDE`` path, or create it in the local -directory and add ``.`` to the ``INCLUDE`` path:: - - SET INCLUDE=%INCLUDE%;. - -Then compile all the ``*.c`` file into ``*.obj``:: - - cl.exe /nologo /MD /O2 *c /c - rem for debug - cl.exe /nologo /MD /O0 /Ob0 /Zi *c /c - -You may need to move some variable declarations to the beginning of the -function, to be compliant with C89 standard. Here is the diff for version 2.2.4 - -.. code-block:: diff - - diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c - index 007aed0..a2dcaad 100644 - --- a/expat/lib/xmltok.c - +++ b/expat/lib/xmltok.c - @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc), - /* Avoid copying partial characters (due to limited space). */ - const ptrdiff_t bytesAvailable = fromLim - *fromP; - const ptrdiff_t bytesStorable = toLim - *toP; - + const char * fromLimBefore; - + ptrdiff_t bytesToCopy; - if (bytesAvailable > bytesStorable) { - fromLim = *fromP + bytesStorable; - output_exhausted = true; - } - - /* Avoid copying partial characters (from incomplete input). */ - - const char * const fromLimBefore = fromLim; - + fromLimBefore = fromLim; - align_limit_to_full_utf8_characters(*fromP, &fromLim); - if (fromLim < fromLimBefore) { - input_incomplete = true; - } - - - const ptrdiff_t bytesToCopy = fromLim - *fromP; - + bytesToCopy = fromLim - *fromP; - memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy); - *fromP += bytesToCopy; - *toP += bytesToCopy; - - -Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests):: - - cl /LD *.obj libexpat.def /Felibexpat.dll - rem for debug - rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll - - rem this will override the export library created in the step above - rem but tests do not need the export library, they load the dll dynamically - lib *.obj /out:libexpat.lib - -Then, copy - -- ``libexpat.lib`` into LIB -- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE -- ``libexpat.dll`` into PATH - - -The OpenSSL library -~~~~~~~~~~~~~~~~~~~ - -OpenSSL needs a Perl interpreter to configure its makefile. You may -use the one distributed by ActiveState, or the one from cygwin.:: - - svn export http://svn.python.org/projects/external/openssl-1.0.2k - cd openssl-1.0.2k - perl Configure VC-WIN32 no-idea no-mdc2 - ms\do_ms.bat - nmake -f ms\nt.mak install - copy out32\*.lib - xcopy /S include\openssl - -For tests you will also need the dlls:: - nmake -f ms\ntdll.mak install - copy out32dll\*.dll - -TkInter module support -~~~~~~~~~~~~~~~~~~~~~~ - -Note that much of this is taken from the cpython build process. -Tkinter is imported via cffi, so the module is optional. To recreate the tcltk -directory found for the release script, create the dlls, libs, headers and -runtime by running:: - - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85 - svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85 - cd tcl85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all - nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install - cd ..\..\tk85\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install - copy ..\..\tcltk\bin\* - copy ..\..\tcltk\lib\*.lib - xcopy /S ..\..\tcltk\include - -The lzma compression library -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Python 3.3 ship with CFFI wrappers for the lzma library, which can be -downloaded from this site http://tukaani.org/xz. Python 3.3-3.5 use version -5.0.5, a prebuilt version can be downloaded from -http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature -http://tukaani.org/xz/xz-5.0.5-windows.zip.sig - -Then copy the headers to the include directory, rename ``liblzma.a`` to -``lzma.lib`` and copy it to the lib directory - +.. _repository: https://bitbucket.org/pypy/external Using the mingw compiler ------------------------ diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -1,7 +1,7 @@ """Python control flow graph generation and bytecode assembly.""" +import math import os -from rpython.rlib import rfloat from rpython.rlib.objectmodel import we_are_translated from pypy.interpreter.astcompiler import ast, misc, symtable @@ -266,7 +266,7 @@ w_type = space.type(obj) if space.is_w(w_type, space.w_float): val = space.float_w(obj) - if val == 0.0 and rfloat.copysign(1., val) < 0: + if val == 0.0 and math.copysign(1., val) < 0: w_key = space.newtuple([obj, space.w_float, space.w_None]) else: w_key = space.newtuple([obj, space.w_float]) @@ -276,9 +276,9 @@ real = space.float_w(w_real) imag = space.float_w(w_imag) real_negzero = (real == 0.0 and - rfloat.copysign(1., real) < 0) + math.copysign(1., real) < 0) imag_negzero = (imag == 0.0 and - rfloat.copysign(1., imag) < 0) + math.copysign(1., imag) < 0) if real_negzero and imag_negzero: tup = [obj, space.w_complex, space.w_None, space.w_None, space.w_None] diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -1096,17 +1096,21 @@ encoding = self.compile_info.encoding flags = self.compile_info.flags unicode_literals = flags & consts.CO_FUTURE_UNICODE_LITERALS - try: - sub_strings_w = [parsestring.parsestr(space, encoding, atom_node.get_child(i).get_value(), - unicode_literals) - for i in range(atom_node.num_children())] - except error.OperationError as e: - if not e.match(space, space.w_UnicodeError): - raise - # UnicodeError in literal: turn into SyntaxError - e.normalize_exception(space) - errmsg = space.text_w(space.str(e.get_w_value(space))) - raise self.error('(unicode error) %s' % errmsg, atom_node) + sub_strings_w = [] + for index in range(atom_node.num_children()): + child = atom_node.get_child(index) + try: + sub_strings_w.append(parsestring.parsestr(space, encoding, child.get_value(), From pypy.commits at gmail.com Sun Mar 18 09:05:40 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 06:05:40 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: fix from merge, add mcbs variants? (mattip, ronan) Message-ID: <5aae6424.12881c0a.477a2.1167@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r93987:d5f6267bf47b Date: 2018-03-18 14:04 +0100 http://bitbucket.org/pypy/pypy/changeset/d5f6267bf47b/ Log: fix from merge, add mcbs variants? (mattip, ronan) diff --git a/pypy/interpreter/test/test_unicodehelper.py b/pypy/interpreter/test/test_unicodehelper.py --- a/pypy/interpreter/test/test_unicodehelper.py +++ b/pypy/interpreter/test/test_unicodehelper.py @@ -2,8 +2,6 @@ from hypothesis import given, strategies import struct import sys -from pypy.interpreter.unicodehelper import ( - encode_utf8, decode_utf8, unicode_encode_utf_32_be) from rpython.rlib import rutf8 diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -221,6 +221,20 @@ pos = rutf8._pos_at_index(s, newindex) return result.build() +if sys.platform == 'win32': + def utf8_encode_mbcs(s, errors, errorhandler): + from rpython.rlib import runicode + slen = len(s) + res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler) + return res.decode('utf8') + + def str_decode_mbcs(s, errors, errorhandler): + from rpython.rlib import runicode + slen = len(s) + res, size = runicode.str_decode_mbcs(s, slen, errors=errors, + errorhandler=errorhandler) + return res.encode('utf8'), size, len(res) + def str_decode_utf8(s, errors, final, errorhandler): """ Same as checking for the valid utf8, but we know the utf8 is not valid so we're trying to either raise or pack stuff with error handler. From pypy.commits at gmail.com Sun Mar 18 11:01:21 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 08:01:21 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: unicode_w -> utf8_w Message-ID: <5aae7f41.9d4a1c0a.d5c85.efaf@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r93988:c7408ca6f535 Date: 2018-03-18 16:00 +0100 http://bitbucket.org/pypy/pypy/changeset/c7408ca6f535/ Log: unicode_w -> utf8_w diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -214,9 +214,9 @@ PyObject_Cmp(space, w(u"\xe9"), w("\xe9"), ptr) def test_unicode(self, space, api): - assert space.unicode_w(api.PyObject_Unicode(None)) == u"" - assert space.unicode_w(api.PyObject_Unicode(space.wrap([]))) == u"[]" - assert space.unicode_w(api.PyObject_Unicode(space.wrap("e"))) == u"e" + assert space.utf8_w(api.PyObject_Unicode(None)) == u"".encode('utf-8') + assert space.utf8_w(api.PyObject_Unicode(space.wrap([]))) == u"[]".encode('utf-8') + assert space.utf8_w(api.PyObject_Unicode(space.wrap("e"))) == u"e".encode('utf-8') with raises_w(space, UnicodeDecodeError): PyObject_Unicode(space, space.wrap("\xe9")) From pypy.commits at gmail.com Sun Mar 18 11:01:48 2018 From: pypy.commits at gmail.com (Raemi) Date: Sun, 18 Mar 2018 08:01:48 -0700 (PDT) Subject: [pypy-commit] stmgc c8-reshare-pages: add some comments Message-ID: <5aae7f5c.89981c0a.32845.fff5@mx.google.com> Author: Remi Meier Branch: c8-reshare-pages Changeset: r2154:9e9180b5bdf3 Date: 2018-03-18 16:01 +0100 http://bitbucket.org/pypy/stmgc/changeset/9e9180b5bdf3/ Log: add some comments diff --git a/c8/TODO b/c8/TODO --- a/c8/TODO +++ b/c8/TODO @@ -1,3 +1,10 @@ +- MOVNTI for non-temporal 32bit store (for read markers) +- do we actually know if paying the cost of getting the current read-version + in every stm_read is lower than paying the cost of zeroing the readmarker + on every commit (and just use a flag)? + +- stm_enable_atomic() simply sets a huge value for nursery_mark, which is + unreliable in theory - investigate if userfaultfd() helps: http://kernelnewbies.org/Linux_4.3#head-3deefea7b0add8c1b171b0e72ce3b69c5ed35cb0 diff --git a/c8/doc/page-resharing.md b/c8/doc/page-resharing.md --- a/c8/doc/page-resharing.md +++ b/c8/doc/page-resharing.md @@ -51,14 +51,14 @@ In signal handler: - if read or write: + if is_read or is_write: if is `RO`: `RO -> ACC` (and `RO -> NOACC` for all others) else if is `NOACC`: if !is_write and noone has `ACC`: `NOACC -> RO` else: - `NOACC -> ACC` + `NOACC -> ACC` (and `RO -> NOACC` for all others) On validate: always imports into `ACC`, into `RO` would be a bug. @@ -66,6 +66,10 @@ 1. Validation of seg0: gets all changes; any `RO` views still around means that there was *no change* in those pages, so the views stay valid. + + XXX: what about the modifications that major GC makes during tracing? how + does it affect the page-sharing in the kernel? + 2. All other segments validate their `ACC` pages; again `RO` pages *cannot* have changes that need importing. 3. While tracing modified objs and overflow objs, remember pages with diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -235,6 +235,7 @@ cl = first_cl; while ((cl = cl->next) != NULL) { if (!needs_abort) { + /* check if there is a conflict: */ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; for (; undo < end; undo++) { @@ -301,6 +302,7 @@ } if (cl->written_count) { + /* copy most recent version of modified objs to our segment: */ struct stm_undo_s *undo = cl->written; struct stm_undo_s *end = cl->written + cl->written_count; @@ -314,6 +316,13 @@ copy_bk_objs_in_page_from (cl->segment_num, -1, /* any page */ !needs_abort); /* if we abort, we still want to copy everything */ + + /* reason we must always update to the last (non-INEV) + * commit log entry: a currently running transaction in + * segment_num may have backup copies that revert the + * objects in cl->written to a more current revision than + * the cl-entry represents. This is fine as long as we + * *also* validate to that more current revision. */ } dprintf(("_stm_validate() to cl=%p, rev=%lu\n", cl, cl->rev_num)); From pypy.commits at gmail.com Sun Mar 18 11:11:56 2018 From: pypy.commits at gmail.com (ariava) Date: Sun, 18 Mar 2018 08:11:56 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: Cleanup rpython/annotator/signature.py Message-ID: <5aae81bc.5c471c0a.717e4.4203@mx.google.com> Author: Arianna Avanzini Branch: rpython-sprint Changeset: r93989:294fd4ce722f Date: 2018-03-18 14:47 +0100 http://bitbucket.org/pypy/pypy/changeset/294fd4ce722f/ Log: Cleanup rpython/annotator/signature.py - Use 'isinstance()' instead of 'is' in _annotation_key() to account for user-defined subclasses of list and dict. - Cleanup assert messages when validating annotations and convert them to exceptions. - Remove intermediate variables used just once before return. - Cleanup comments. diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,37 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t + if len(t) == 0: + raise ValueError("Cannot handle empty %s in args enforcing", type(t)) + if isinstance(t, tuple) or len(t) == 1: # we accept tuples with len > 1 because + return t # tuple items are all of same type + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,13 +97,12 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: raise AssertionError("annotationoftype(%r)" % (t,)) From pypy.commits at gmail.com Sun Mar 18 12:19:04 2018 From: pypy.commits at gmail.com (arigo) Date: Sun, 18 Mar 2018 09:19:04 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: (ronan, arigo) Message-ID: <5aae9178.9585df0a.96fe.41ab@mx.google.com> Author: Armin Rigo Branch: unicode-utf8 Changeset: r93990:52a945178f9b Date: 2018-03-18 17:18 +0100 http://bitbucket.org/pypy/pypy/changeset/52a945178f9b/ Log: (ronan, arigo) Attempt to fix the merge for module/_sre diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -156,18 +156,21 @@ return rsre_core.BufMatchContext(self.code, buf, pos, endpos, self.flags) - def fresh_copy(self, ctx, start): + def fresh_copy(self, ctx): if isinstance(ctx, rsre_utf8.Utf8MatchContext): result = rsre_utf8.Utf8MatchContext( - ctx.pattern, ctx._utf8, start, ctx.end, ctx.flags) + ctx.pattern, ctx._utf8, ctx.match_start, ctx.end, ctx.flags) result.w_unicode_obj = ctx.w_unicode_obj - return result - if isinstance(ctx, rsre_core.StrMatchContext): - return self._make_str_match_context(ctx._string, start, ctx.end) - if isinstance(ctx, rsre_core.BufMatchContext): - return rsre_core.BufMatchContext( - ctx.pattern, ctx._buffer, start, ctx.end, ctx.flags) - raise AssertionError("bad ctx type") + elif isinstance(ctx, rsre_core.StrMatchContext): + result = self._make_str_match_context( + ctx._string, ctx.match_start, ctx.end) + elif isinstance(ctx, rsre_core.BufMatchContext): + result = rsre_core.BufMatchContext( + ctx.pattern, ctx._buffer, ctx.match_start, ctx.end, ctx.flags) + else: + raise AssertionError("bad ctx type") + result.match_end = ctx.match_end + return result def _make_str_match_context(self, str, pos, endpos): # for tests to override @@ -343,7 +346,7 @@ if filter_is_callable: w_match = self.getmatch(ctx, True) # make a copy of 'ctx'; see test_sub_matches_stay_valid - ctx = ctx.fresh_copy(start) # match_start/match_end dropped + ctx = self.fresh_copy(ctx) w_piece = space.call_function(w_filter, w_match) if not space.is_w(w_piece, space.w_None): assert strbuilder is None @@ -720,7 +723,8 @@ if exhausted: self.ctx = None else: - self.ctx = self.srepat.fresh_copy(ctx, nextstart) + self.ctx = self.srepat.fresh_copy(ctx) + self.ctx.match_start = nextstart match = W_SRE_Match(self.srepat, ctx) return match else: diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -92,8 +92,6 @@ class AbstractMatchContext(object): """Abstract base class""" _immutable_fields_ = ['pattern[*]', 'flags', 'end'] - match_start = 0 - match_end = 0 match_marks = None match_marks_flat = None fullmatch_only = False @@ -105,6 +103,7 @@ check_nonneg(end) self.pattern = pattern self.match_start = match_start + self.match_end = self.ZERO self.end = end self.flags = flags # check we don't get the old value of MAXREPEAT @@ -309,9 +308,6 @@ def get_single_byte(self, base_position, index): return self.str(base_position + index) - def fresh_copy(self, start): - return UnicodeMatchContext(self.pattern, self._unicodestr, start, - self.end, self.flags) # ____________________________________________________________ diff --git a/rpython/rlib/rsre/test/support.py b/rpython/rlib/rsre/test/support.py --- a/rpython/rlib/rsre/test/support.py +++ b/rpython/rlib/rsre/test/support.py @@ -106,10 +106,6 @@ assert isinstance(index, int) return Position(base_position._p + index) - def fresh_copy(self, start): - return MatchContextForTests(self.pattern, self._string, start, - self.end, self.flags) - def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False): start, end = _adjust(start, end, len(string)) From pypy.commits at gmail.com Sun Mar 18 13:29:05 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 10:29:05 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: fix winreg for utf8 on win32 Message-ID: <5aaea1e1.98bf1c0a.64927.de9c@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r93991:341ddb374260 Date: 2018-03-18 18:28 +0100 http://bitbucket.org/pypy/pypy/changeset/341ddb374260/ Log: fix winreg for utf8 on win32 diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -226,12 +226,12 @@ from rpython.rlib import runicode slen = len(s) res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler) - return res.decode('utf8') + return res - def str_decode_mbcs(s, errors, errorhandler): + def str_decode_mbcs(s, errors, final, errorhandler): from rpython.rlib import runicode slen = len(s) - res, size = runicode.str_decode_mbcs(s, slen, errors=errors, + res, size = runicode.str_decode_mbcs(s, slen, final=final, errors=errors, errorhandler=errorhandler) return res.encode('utf8'), size, len(res) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -697,11 +697,12 @@ raiseWindowsError(space, ret, 'RegConnectRegistry') return W_HKEY(space, rethkey[0]) - at unwrap_spec(source=unicode) -def ExpandEnvironmentStrings(space, source): +def ExpandEnvironmentStrings(space, w_source): "string = ExpandEnvironmentStrings(string) - Expand environment vars." try: - return space.newunicode(rwinreg.ExpandEnvironmentStrings(source)) + source, source_ulen = space.utf8_len_w(w_source) + res, res_ulen = rwinreg.ExpandEnvironmentStrings(source, source_ulen) + return space.newutf8(res, res_ulen) except WindowsError as e: raise wrap_windowserror(space, e) diff --git a/rpython/rlib/rwinreg.py b/rpython/rlib/rwinreg.py --- a/rpython/rlib/rwinreg.py +++ b/rpython/rlib/rwinreg.py @@ -165,8 +165,8 @@ rwin32.DWORD, save_err=rffi.RFFI_SAVE_LASTERROR) -def ExpandEnvironmentStrings(source): - with rffi.scoped_unicode2wcharp(source) as src_buf: +def ExpandEnvironmentStrings(source, unicode_len): + with rffi.scoped_utf82wcharp(source, unicode_len) as src_buf: size = _ExpandEnvironmentStringsW(src_buf, lltype.nullptr(rffi.CWCHARP.TO), 0) if size == 0: @@ -176,4 +176,5 @@ if _ExpandEnvironmentStringsW(src_buf, dest_buf.raw, size) == 0: raise rwin32.lastSavedWindowsError("ExpandEnvironmentStrings") - return dest_buf.str(size - 1) # remove trailing \0 + res = dest_buf.str(size-1) # remove trailing \0 + return res.encode('utf8'), len(res) diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -1278,6 +1278,18 @@ if self.buf: free_wcharp(self.buf) +class scoped_utf82wcharp: + def __init__(self, value, unicode_len): + if value is not None: + self.buf = utf82wcharp(value, unicode_len) + else: + self.buf = lltype.nullptr(CWCHARP.TO) + def __enter__(self): + return self.buf + def __exit__(self, *args): + if self.buf: + free_wcharp(self.buf) + class scoped_nonmovingbuffer: @@ -1316,7 +1328,7 @@ def __init__(self, data): self.data = data def __enter__(self): - self.buf, self.flag = get_nonmoving_unicodebuffer(self.data) + self.buf, self.flag = get_nonmoving_unicodebuffer(self.data.decode('utf-8')) return self.buf def __exit__(self, *args): free_nonmoving_unicodebuffer(self.data, self.buf, self.flag) From pypy.commits at gmail.com Sun Mar 18 13:49:45 2018 From: pypy.commits at gmail.com (ariava) Date: Sun, 18 Mar 2018 10:49:45 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: Remove check added in last commit. Message-ID: <5aaea6b9.88d31c0a.91512.2dde@mx.google.com> Author: Arianna Avanzini Branch: rpython-sprint Changeset: r93992:737e0369f12e Date: 2018-03-18 18:48 +0100 http://bitbucket.org/pypy/pypy/changeset/737e0369f12e/ Log: Remove check added in last commit. Check was about disallowing annotation of iterable objects with zero length. Annotation is also used by marshalling, where having zero length iterables is accepted. diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -43,12 +43,11 @@ try: _ = iter(t) except TypeError: # if it's not an iterable, just return - return t - if len(t) == 0: - raise ValueError("Cannot handle empty %s in args enforcing", type(t)) - if isinstance(t, tuple) or len(t) == 1: # we accept tuples with len > 1 because - return t # tuple items are all of same type - raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) def _compute_annotation(t, bookkeeper=None): From pypy.commits at gmail.com Sun Mar 18 18:12:10 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 15:12:10 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: revert unintended commit in 341ddb374260 Message-ID: <5aaee43a.c7141c0a.d33c5.fda4@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r93993:f8aaef6e3548 Date: 2018-03-18 23:11 +0100 http://bitbucket.org/pypy/pypy/changeset/f8aaef6e3548/ Log: revert unintended commit in 341ddb374260 diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -1328,7 +1328,7 @@ def __init__(self, data): self.data = data def __enter__(self): - self.buf, self.flag = get_nonmoving_unicodebuffer(self.data.decode('utf-8')) + self.buf, self.flag = get_nonmoving_unicodebuffer(self.data) return self.buf def __exit__(self, *args): free_nonmoving_unicodebuffer(self.data, self.buf, self.flag) From pypy.commits at gmail.com Sun Mar 18 19:10:22 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 16:10:22 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: document and close branch to be merged Message-ID: <5aaef1de.9585df0a.96fe.bad8@mx.google.com> Author: Matti Picus Branch: rpython-sprint Changeset: r93995:137001874b39 Date: 2018-03-19 00:08 +0100 http://bitbucket.org/pypy/pypy/changeset/137001874b39/ Log: document and close branch to be merged diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -54,3 +54,7 @@ Speed up branchy code that does a lot of function inlining by saving one call to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures From pypy.commits at gmail.com Sun Mar 18 19:10:24 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 16:10:24 -0700 (PDT) Subject: [pypy-commit] pypy default: merge rpython-sprint which refactors rpython signature Message-ID: <5aaef1e0.0e0b1c0a.27f36.319f@mx.google.com> Author: Matti Picus Branch: Changeset: r93996:17ba5f6cbab7 Date: 2018-03-19 00:09 +0100 http://bitbucket.org/pypy/pypy/changeset/17ba5f6cbab7/ Log: merge rpython-sprint which refactors rpython signature diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -54,3 +54,7 @@ Speed up branchy code that does a lot of function inlining by saving one call to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,13 +96,12 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: raise AssertionError("annotationoftype(%r)" % (t,)) From pypy.commits at gmail.com Sun Mar 18 19:10:20 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 18 Mar 2018 16:10:20 -0700 (PDT) Subject: [pypy-commit] pypy default: use assume so the test passes rather than skips Message-ID: <5aaef1dc.92181c0a.b6096.1bb3@mx.google.com> Author: Matti Picus Branch: Changeset: r93994:585d5c0fb2ff Date: 2018-03-19 00:04 +0100 http://bitbucket.org/pypy/pypy/changeset/585d5c0fb2ff/ Log: use assume so the test passes rather than skips diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,7 +1,7 @@ import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -41,11 +41,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 - at example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): - if s == u'\ufacf' and sys.maxunicode == 65535: - pytest.skip('chr(0xfacf) normalizes to chr(0x2284a), which is too big') + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) From pypy.commits at gmail.com Mon Mar 19 05:00:22 2018 From: pypy.commits at gmail.com (ariava) Date: Mon, 19 Mar 2018 02:00:22 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: Spellchecking and cleanup. Message-ID: <5aaf7c26.1dbf1c0a.f0fd3.004c@mx.google.com> Author: Arianna Avanzini Branch: rpython-sprint Changeset: r93997:5f1376bbe863 Date: 2018-03-18 19:09 +0100 http://bitbucket.org/pypy/pypy/changeset/5f1376bbe863/ Log: Spellchecking and cleanup. - Fix spellchecking nits in few comments and in translation.rst - Fix TypeError message in rpython/annotator/signature.py diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -103,7 +103,7 @@ elif bookkeeper and not hasattr(t, '_freeze_'): return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: - raise AssertionError("annotationoftype(%r)" % (t,)) + raise TypeError("Annotation of type %r not supported" % (t,)) class Sig(object): diff --git a/rpython/doc/translation.rst b/rpython/doc/translation.rst --- a/rpython/doc/translation.rst +++ b/rpython/doc/translation.rst @@ -48,7 +48,7 @@ be present in memory as a form that is "static enough" in the sense of :doc:`RPython `. -2. The Annotator_ performs a global analysis starting from an specified +2. The Annotator_ performs a global analysis starting from a specified entry point to deduce type and other information about what each variable can contain at run-time, :ref:`building flow graphs ` as it encounters them. diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -120,7 +120,7 @@ """ Decorate a function with forcing of RPython-level types on arguments. None means no enforcing. - When not translated, the type of the actual arguments are checked against + When not translated, the type of the actual arguments is checked against the enforced types every time the function is called. You can disable the typechecking by passing ``typecheck=False`` to @enforceargs. """ @@ -147,8 +147,7 @@ # they are already homogeneous, so we only check the first # item. The case of empty list/dict is handled inside typecheck() if isinstance(arg, list): - item = arg[0] - return [get_type_descr_of_argument(item)] + return [get_type_descr_of_argument(arg[0])] elif isinstance(arg, dict): key, value = next(arg.iteritems()) return {get_type_descr_of_argument(key): get_type_descr_of_argument(value)} From pypy.commits at gmail.com Mon Mar 19 05:00:25 2018 From: pypy.commits at gmail.com (ariava) Date: Mon, 19 Mar 2018 02:00:25 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: Document enforceargs() decorator on RPython docs. Message-ID: <5aaf7c29.85591c0a.5d6e0.1d9c@mx.google.com> Author: Arianna Avanzini Branch: rpython-sprint Changeset: r93998:76a4a4c1f521 Date: 2018-03-19 09:57 +0100 http://bitbucket.org/pypy/pypy/changeset/76a4a4c1f521/ Log: Document enforceargs() decorator on RPython docs. diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -259,6 +259,26 @@ intmask(). +Type Enforcing and Checking +--------------------------- + +RPython provides a helper decorator to force RPython-level types on function +arguments. The decorator, called `enforceargs()`, accepts as parameters the +types expected to match the arguments of the function. + +Functions decorated with `enforceargs()` have their function signature analyzed +and their RPython-level type inferred at import time (for further details about +the flavor of translation performed in RPython, see the +`Annotation pass documentation`_). Encountering types not supported by RPython +will raise a `TypeError`. + +`enforceargs()` by default also performs type checking of parameter types each +time the function is invoked. To disable this behavior, it's possible to +pass the `typecheck=False` parameter to the decorator. + +.. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator + + Exception rules --------------- From pypy.commits at gmail.com Mon Mar 19 07:25:33 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Mar 2018 04:25:33 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: try to debug buildslave failure to find cl.exe Message-ID: <5aaf9e2d.4192df0a.abd3c.adde@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r93999:226287429316 Date: 2018-03-19 12:22 +0100 http://bitbucket.org/pypy/pypy/changeset/226287429316/ Log: try to debug buildslave failure to find cl.exe diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -295,6 +295,7 @@ if len(result) != len(interesting): raise ValueError(str(list(result.keys()))) + log.debug('Got', result) return result # More globals From pypy.commits at gmail.com Mon Mar 19 08:17:04 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Mar 2018 05:17:04 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: remove remaining space.newunicode, continue fix in f8aaef6e3548, fix translation Message-ID: <5aafaa40.78c3df0a.29faf.5fb7@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r94000:1ea028ef8faa Date: 2018-03-19 13:16 +0100 http://bitbucket.org/pypy/pypy/changeset/1ea028ef8faa/ Log: remove remaining space.newunicode, continue fix in f8aaef6e3548, fix translation diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1783,9 +1783,8 @@ def unicode0_w(self, w_obj): "Like unicode_w, but rejects strings with NUL bytes." - xxxx from rpython.rlib import rstring - result = w_obj.unicode_w(self) + result = w_obj.utf8_w(self).decode('utf8') if u'\x00' in result: raise oefmt(self.w_TypeError, "argument must be a unicode string without NUL " diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -224,6 +224,7 @@ if sys.platform == 'win32': def utf8_encode_mbcs(s, errors, errorhandler): from rpython.rlib import runicode + s = s.decode('utf-8') slen = len(s) res = runicode.unicode_encode_mbcs(s, slen, errors, errorhandler) return res @@ -512,7 +513,7 @@ builder.append_char('\\') builder.append_code(ord(ch)) - return builder.build(), pos, builder.get_length() + return builder.build(), pos, builder.getlength() def wcharpsize2utf8(space, wcharp, size): """Safe version of rffi.wcharpsize2utf8. @@ -574,7 +575,7 @@ pos = hexescape(builder, s, pos, digits, "rawunicodeescape", errorhandler, message, errors) - return builder.build(), pos, builder.get_length() + return builder.build(), pos, builder.getlength() _utf8_encode_unicode_escape = rutf8.make_utf8_escape_function() diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -77,7 +77,7 @@ self.builder.append_slice(w_unicode._utf8, byte_start, byte_end) def descr_build(self, space): - w_s = space.newutf8(self.builder.build(), self.builder.get_length()) + w_s = space.newutf8(self.builder.build(), self.builder.getlength()) # after build(), we can continue to append more strings # to the same builder. This is supported since # 2ff5087aca28 in RPython. diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -703,7 +703,7 @@ builder.append(data) remaining -= len(data) - return space.newutf8(builder.build(), builder.get_length()) + return space.newutf8(builder.build(), builder.getlength()) def _scan_line_ending(self, limit): if self.readuniversal: diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py --- a/pypy/module/_pypyjson/targetjson.py +++ b/pypy/module/_pypyjson/targetjson.py @@ -93,9 +93,6 @@ assert isinstance(key, W_Unicode) d.dictval[key.unival] = value - def newunicode(self, x): - return W_Unicode(x) - def newtext(self, x): return W_String(x) newbytes = newtext diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -97,6 +97,9 @@ return func(fname1, fname2, *args) return dispatch +def u2utf8(space, u_str): + return space.newutf8(u_str.encode('utf-8'), len(u_str)) + @unwrap_spec(flag=c_int, mode=c_int) def open(space, w_fname, flag, mode=0777): """Open a file (for low level IO). @@ -422,7 +425,7 @@ if space.isinstance_w(w_path, space.w_unicode): path = FileEncoder(space, w_path) fullpath = rposix.getfullpathname(path) - w_fullpath = space.newunicode(fullpath) + w_fullpath = u2utf8(space, fullpath) else: path = space.bytes0_w(w_path) fullpath = rposix.getfullpathname(path) @@ -449,7 +452,7 @@ except OSError as e: raise wrap_oserror(space, e) else: - return space.newunicode(cur) + return u2utf8(space, cur) else: def getcwdu(space): """Return the current working directory as a unicode string.""" @@ -588,7 +591,7 @@ raise w_res = w_bytes elif isinstance(res, unicode): - w_res = space.newunicode(res) + w_res = u2utf8(space, res) else: assert False result_w[i] = w_res diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py --- a/pypy/module/struct/formatiterator.py +++ b/pypy/module/struct/formatiterator.py @@ -186,7 +186,7 @@ elif isinstance(value, str): w_value = self.space.newbytes(value) elif isinstance(value, unicode): - w_value = self.space.newunicode(value) + w_value = self.space.newutf8(value.decode('utf-8'), len(value)) else: assert 0, "unreachable" self.result_w.append(w_value) diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -212,9 +212,6 @@ def newutf8(self, x, l): return w_some_obj() - def newunicode(self, a): - return w_some_obj() - newtext = newbytes newtext_or_none = newbytes newfilename = newbytes diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -50,7 +50,7 @@ @staticmethod def from_utf8builder(builder): return W_UnicodeObject( - builder.build(), builder.get_length()) + builder.build(), builder.getlength()) def __repr__(self): """representation for debugging purposes""" diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py --- a/rpython/rlib/rutf8.py +++ b/rpython/rlib/rutf8.py @@ -725,7 +725,7 @@ return self._s.build() @always_inline - def get_length(self): + def getlength(self): return self._lgt class Utf8StringIterator(object): diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py --- a/rpython/rlib/test/test_rutf8.py +++ b/rpython/rlib/test/test_rutf8.py @@ -169,41 +169,41 @@ s = rutf8.Utf8StringBuilder() s.append("foo") s.append_char("x") - assert s.get_length() == 4 + assert s.getlength() == 4 assert s.build() == "foox" s.append(u"\u1234".encode("utf8")) - assert s.get_length() == 5 + assert s.getlength() == 5 assert s.build().decode("utf8") == u"foox\u1234" s.append("foo") s.append_char("x") - assert s.get_length() == 9 + assert s.getlength() == 9 assert s.build().decode("utf8") == u"foox\u1234foox" s = rutf8.Utf8StringBuilder() s.append_code(0x1234) assert s.build().decode("utf8") == u"\u1234" - assert s.get_length() == 1 + assert s.getlength() == 1 s.append_code(0xD800) - assert s.get_length() == 2 + assert s.getlength() == 2 s = rutf8.Utf8StringBuilder() s.append_utf8("abc", 3) - assert s.get_length() == 3 + assert s.getlength() == 3 assert s.build().decode("utf8") == u"abc" s.append_utf8(u"\u1234".encode("utf8"), 1) assert s.build().decode("utf8") == u"abc\u1234" - assert s.get_length() == 4 + assert s.getlength() == 4 s.append_code(0xD800) - assert s.get_length() == 5 + assert s.getlength() == 5 def test_utf8_string_builder_bad_code(): s = rutf8.Utf8StringBuilder() with pytest.raises(ValueError): s.append_code(0x110000) assert s.build() == '' - assert s.get_length() == 0 + assert s.getlength() == 0 @given(strategies.text()) def test_utf8_iterator(arg): From pypy.commits at gmail.com Mon Mar 19 10:23:24 2018 From: pypy.commits at gmail.com (ariava) Date: Mon, 19 Mar 2018 07:23:24 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: Fix formatting of code blocks from previous commit. Message-ID: <5aafc7dc.85c9df0a.1737e.1481@mx.google.com> Author: Arianna Avanzini Branch: rpython-sprint Changeset: r94001:0c4b7c3e09fa Date: 2018-03-19 15:22 +0100 http://bitbucket.org/pypy/pypy/changeset/0c4b7c3e09fa/ Log: Fix formatting of code blocks from previous commit. diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -263,18 +263,18 @@ --------------------------- RPython provides a helper decorator to force RPython-level types on function -arguments. The decorator, called `enforceargs()`, accepts as parameters the +arguments. The decorator, called ``enforceargs()``, accepts as parameters the types expected to match the arguments of the function. -Functions decorated with `enforceargs()` have their function signature analyzed -and their RPython-level type inferred at import time (for further details about -the flavor of translation performed in RPython, see the +Functions decorated with ``enforceargs()`` have their function signature +analyzed and their RPython-level type inferred at import time (for further +details about the flavor of translation performed in RPython, see the `Annotation pass documentation`_). Encountering types not supported by RPython -will raise a `TypeError`. +will raise a ``TypeError``. -`enforceargs()` by default also performs type checking of parameter types each -time the function is invoked. To disable this behavior, it's possible to -pass the `typecheck=False` parameter to the decorator. +``enforceargs()`` by default also performs type checking of parameter types +each time the function is invoked. To disable this behavior, it's possible to +pass the ``typecheck=False`` parameter to the decorator. .. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator From pypy.commits at gmail.com Mon Mar 19 10:43:24 2018 From: pypy.commits at gmail.com (Raemi) Date: Mon, 19 Mar 2018 07:43:24 -0700 (PDT) Subject: [pypy-commit] stmgc c8-reshare-pages: do not check for safepoint in the become_inev(DONT_WAIT) case Message-ID: <5aafcc8c.f180df0a.e54f5.167b@mx.google.com> Author: Remi Meier Branch: c8-reshare-pages Changeset: r2155:ed6411636c6f Date: 2018-03-19 15:42 +0100 http://bitbucket.org/pypy/stmgc/changeset/ed6411636c6f/ Log: do not check for safepoint in the become_inev(DONT_WAIT) case there is a deadlock if stm_next_to_finalize() calls become_inevitable while keeping the g_finalizers.lock; and _commit_finalizers() just spins while trying to acquire it. In that case, the second thread will not enter a safepoint ever, so all other threads are waiting. This commit *should* fix it by really not waiting in become_inevitable when DONT_SLEEP is passed as the option. diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -1594,9 +1594,9 @@ retry_from_start: assert(STM_PSEGMENT->transaction_state == TS_REGULAR); - _stm_collectable_safe_point(); if (msg != MSG_INEV_DONT_SLEEP) { + _stm_collectable_safe_point(); dprintf(("become_inevitable: %s\n", msg)); if (any_soon_finished_or_inevitable_thread_segment() && From pypy.commits at gmail.com Mon Mar 19 11:01:43 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 08:01:43 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Change socket.close() to raise OSError if closing the underlying socket fails and remove app-level test checking the old behavior. Message-ID: <5aafd0d7.4187df0a.db8c1.1e20@mx.google.com> Author: Manuel Jacob Branch: py3.6 Changeset: r94002:c386ba74c681 Date: 2018-03-19 16:00 +0100 http://bitbucket.org/pypy/pypy/changeset/c386ba74c681/ Log: Change socket.close() to raise OSError if closing the underlying socket fails and remove app-level test checking the old behavior. This is a port of CPython commit 50ab1a3694c43b9ab6798b98d9e5983c78cb17e2: "Issue #26685: Raise OSError if closing a socket fails" diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -292,9 +292,8 @@ """ try: self.sock.close() - except SocketError: - # cpython doesn't return any errors on close - pass + except SocketError as e: + raise converted_error(space, e) self.may_unregister_rpython_finalizer(space) def connect_w(self, space, w_addr): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -395,13 +395,12 @@ if os.name != 'nt': raises(OSError, os.close, fileno) - def test_socket_close_error(self): - import _socket, os - if os.name == 'nt': - skip("Windows sockets are not files") + def test_socket_close_exception(self): + import errno, _socket s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) - os.close(s.fileno()) - s.close() + _socket.socket(fileno=s.fileno()).close() + e = raises(OSError, s.close) + assert e.value.errno in (errno.EBADF, errno.ENOTSOCK) def test_socket_connect(self): import _socket, os From pypy.commits at gmail.com Mon Mar 19 11:55:22 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 08:55:22 -0700 (PDT) Subject: [pypy-commit] pypy default: Add 'e' (half-float) struct format to native format table. Copy CPython's behavior of using short's size and alignment for half-floats. Message-ID: <5aafdd6a.c89edf0a.ca487.28c0@mx.google.com> Author: Manuel Jacob Branch: Changeset: r94003:f65cb735be28 Date: 2018-03-19 16:54 +0100 http://bitbucket.org/pypy/pypy/changeset/f65cb735be28/ Log: Add 'e' (half-float) struct format to native format table. Copy CPython's behavior of using short's size and alignment for half- floats. diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py --- a/rpython/rlib/rstruct/nativefmttable.py +++ b/rpython/rlib/rstruct/nativefmttable.py @@ -130,6 +130,13 @@ sizeof_double = native_fmttable['d']['size'] sizeof_float = native_fmttable['f']['size'] +# Copy CPython's behavior of using short's size and alignment for half-floats. +native_fmttable['e'] = {'size': native_fmttable['h']['size'], + 'alignment': native_fmttable['h']['alignment'], + 'pack': std.pack_halffloat, + 'unpack': std.unpack_halffloat, + } + # ____________________________________________________________ # # A PyPy extension: accepts the 'u' format character in native mode, diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py --- a/rpython/rlib/rstruct/test/test_pack.py +++ b/rpython/rlib/rstruct/test/test_pack.py @@ -139,9 +139,6 @@ self.check('d', 123.456789) def test_pack_halffloat(self): - if self.fmttable is nativefmttable.native_fmttable: - # Host Python cannot handle half floats. - return size = 2 wbuf = MutableStringBuffer(size) self.mypack_into('e', wbuf, 6.5e+04) From pypy.commits at gmail.com Mon Mar 19 12:07:25 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 09:07:25 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5aafe03d.15741c0a.1ade7.60d3@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r94004:fcbdf6d13402 Date: 2018-03-19 16:55 +0100 http://bitbucket.org/pypy/pypy/changeset/fcbdf6d13402/ Log: hg merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -52,3 +52,13 @@ .. branch: refactor-slots Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1403,6 +1403,7 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + @py.test.mark.skipif("sys.platform != 'win32'") def test_rename(self): os = self.posix fname = self.path2 + 'rename.txt' diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,6 +1,7 @@ +import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -40,9 +41,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 - at example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,13 +96,12 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: raise AssertionError("annotationoftype(%r)" % (t,)) diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # # ( ) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # # ( ) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# +# ( ) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -27,6 +30,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +65,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +105,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) dct = {box: cls for box, known_class in boxes_known_classes @@ -321,3 +326,74 @@ self.check_trace_count(3) self.check_resops(guard_value=1) self.check_resops(getarrayitem_gc_i=5) + + def test_bridge_call_loopinvariant(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + a = get() + a = get() + res += a.x + if y > n: + res += 1 + res += get().x + a.x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) + + @pytest.mark.xfail() + def test_bridge_call_loopinvariant_2(self): + class A(object): + pass + class B(object): + pass + + aholder = B() + aholder.a = A() + + @jit.loop_invariant + def get(): + return aholder.a + + myjitdriver = jit.JitDriver(greens=[], reds=['y', 'res', 'n']) + def f(x, y, n): + if x == 10001121: + aholder.a = A() + if x: + get().x = 1 + else: + get().x = 2 + res = 0 + while y > 0: + myjitdriver.jit_merge_point(y=y, n=n, res=res) + if y > n: + res += get().x + res += 1 + res += get().x + y -= 1 + return res + res = self.meta_interp(f, [6, 32, 16]) + self.check_trace_count(3) + self.check_resops(call_r=1) diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -40,7 +40,7 @@ class FakeOptimizer(object): metainterp_sd = None - optheap = None + optheap = optrewrite = None def __init__(self, trace=None): self.trace = trace diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py --- a/rpython/rlib/rstruct/nativefmttable.py +++ b/rpython/rlib/rstruct/nativefmttable.py @@ -130,6 +130,13 @@ sizeof_double = native_fmttable['d']['size'] sizeof_float = native_fmttable['f']['size'] +# Copy CPython's behavior of using short's size and alignment for half-floats. +native_fmttable['e'] = {'size': native_fmttable['h']['size'], + 'alignment': native_fmttable['h']['alignment'], + 'pack': std.pack_halffloat, + 'unpack': std.unpack_halffloat, + } + # ____________________________________________________________ # # A PyPy extension: accepts the 'u' format character in native mode, diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py --- a/rpython/rlib/rstruct/test/test_pack.py +++ b/rpython/rlib/rstruct/test/test_pack.py @@ -139,9 +139,6 @@ self.check('d', 123.456789) def test_pack_halffloat(self): - if self.fmttable is nativefmttable.native_fmttable: - # Host Python cannot handle half floats. - return size = 2 wbuf = MutableStringBuffer(size) self.mypack_into('e', wbuf, 6.5e+04) From pypy.commits at gmail.com Mon Mar 19 12:07:33 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 09:07:33 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge py3.5 Message-ID: <5aafe045.9585df0a.73768.2d95@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r94005:02491c2d50b2 Date: 2018-03-19 16:57 +0100 http://bitbucket.org/pypy/pypy/changeset/02491c2d50b2/ Log: hg merge py3.5 diff too long, truncating to 2000 out of 126938 lines diff --git a/lib-python/3/_collections_abc.py b/lib-python/3/_collections_abc.py --- a/lib-python/3/_collections_abc.py +++ b/lib-python/3/_collections_abc.py @@ -9,9 +9,10 @@ from abc import ABCMeta, abstractmethod import sys -__all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", - "Hashable", "Iterable", "Iterator", "Generator", - "Sized", "Container", "Callable", +__all__ = ["Awaitable", "Coroutine", + "AsyncIterable", "AsyncIterator", "AsyncGenerator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", + "Sized", "Container", "Callable", "Collection", "Set", "MutableSet", "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", @@ -59,10 +60,29 @@ coroutine = type(_coro) _coro.close() # Prevent ResourceWarning del _coro +## asynchronous generator ## +## This should be reverted, once async generators are supported. +## Temporary fix. +#async def _ag(): yield +#_ag = _ag() +#async_generator = type(_ag) +#del _ag ### ONE-TRICK PONIES ### +def _check_methods(C, *methods): + mro = C.__mro__ + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + class Hashable(metaclass=ABCMeta): __slots__ = () @@ -74,11 +94,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Hashable: - for B in C.__mro__: - if "__hash__" in B.__dict__: - if B.__dict__["__hash__"]: - return True - break + return _check_methods(C, "__hash__") return NotImplemented @@ -93,11 +109,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Awaitable: - for B in C.__mro__: - if "__await__" in B.__dict__: - if B.__dict__["__await__"]: - return True - break + return _check_methods(C, "__await__") return NotImplemented @@ -138,14 +150,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Coroutine: - mro = C.__mro__ - for method in ('__await__', 'send', 'throw', 'close'): - for base in mro: - if method in base.__dict__: - break - else: - return NotImplemented - return True + return _check_methods(C, '__await__', 'send', 'throw', 'close') return NotImplemented @@ -163,8 +168,7 @@ @classmethod def __subclasshook__(cls, C): if cls is AsyncIterable: - if any("__aiter__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__aiter__") return NotImplemented @@ -183,12 +187,61 @@ @classmethod def __subclasshook__(cls, C): if cls is AsyncIterator: - if (any("__anext__" in B.__dict__ for B in C.__mro__) and - any("__aiter__" in B.__dict__ for B in C.__mro__)): - return True + return _check_methods(C, "__anext__", "__aiter__") return NotImplemented +class AsyncGenerator(AsyncIterator): + + __slots__ = () + + async def __anext__(self): + """Return the next item from the asynchronous generator. + When exhausted, raise StopAsyncIteration. + """ + return await self.asend(None) + + @abstractmethod + async def asend(self, value): + """Send a value into the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + raise StopAsyncIteration + + @abstractmethod + async def athrow(self, typ, val=None, tb=None): + """Raise an exception in the asynchronous generator. + Return next yielded value or raise StopAsyncIteration. + """ + if val is None: + if tb is None: + raise typ + val = typ() + if tb is not None: + val = val.with_traceback(tb) + raise val + + async def aclose(self): + """Raise GeneratorExit inside coroutine. + """ + try: + await self.athrow(GeneratorExit) + except (GeneratorExit, StopAsyncIteration): + pass + else: + raise RuntimeError("asynchronous generator ignored GeneratorExit") + + @classmethod + def __subclasshook__(cls, C): + if cls is AsyncGenerator: + return _check_methods(C, '__aiter__', '__anext__', + 'asend', 'athrow', 'aclose') + return NotImplemented + + +# AsyncGenerator.register(async_generator) + + class Iterable(metaclass=ABCMeta): __slots__ = () @@ -201,8 +254,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterable: - if any("__iter__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__iter__") return NotImplemented @@ -221,9 +273,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Iterator: - if (any("__next__" in B.__dict__ for B in C.__mro__) and - any("__iter__" in B.__dict__ for B in C.__mro__)): - return True + return _check_methods(C, '__iter__', '__next__') return NotImplemented Iterator.register(bytes_iterator) @@ -242,6 +292,22 @@ Iterator.register(zip_iterator) +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + return _check_methods(C, "__reversed__", "__iter__") + return NotImplemented + + class Generator(Iterator): __slots__ = () @@ -285,17 +351,10 @@ @classmethod def __subclasshook__(cls, C): if cls is Generator: - mro = C.__mro__ - for method in ('__iter__', '__next__', 'send', 'throw', 'close'): - for base in mro: - if method in base.__dict__: - break - else: - return NotImplemented - return True + return _check_methods(C, '__iter__', '__next__', + 'send', 'throw', 'close') return NotImplemented - Generator.register(generator) @@ -310,8 +369,7 @@ @classmethod def __subclasshook__(cls, C): if cls is Sized: - if any("__len__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__len__") return NotImplemented @@ -326,10 +384,18 @@ @classmethod def __subclasshook__(cls, C): if cls is Container: - if any("__contains__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__contains__") return NotImplemented +class Collection(Sized, Iterable, Container): + + __slots__ = () + + @classmethod + def __subclasshook__(cls, C): + if cls is Collection: + return _check_methods(C, "__len__", "__iter__", "__contains__") + return NotImplemented class Callable(metaclass=ABCMeta): @@ -342,15 +408,14 @@ @classmethod def __subclasshook__(cls, C): if cls is Callable: - if any("__call__" in B.__dict__ for B in C.__mro__): - return True + return _check_methods(C, "__call__") return NotImplemented ### SETS ### -class Set(Sized, Iterable, Container): +class Set(Collection): """A set is a finite, iterable container. @@ -575,7 +640,7 @@ ### MAPPINGS ### -class Mapping(Sized, Iterable, Container): +class Mapping(Collection): __slots__ = () @@ -623,6 +688,8 @@ return NotImplemented return dict(self.items()) == dict(other.items()) + __reversed__ = None + Mapping.register(mappingproxy) @@ -672,7 +739,7 @@ except KeyError: return False else: - return v == value + return v is value or v == value def __iter__(self): for key in self._mapping: @@ -687,7 +754,8 @@ def __contains__(self, value): for key in self._mapping: - if value == self._mapping[key]: + v = self._mapping[key] + if v is value or v == value: return True return False @@ -796,7 +864,7 @@ ### SEQUENCES ### -class Sequence(Sized, Iterable, Container): +class Sequence(Reversible, Collection): """All the operations on a read-only sequence. @@ -822,7 +890,7 @@ def __contains__(self, value): for v in self: - if v == value: + if v is value or v == value: return True return False diff --git a/lib-python/3/_compat_pickle.py b/lib-python/3/_compat_pickle.py --- a/lib-python/3/_compat_pickle.py +++ b/lib-python/3/_compat_pickle.py @@ -242,3 +242,10 @@ for excname in PYTHON3_OSERROR_EXCEPTIONS: REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'OSError') + +PYTHON3_IMPORTERROR_EXCEPTIONS = ( + 'ModuleNotFoundError', +) + +for excname in PYTHON3_IMPORTERROR_EXCEPTIONS: + REVERSE_NAME_MAPPING[('builtins', excname)] = ('exceptions', 'ImportError') diff --git a/lib-python/3/_osx_support.py b/lib-python/3/_osx_support.py --- a/lib-python/3/_osx_support.py +++ b/lib-python/3/_osx_support.py @@ -210,7 +210,7 @@ # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub('-arch\s+\w+\s', ' ', flags, re.ASCII) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags, re.ASCII) flags = re.sub('-isysroot [^ \t]*', ' ', flags) _save_modified_value(_config_vars, cv, flags) @@ -232,7 +232,7 @@ if 'CC' in os.environ: return _config_vars - if re.search('-arch\s+ppc', _config_vars['CFLAGS']) is not None: + if re.search(r'-arch\s+ppc', _config_vars['CFLAGS']) is not None: # NOTE: Cannot use subprocess here because of bootstrap # issues when building Python itself status = os.system( @@ -251,7 +251,7 @@ for cv in _UNIVERSAL_CONFIG_VARS: if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub('-arch\s+ppc\w*\s', ' ', flags) + flags = re.sub(r'-arch\s+ppc\w*\s', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars @@ -267,7 +267,7 @@ for cv in _UNIVERSAL_CONFIG_VARS: if cv in _config_vars and '-arch' in _config_vars[cv]: flags = _config_vars[cv] - flags = re.sub('-arch\s+\w+\s', ' ', flags) + flags = re.sub(r'-arch\s+\w+\s', ' ', flags) flags = flags + ' ' + arch _save_modified_value(_config_vars, cv, flags) @@ -465,7 +465,7 @@ machine = 'fat' - archs = re.findall('-arch\s+(\S+)', cflags) + archs = re.findall(r'-arch\s+(\S+)', cflags) archs = tuple(sorted(set(archs))) if len(archs) == 1: diff --git a/lib-python/3/_pydecimal.py b/lib-python/3/_pydecimal.py --- a/lib-python/3/_pydecimal.py +++ b/lib-python/3/_pydecimal.py @@ -148,7 +148,7 @@ __name__ = 'decimal' # For pickling __version__ = '1.70' # Highest version of the spec this complies with # See http://speleotrove.com/decimal/ -__libmpdec_version__ = "2.4.1" # compatible libmpdec version +__libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math import numbers as _numbers @@ -589,7 +589,7 @@ # From a string # REs insist on real strings, so we can too. if isinstance(value, str): - m = _parser(value.strip()) + m = _parser(value.strip().replace("_", "")) if m is None: if context is None: context = getcontext() @@ -1010,6 +1010,56 @@ """ return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + def as_integer_ratio(self): + """Express a finite Decimal instance in the form n / d. + + Returns a pair (n, d) of integers. When called on an infinity + or NaN, raises OverflowError or ValueError respectively. + + >>> Decimal('3.14').as_integer_ratio() + (157, 50) + >>> Decimal('-123e5').as_integer_ratio() + (-12300000, 1) + >>> Decimal('0.00').as_integer_ratio() + (0, 1) + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("cannot convert NaN to integer ratio") + else: + raise OverflowError("cannot convert Infinity to integer ratio") + + if not self: + return 0, 1 + + # Find n, d in lowest terms such that abs(self) == n / d; + # we'll deal with the sign later. + n = int(self._int) + if self._exp >= 0: + # self is an integer. + n, d = n * 10**self._exp, 1 + else: + # Find d2, d5 such that abs(self) = n / (2**d2 * 5**d5). + d5 = -self._exp + while d5 > 0 and n % 5 == 0: + n //= 5 + d5 -= 1 + + # (n & -n).bit_length() - 1 counts trailing zeros in binary + # representation of n (provided n is nonzero). + d2 = -self._exp + shift2 = min((n & -n).bit_length() - 1, d2) + if shift2: + n >>= shift2 + d2 -= shift2 + + d = 5**d5 << d2 + + if self._sign: + n = -n + return n, d + def __repr__(self): """Represents the number as an instance of Decimal.""" # Invariant: eval(repr(d)) == d @@ -4075,7 +4125,7 @@ This will make it round up for that operation. """ rounding = self.rounding - self.rounding= type + self.rounding = type return rounding def create_decimal(self, num='0'): @@ -4084,10 +4134,10 @@ This method implements the to-number operation of the IBM Decimal specification.""" - if isinstance(num, str) and num != num.strip(): + if isinstance(num, str) and (num != num.strip() or '_' in num): return self._raise_error(ConversionSyntax, - "no trailing or leading whitespace is " - "permitted.") + "trailing or leading whitespace and " + "underscores are not permitted.") d = Decimal(num, context=self) if d._isnan() and len(d._int) > self.prec - self.clamp: diff --git a/lib-python/3/_pyio.py b/lib-python/3/_pyio.py --- a/lib-python/3/_pyio.py +++ b/lib-python/3/_pyio.py @@ -6,7 +6,6 @@ import abc import codecs import errno -import array import stat import sys # Import _thread instead of threading to reduce startup cost @@ -161,6 +160,8 @@ opened in a text mode, and for bytes a BytesIO can be used like a file opened in a binary mode. """ + if not isinstance(file, int): + file = os.fspath(file) if not isinstance(file, (str, bytes, int)): raise TypeError("invalid file: %r" % file) if not isinstance(mode, str): @@ -182,8 +183,8 @@ text = "t" in modes binary = "b" in modes if "U" in modes: - if creating or writing or appending: - raise ValueError("can't use U and writing mode at once") + if creating or writing or appending or updating: + raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) @@ -1516,7 +1517,7 @@ if self._fd >= 0 and self._closefd and not self.closed: import warnings warnings.warn('unclosed file %r' % (self,), ResourceWarning, - stacklevel=2) + stacklevel=2, source=self) self.close() def __getstate__(self): diff --git a/lib-python/3/_strptime.py b/lib-python/3/_strptime.py --- a/lib-python/3/_strptime.py +++ b/lib-python/3/_strptime.py @@ -199,12 +199,15 @@ 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", 'M': r"(?P[0-5]\d|\d)", 'S': r"(?P6[0-1]|[0-5]\d|\d)", 'U': r"(?P5[0-3]|[0-4]\d|\d)", 'w': r"(?P[0-6])", + 'u': r"(?P[1-7])", + 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", #XXX: Does 'Y' need to worry about having less or more than @@ -299,6 +302,22 @@ return 1 + days_to_week + day_of_week +def _calc_julian_from_V(iso_year, iso_week, iso_weekday): + """Calculate the Julian day based on the ISO 8601 year, week, and weekday. + ISO weeks start on Mondays, with week 01 being the week containing 4 Jan. + ISO week days range from 1 (Monday) to 7 (Sunday). + """ + correction = datetime_date(iso_year, 1, 4).isoweekday() + 3 + ordinal = (iso_week * 7) + iso_weekday - correction + # ordinal may be negative or 0 now, which means the date is in the previous + # calendar year + if ordinal < 1: + ordinal += datetime_date(iso_year, 1, 1).toordinal() + iso_year -= 1 + ordinal -= datetime_date(iso_year, 1, 1).toordinal() + return iso_year, ordinal + + def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a 2-tuple consisting of a time struct and an int containing the number of microseconds based on the input string and the @@ -345,15 +364,15 @@ raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = None + iso_year = year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 tzoffset = None # Default to -1 to signify that values not known; not critical to have, # though - week_of_year = -1 - week_of_year_start = -1 + iso_week = week_of_year = None + week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate # values weekday = julian = None @@ -375,6 +394,8 @@ year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) + elif group_key == 'G': + iso_year = int(found_dict['G']) elif group_key == 'm': month = int(found_dict['m']) elif group_key == 'B': @@ -420,6 +441,9 @@ weekday = 6 else: weekday -= 1 + elif group_key == 'u': + weekday = int(found_dict['u']) + weekday -= 1 elif group_key == 'j': julian = int(found_dict['j']) elif group_key in ('U', 'W'): @@ -430,6 +454,8 @@ else: # W starts week on Monday. week_of_year_start = 0 + elif group_key == 'V': + iso_week = int(found_dict['V']) elif group_key == 'z': z = found_dict['z'] tzoffset = int(z[1:3]) * 60 + int(z[3:5]) @@ -450,32 +476,61 @@ else: tz = value break + # Deal with the cases where ambiguities arize + # don't assume default values for ISO week/year + if year is None and iso_year is not None: + if iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + if julian is not None: + raise ValueError("Day of the year directive '%j' is not " + "compatible with ISO year directive '%G'. " + "Use '%Y' instead.") + elif week_of_year is None and iso_week is not None: + if weekday is None: + raise ValueError("ISO week directive '%V' must be used with " + "the ISO year directive '%G' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + else: + raise ValueError("ISO week directive '%V' is incompatible with " + "the year directive '%Y'. Use the ISO year '%G' " + "instead.") + leap_year_fix = False if year is None and month == 2 and day == 29: year = 1904 # 1904 is first leap year of 20th century leap_year_fix = True elif year is None: year = 1900 + + # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. - if julian is None and week_of_year != -1 and weekday is not None: - week_starts_Mon = True if week_of_year_start == 0 else False - julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, - week_starts_Mon) - if julian <= 0: + if julian is None and weekday is not None: + if week_of_year is not None: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + elif iso_year is not None and iso_week is not None: + year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1) + if julian is not None and julian <= 0: year -= 1 yday = 366 if calendar.isleap(year) else 365 julian += yday - # Cannot pre-calculate datetime_date() since can change in Julian - # calculation and thus could have different value for the day of the week - # calculation. + if julian is None: + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of + # the week calculation. # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 - else: # Assume that if they bothered to include Julian day it will - # be accurate. - datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal()) + else: # Assume that if they bothered to include Julian day (or if it was + # calculated above with year/week/weekday) it will be accurate. + datetime_result = datetime_date.fromordinal( + (julian - 1) + + datetime_date(year, 1, 1).toordinal()) year = datetime_result.year month = datetime_result.month day = datetime_result.day diff --git a/lib-python/3/aifc.py b/lib-python/3/aifc.py --- a/lib-python/3/aifc.py +++ b/lib-python/3/aifc.py @@ -257,6 +257,15 @@ _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') +_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' +_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' +_aifc_params.framerate.__doc__ = 'Sampling frequency' +_aifc_params.nframes.__doc__ = 'Number of audio frames' +_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' +_aifc_params.compname.__doc__ = ("""\ +A human-readable version of the compression type +('not compressed' for AIFF files)""") + class Aifc_read: # Variables used in this class: @@ -294,6 +303,8 @@ # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk # _framesize -- size of one frame in the file + _file = None # Set here since __del__ checks it + def initfp(self, file): self._version = 0 self._convert = None @@ -335,9 +346,15 @@ def __init__(self, f): if isinstance(f, str): - f = builtins.open(f, 'rb') - # else, assume it is an open file object already - self.initfp(f) + file_object = builtins.open(f, 'rb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + else: + # assume it is an open file object already + self.initfp(f) def __enter__(self): return self @@ -532,18 +549,23 @@ # _datalength -- the size of the audio samples written to the header # _datawritten -- the size of the audio samples actually written + _file = None # Set here since __del__ checks it + def __init__(self, f): if isinstance(f, str): - filename = f - f = builtins.open(f, 'wb') + file_object = builtins.open(f, 'wb') + try: + self.initfp(file_object) + except: + file_object.close() + raise + + # treat .aiff file extensions as non-compressed audio + if f.endswith('.aiff'): + self._aifc = 0 else: - # else, assume it is an open file object already - filename = '???' - self.initfp(f) - if filename[-5:] == '.aiff': - self._aifc = 0 - else: - self._aifc = 1 + # assume it is an open file object already + self.initfp(f) def initfp(self, file): self._file = file diff --git a/lib-python/3/argparse.py b/lib-python/3/argparse.py --- a/lib-python/3/argparse.py +++ b/lib-python/3/argparse.py @@ -118,10 +118,16 @@ def __repr__(self): type_name = type(self).__name__ arg_strings = [] + star_args = {} for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) + if name.isidentifier(): + arg_strings.append('%s=%r' % (name, value)) + else: + star_args[name] = value + if star_args: + arg_strings.append('**%s' % repr(star_args)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): @@ -176,7 +182,7 @@ self._root_section = self._Section(self, None) self._current_section = self._root_section - self._whitespace_matcher = _re.compile(r'\s+') + self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') # =============================== @@ -204,8 +210,6 @@ if self.parent is not None: self.formatter._indent() join = self.formatter._join_parts - for func, args in self.items: - func(*args) item_help = join([func(*args) for func, args in self.items]) if self.parent is not None: self.formatter._dedent() diff --git a/lib-python/3/ast.py b/lib-python/3/ast.py --- a/lib-python/3/ast.py +++ b/lib-python/3/ast.py @@ -35,6 +35,8 @@ return compile(source, filename, mode, PyCF_ONLY_AST) +_NUM_TYPES = (int, float, complex) + def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python @@ -47,7 +49,9 @@ if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): + if isinstance(node, Constant): + return node.value + elif isinstance(node, (Str, Bytes)): return node.s elif isinstance(node, Num): return node.n @@ -62,24 +66,21 @@ in zip(node.keys, node.values)) elif isinstance(node, NameConstant): return node.value - elif isinstance(node, UnaryOp) and \ - isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - elif isinstance(node, BinOp) and \ - isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + if isinstance(operand, _NUM_TYPES): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): left = _convert(node.left) right = _convert(node.right) - if isinstance(node.op, Add): - return left + right - else: - return left - right + if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES): + if isinstance(node.op, Add): + return left + right + else: + return left - right raise ValueError('malformed node or string: ' + repr(node)) return _convert(node_or_string) @@ -196,12 +197,19 @@ """ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if not(node.body and isinstance(node.body[0], Expr)): + return + node = node.body[0].value + if isinstance(node, Str): + text = node.s + elif isinstance(node, Constant) and isinstance(node.value, str): + text = node.value + else: + return + if clean: + import inspect + text = inspect.cleandoc(text) + return text def walk(node): diff --git a/lib-python/3/asynchat.py b/lib-python/3/asynchat.py --- a/lib-python/3/asynchat.py +++ b/lib-python/3/asynchat.py @@ -285,35 +285,6 @@ return result -class fifo: - def __init__(self, list=None): - import warnings - warnings.warn('fifo class will be removed in Python 3.6', - DeprecationWarning, stacklevel=2) - if not list: - self.list = deque() - else: - self.list = deque(list) - - def __len__(self): - return len(self.list) - - def is_empty(self): - return not self.list - - def first(self): - return self.list[0] - - def push(self, data): - self.list.append(data) - - def pop(self): - if self.list: - return (1, self.list.popleft()) - else: - return (0, None) - - # Given 'haystack', see if any prefix of 'needle' is at its end. This # assumes an exact match has already been checked. Return the number of # characters matched. diff --git a/lib-python/3/asyncio/base_events.py b/lib-python/3/asyncio/base_events.py --- a/lib-python/3/asyncio/base_events.py +++ b/lib-python/3/asyncio/base_events.py @@ -57,7 +57,7 @@ def _format_handle(handle): cb = handle._callback - if inspect.ismethod(cb) and isinstance(cb.__self__, tasks.Task): + if isinstance(getattr(cb, '__self__', None), tasks.Task): # format the task return repr(cb.__self__) else: @@ -505,7 +505,8 @@ if compat.PY34: def __del__(self): if not self.is_closed(): - warnings.warn("unclosed event loop %r" % self, ResourceWarning) + warnings.warn("unclosed event loop %r" % self, ResourceWarning, + source=self) if not self.is_running(): self.close() diff --git a/lib-python/3/asyncio/base_futures.py b/lib-python/3/asyncio/base_futures.py new file mode 100644 --- /dev/null +++ b/lib-python/3/asyncio/base_futures.py @@ -0,0 +1,71 @@ +__all__ = [] + +import concurrent.futures._base +import reprlib + +from . import events + +Error = concurrent.futures._base.Error +CancelledError = concurrent.futures.CancelledError +TimeoutError = concurrent.futures.TimeoutError + + +class InvalidStateError(Error): + """The operation is not allowed in this state.""" + + +# States for Future. +_PENDING = 'PENDING' +_CANCELLED = 'CANCELLED' +_FINISHED = 'FINISHED' + + +def isfuture(obj): + """Check for a Future. + + This returns True when obj is a Future instance or is advertising + itself as duck-type compatible by setting _asyncio_future_blocking. + See comment in Future for more details. + """ + return (hasattr(obj.__class__, '_asyncio_future_blocking') and + obj._asyncio_future_blocking is not None) + + +def _format_callbacks(cb): + """helper function for Future.__repr__""" + size = len(cb) + if not size: + cb = '' + + def format_cb(callback): + return events._format_callback_source(callback, ()) + + if size == 1: + cb = format_cb(cb[0]) + elif size == 2: + cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1])) + elif size > 2: + cb = '{}, <{} more>, {}'.format(format_cb(cb[0]), + size - 2, + format_cb(cb[-1])) + return 'cb=[%s]' % cb + + +def _future_repr_info(future): + # (Future) -> str + """helper function for Future.__repr__""" + info = [future._state.lower()] + if future._state == _FINISHED: + if future._exception is not None: + info.append('exception={!r}'.format(future._exception)) + else: + # use reprlib to limit the length of the output, especially + # for very long strings + result = reprlib.repr(future._result) + info.append('result={}'.format(result)) + if future._callbacks: + info.append(_format_callbacks(future._callbacks)) + if future._source_traceback: + frame = future._source_traceback[-1] + info.append('created at %s:%s' % (frame[0], frame[1])) + return info diff --git a/lib-python/3/asyncio/base_subprocess.py b/lib-python/3/asyncio/base_subprocess.py --- a/lib-python/3/asyncio/base_subprocess.py +++ b/lib-python/3/asyncio/base_subprocess.py @@ -127,7 +127,8 @@ if compat.PY34: def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def get_pid(self): diff --git a/lib-python/3/asyncio/base_tasks.py b/lib-python/3/asyncio/base_tasks.py new file mode 100644 --- /dev/null +++ b/lib-python/3/asyncio/base_tasks.py @@ -0,0 +1,76 @@ +import linecache +import traceback + +from . import base_futures +from . import coroutines + + +def _task_repr_info(task): + info = base_futures._future_repr_info(task) + + if task._must_cancel: + # replace status + info[0] = 'cancelling' + + coro = coroutines._format_coroutine(task._coro) + info.insert(1, 'coro=<%s>' % coro) + + if task._fut_waiter is not None: + info.insert(2, 'wait_for=%r' % task._fut_waiter) + return info + + +def _task_get_stack(task, limit): + frames = [] + try: + # 'async def' coroutines + f = task._coro.cr_frame + except AttributeError: + f = task._coro.gi_frame + if f is not None: + while f is not None: + if limit is not None: + if limit <= 0: + break + limit -= 1 + frames.append(f) + f = f.f_back + frames.reverse() + elif task._exception is not None: + tb = task._exception.__traceback__ + while tb is not None: + if limit is not None: + if limit <= 0: + break + limit -= 1 + frames.append(tb.tb_frame) + tb = tb.tb_next + return frames + + +def _task_print_stack(task, limit, file): + extracted_list = [] + checked = set() + for f in task.get_stack(limit=limit): + lineno = f.f_lineno + co = f.f_code + filename = co.co_filename + name = co.co_name + if filename not in checked: + checked.add(filename) + linecache.checkcache(filename) + line = linecache.getline(filename, lineno, f.f_globals) + extracted_list.append((filename, lineno, name, line)) + exc = task._exception + if not extracted_list: + print('No stack for %r' % task, file=file) + elif exc is not None: + print('Traceback for %r (most recent call last):' % task, + file=file) + else: + print('Stack for %r (most recent call last):' % task, + file=file) + traceback.print_list(extracted_list, file=file) + if exc is not None: + for line in traceback.format_exception_only(exc.__class__, exc): + print(line, file=file, end='') diff --git a/lib-python/3/asyncio/coroutines.py b/lib-python/3/asyncio/coroutines.py --- a/lib-python/3/asyncio/coroutines.py +++ b/lib-python/3/asyncio/coroutines.py @@ -11,7 +11,7 @@ from . import compat from . import events -from . import futures +from . import base_futures from .log import logger @@ -208,7 +208,7 @@ @functools.wraps(func) def coro(*args, **kw): res = func(*args, **kw) - if (futures.isfuture(res) or inspect.isgenerator(res) or + if (base_futures.isfuture(res) or inspect.isgenerator(res) or isinstance(res, CoroWrapper)): res = yield from res elif _AwaitableABC is not None: diff --git a/lib-python/3/asyncio/events.py b/lib-python/3/asyncio/events.py --- a/lib-python/3/asyncio/events.py +++ b/lib-python/3/asyncio/events.py @@ -11,6 +11,7 @@ import functools import inspect +import os import reprlib import socket import subprocess @@ -611,6 +612,9 @@ # A TLS for the running event loop, used by _get_running_loop. class _RunningLoop(threading.local): _loop = None + _pid = None + + _running_loop = _RunningLoop() @@ -620,7 +624,9 @@ This is a low-level function intended to be used by event loops. This function is thread-specific. """ - return _running_loop._loop + running_loop = _running_loop._loop + if running_loop is not None and _running_loop._pid == os.getpid(): + return running_loop def _set_running_loop(loop): @@ -629,6 +635,7 @@ This is a low-level function intended to be used by event loops. This function is thread-specific. """ + _running_loop._pid = os.getpid() _running_loop._loop = loop diff --git a/lib-python/3/asyncio/futures.py b/lib-python/3/asyncio/futures.py --- a/lib-python/3/asyncio/futures.py +++ b/lib-python/3/asyncio/futures.py @@ -1,35 +1,32 @@ """A Future class similar to the one in PEP 3148.""" -__all__ = ['CancelledError', 'TimeoutError', - 'InvalidStateError', - 'Future', 'wrap_future', 'isfuture', - ] +__all__ = ['CancelledError', 'TimeoutError', 'InvalidStateError', + 'Future', 'wrap_future', 'isfuture'] -import concurrent.futures._base +import concurrent.futures import logging -import reprlib import sys import traceback +from . import base_futures from . import compat from . import events -# States for Future. -_PENDING = 'PENDING' -_CANCELLED = 'CANCELLED' -_FINISHED = 'FINISHED' -Error = concurrent.futures._base.Error -CancelledError = concurrent.futures.CancelledError -TimeoutError = concurrent.futures.TimeoutError +CancelledError = base_futures.CancelledError +InvalidStateError = base_futures.InvalidStateError +TimeoutError = base_futures.TimeoutError +isfuture = base_futures.isfuture + + +_PENDING = base_futures._PENDING +_CANCELLED = base_futures._CANCELLED +_FINISHED = base_futures._FINISHED + STACK_DEBUG = logging.DEBUG - 1 # heavy-duty debugging -class InvalidStateError(Error): - """The operation is not allowed in this state.""" - - class _TracebackLogger: """Helper to log a traceback upon destruction if not cleared. @@ -173,45 +170,10 @@ if self._loop.get_debug(): self._source_traceback = traceback.extract_stack(sys._getframe(1)) - def __format_callbacks(self): - cb = self._callbacks - size = len(cb) - if not size: - cb = '' - - def format_cb(callback): - return events._format_callback_source(callback, ()) - - if size == 1: - cb = format_cb(cb[0]) - elif size == 2: - cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1])) - elif size > 2: - cb = '{}, <{} more>, {}'.format(format_cb(cb[0]), - size-2, - format_cb(cb[-1])) - return 'cb=[%s]' % cb - - def _repr_info(self): - info = [self._state.lower()] - if self._state == _FINISHED: - if self._exception is not None: - info.append('exception={!r}'.format(self._exception)) - else: - # use reprlib to limit the length of the output, especially - # for very long strings - result = reprlib.repr(self._result) - info.append('result={}'.format(result)) - if self._callbacks: - info.append(self.__format_callbacks()) - if self._source_traceback: - frame = self._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) - return info + _repr_info = base_futures._future_repr_info def __repr__(self): - info = self._repr_info() - return '<%s %s>' % (self.__class__.__name__, ' '.join(info)) + return '<%s %s>' % (self.__class__.__name__, ' '.join(self._repr_info())) # On Python 3.3 and older, objects with a destructor part of a reference # cycle are never destroyed. It's not more the case on Python 3.4 thanks @@ -385,6 +347,10 @@ __await__ = __iter__ # make compatible with 'await' expression +# Needed for testing purposes. +_PyFuture = Future + + def _set_result_unless_cancelled(fut, result): """Helper setting the result only if the future was not cancelled.""" if fut.cancelled(): @@ -477,3 +443,12 @@ new_future = loop.create_future() _chain_future(future, new_future) return new_future + + +try: + import _asyncio +except ImportError: + pass +else: + # _CFuture is needed for tests. + Future = _CFuture = _asyncio.Future diff --git a/lib-python/3/asyncio/proactor_events.py b/lib-python/3/asyncio/proactor_events.py --- a/lib-python/3/asyncio/proactor_events.py +++ b/lib-python/3/asyncio/proactor_events.py @@ -92,7 +92,8 @@ if compat.PY34: def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): diff --git a/lib-python/3/asyncio/selector_events.py b/lib-python/3/asyncio/selector_events.py --- a/lib-python/3/asyncio/selector_events.py +++ b/lib-python/3/asyncio/selector_events.py @@ -627,7 +627,8 @@ if compat.PY34: def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._sock.close() def _fatal_error(self, exc, message='Fatal error on transport'): diff --git a/lib-python/3/asyncio/sslproto.py b/lib-python/3/asyncio/sslproto.py --- a/lib-python/3/asyncio/sslproto.py +++ b/lib-python/3/asyncio/sslproto.py @@ -331,7 +331,8 @@ if compat.PY34: def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def pause_reading(self): diff --git a/lib-python/3/asyncio/subprocess.py b/lib-python/3/asyncio/subprocess.py --- a/lib-python/3/asyncio/subprocess.py +++ b/lib-python/3/asyncio/subprocess.py @@ -24,6 +24,8 @@ self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +45,14 @@ self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -86,9 +90,18 @@ else: reader.set_exception(exc) + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport() + def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None class Process: diff --git a/lib-python/3/asyncio/tasks.py b/lib-python/3/asyncio/tasks.py --- a/lib-python/3/asyncio/tasks.py +++ b/lib-python/3/asyncio/tasks.py @@ -9,11 +9,10 @@ import concurrent.futures import functools import inspect -import linecache -import traceback import warnings import weakref +from . import base_tasks from . import compat from . import coroutines from . import events @@ -93,18 +92,7 @@ futures.Future.__del__(self) def _repr_info(self): - info = super()._repr_info() - - if self._must_cancel: - # replace status - info[0] = 'cancelling' - - coro = coroutines._format_coroutine(self._coro) - info.insert(1, 'coro=<%s>' % coro) - - if self._fut_waiter is not None: - info.insert(2, 'wait_for=%r' % self._fut_waiter) - return info + return base_tasks._task_repr_info(self) def get_stack(self, *, limit=None): """Return the list of stack frames for this task's coroutine. @@ -127,31 +115,7 @@ For reasons beyond our control, only one stack frame is returned for a suspended coroutine. """ - frames = [] - try: - # 'async def' coroutines - f = self._coro.cr_frame - except AttributeError: - f = self._coro.gi_frame - if f is not None: - while f is not None: - if limit is not None: - if limit <= 0: - break - limit -= 1 - frames.append(f) - f = f.f_back - frames.reverse() - elif self._exception is not None: - tb = self._exception.__traceback__ - while tb is not None: - if limit is not None: - if limit <= 0: - break - limit -= 1 - frames.append(tb.tb_frame) - tb = tb.tb_next - return frames + return base_tasks._task_get_stack(self, limit) def print_stack(self, *, limit=None, file=None): """Print the stack or traceback for this task's coroutine. @@ -162,31 +126,7 @@ to which the output is written; by default output is written to sys.stderr. """ - extracted_list = [] - checked = set() - for f in self.get_stack(limit=limit): - lineno = f.f_lineno - co = f.f_code - filename = co.co_filename - name = co.co_name - if filename not in checked: - checked.add(filename) - linecache.checkcache(filename) - line = linecache.getline(filename, lineno, f.f_globals) - extracted_list.append((filename, lineno, name, line)) - exc = self._exception - if not extracted_list: - print('No stack for %r' % self, file=file) - elif exc is not None: - print('Traceback for %r (most recent call last):' % self, - file=file) - else: - print('Stack for %r (most recent call last):' % self, - file=file) - traceback.print_list(extracted_list, file=file) - if exc is not None: - for line in traceback.format_exception_only(exc.__class__, exc): - print(line, file=file, end='') + return base_tasks._task_print_stack(self, limit, file) def cancel(self): """Request that this task cancel itself. @@ -316,6 +256,18 @@ self = None # Needed to break cycles when an exception occurs. +_PyTask = Task + + +try: + import _asyncio +except ImportError: + pass +else: + # _CTask is needed for tests. + Task = _CTask = _asyncio.Task + + # wait() and as_completed() similar to those in PEP 3148. FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED @@ -535,7 +487,8 @@ """ warnings.warn("asyncio.async() function is deprecated, use ensure_future()", - DeprecationWarning) + DeprecationWarning, + stacklevel=2) return ensure_future(coro_or_future, loop=loop) diff --git a/lib-python/3/asyncio/test_utils.py b/lib-python/3/asyncio/test_utils.py --- a/lib-python/3/asyncio/test_utils.py +++ b/lib-python/3/asyncio/test_utils.py @@ -119,10 +119,10 @@ 'test', 'test_asyncio') keyfile = os.path.join(here, 'ssl_key.pem') certfile = os.path.join(here, 'ssl_cert.pem') - ssock = ssl.wrap_socket(request, - keyfile=keyfile, - certfile=certfile, - server_side=True) + context = ssl.SSLContext() + context.load_cert_chain(certfile, keyfile) + + ssock = context.wrap_socket(request, server_side=True) try: self.RequestHandlerClass(ssock, client_address, self) ssock.close() @@ -449,12 +449,15 @@ self.set_event_loop(loop) return loop + def unpatch_get_running_loop(self): + events._get_running_loop = self._get_running_loop + def setUp(self): self._get_running_loop = events._get_running_loop events._get_running_loop = lambda: None def tearDown(self): - events._get_running_loop = self._get_running_loop + self.unpatch_get_running_loop() events.set_event_loop(None) diff --git a/lib-python/3/asyncio/unix_events.py b/lib-python/3/asyncio/unix_events.py --- a/lib-python/3/asyncio/unix_events.py +++ b/lib-python/3/asyncio/unix_events.py @@ -419,7 +419,8 @@ if compat.PY34: def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._pipe.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): @@ -619,7 +620,8 @@ if compat.PY34: def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._pipe.close() def abort(self): diff --git a/lib-python/3/asyncio/windows_events.py b/lib-python/3/asyncio/windows_events.py --- a/lib-python/3/asyncio/windows_events.py +++ b/lib-python/3/asyncio/windows_events.py @@ -171,8 +171,13 @@ def cancel(self): raise RuntimeError("_WaitCancelFuture must not be cancelled") - def _schedule_callbacks(self): - super(_WaitCancelFuture, self)._schedule_callbacks() + def set_result(self, result): + super().set_result(result) + if self._done_callback is not None: + self._done_callback(self) + + def set_exception(self, exception): + super().set_exception(exception) if self._done_callback is not None: self._done_callback(self) diff --git a/lib-python/3/asyncio/windows_utils.py b/lib-python/3/asyncio/windows_utils.py --- a/lib-python/3/asyncio/windows_utils.py +++ b/lib-python/3/asyncio/windows_utils.py @@ -159,7 +159,8 @@ def __del__(self): if self._handle is not None: - warnings.warn("unclosed %r" % self, ResourceWarning) + warnings.warn("unclosed %r" % self, ResourceWarning, + source=self) self.close() def __enter__(self): diff --git a/lib-python/3/asyncore.py b/lib-python/3/asyncore.py --- a/lib-python/3/asyncore.py +++ b/lib-python/3/asyncore.py @@ -333,7 +333,7 @@ self.connecting = True err = self.socket.connect_ex(address) if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ - or err == EINVAL and os.name in ('nt', 'ce'): + or err == EINVAL and os.name == 'nt': self.addr = address return if err in (0, EISCONN): @@ -595,7 +595,8 @@ def __del__(self): if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning) + warnings.warn("unclosed file %r" % self, ResourceWarning, + source=self) self.close() def recv(self, *args): diff --git a/lib-python/3/base64.py b/lib-python/3/base64.py --- a/lib-python/3/base64.py +++ b/lib-python/3/base64.py @@ -55,8 +55,7 @@ alternative alphabet for the '+' and '/' characters. This allows an application to e.g. generate url or filesystem safe Base64 strings. """ - # Strip off the trailing newline - encoded = binascii.b2a_base64(s)[:-1] + encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: assert len(altchars) == 2, repr(altchars) return encoded.translate(bytes.maketrans(b'+/', altchars)) @@ -156,7 +155,7 @@ leftover = len(s) % 5 # Pad the last quantum with zero bits if necessary if leftover: - s = s + bytes(5 - leftover) # Don't use += ! + s = s + b'\0' * (5 - leftover) # Don't use += ! encoded = bytearray() from_bytes = int.from_bytes b32tab2 = _b32tab2 @@ -542,7 +541,8 @@ def encodestring(s): """Legacy alias of encodebytes().""" import warnings - warnings.warn("encodestring() is a deprecated alias, use encodebytes()", + warnings.warn("encodestring() is a deprecated alias since 3.1, " + "use encodebytes()", DeprecationWarning, 2) return encodebytes(s) @@ -555,7 +555,8 @@ def decodestring(s): """Legacy alias of decodebytes().""" import warnings - warnings.warn("decodestring() is a deprecated alias, use decodebytes()", + warnings.warn("decodestring() is a deprecated alias since Python 3.1, " + "use decodebytes()", DeprecationWarning, 2) return decodebytes(s) diff --git a/lib-python/3/bz2.py b/lib-python/3/bz2.py --- a/lib-python/3/bz2.py +++ b/lib-python/3/bz2.py @@ -11,6 +11,7 @@ from builtins import open as _builtin_open import io +import os import warnings import _compression @@ -42,9 +43,9 @@ def __init__(self, filename, mode="r", buffering=None, compresslevel=9): """Open a bzip2-compressed file. - If filename is a str or bytes object, it gives the name - of the file to be opened. Otherwise, it should be a file object, - which will be used to read or write the compressed data. + If filename is a str, bytes, or PathLike object, it gives the + name of the file to be opened. Otherwise, it should be a file + object, which will be used to read or write the compressed data. mode can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can @@ -91,7 +92,7 @@ else: raise ValueError("Invalid mode: %r" % (mode,)) - if isinstance(filename, (str, bytes)): + if isinstance(filename, (str, bytes, os.PathLike)): self._fp = _builtin_open(filename, mode) self._closefp = True self._mode = mode_code @@ -99,7 +100,7 @@ self._fp = filename self._mode = mode_code else: - raise TypeError("filename must be a str or bytes object, or a file") + raise TypeError("filename must be a str, bytes, file or PathLike object") if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, @@ -289,8 +290,9 @@ encoding=None, errors=None, newline=None): """Open a bzip2-compressed file in binary or text mode. - The filename argument can be an actual filename (a str or bytes - object), or an existing file object to read from or write to. + The filename argument can be an actual filename (a str, bytes, or + PathLike object), or an existing file object to read from or write + to. The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode. diff --git a/lib-python/3/calendar.py b/lib-python/3/calendar.py --- a/lib-python/3/calendar.py +++ b/lib-python/3/calendar.py @@ -13,7 +13,9 @@ __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr"] + "timegm", "month_name", "month_abbr", "day_name", "day_abbr", + "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "LocaleHTMLCalendar", "weekheader"] # Exception raised for bad input (with string parameter for details) error = ValueError @@ -604,51 +606,63 @@ def main(args): - import optparse - parser = optparse.OptionParser(usage="usage: %prog [options] [year [month]]") - parser.add_option( + import argparse + parser = argparse.ArgumentParser() + textgroup = parser.add_argument_group('text only arguments') + htmlgroup = parser.add_argument_group('html only arguments') + textgroup.add_argument( "-w", "--width", - dest="width", type="int", default=2, - help="width of date column (default 2, text only)" + type=int, default=2, + help="width of date column (default 2)" ) - parser.add_option( + textgroup.add_argument( "-l", "--lines", - dest="lines", type="int", default=1, - help="number of lines for each week (default 1, text only)" + type=int, default=1, + help="number of lines for each week (default 1)" ) - parser.add_option( + textgroup.add_argument( "-s", "--spacing", - dest="spacing", type="int", default=6, - help="spacing between months (default 6, text only)" + type=int, default=6, + help="spacing between months (default 6)" ) - parser.add_option( + textgroup.add_argument( "-m", "--months", - dest="months", type="int", default=3, - help="months per row (default 3, text only)" + type=int, default=3, + help="months per row (default 3)" ) - parser.add_option( + htmlgroup.add_argument( "-c", "--css", - dest="css", default="calendar.css", - help="CSS to use for page (html only)" + default="calendar.css", + help="CSS to use for page" ) - parser.add_option( + parser.add_argument( "-L", "--locale", - dest="locale", default=None, + default=None, help="locale to be used from month and weekday names" ) - parser.add_option( + parser.add_argument( "-e", "--encoding", - dest="encoding", default=None, - help="Encoding to use for output." + default=None, + help="encoding to use for output" ) - parser.add_option( + parser.add_argument( "-t", "--type", - dest="type", default="text", + default="text", choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "year", + nargs='?', type=int, + help="year number (1-9999)" + ) + parser.add_argument( + "month", + nargs='?', type=int, + help="month number (1-12, text only)" + ) - (options, args) = parser.parse_args(args) + options = parser.parse_args(args[1:]) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -666,10 +680,10 @@ encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write - if len(args) == 1: + if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif len(args) == 2: - write(cal.formatyearpage(int(args[1]), **optdict)) + elif options.month is None: + write(cal.formatyearpage(options.year, **optdict)) else: parser.error("incorrect number of arguments") sys.exit(1) @@ -679,18 +693,15 @@ else: cal = TextCalendar() optdict = dict(w=options.width, l=options.lines) - if len(args) != 3: + if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if len(args) == 1: + if options.year is None: result = cal.formatyear(datetime.date.today().year, **optdict) - elif len(args) == 2: - result = cal.formatyear(int(args[1]), **optdict) - elif len(args) == 3: - result = cal.formatmonth(int(args[1]), int(args[2]), **optdict) + elif options.month is None: + result = cal.formatyear(options.year, **optdict) else: - parser.error("incorrect number of arguments") - sys.exit(1) + result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: result = result.encode(options.encoding) diff --git a/lib-python/3/cgi.py b/lib-python/3/cgi.py --- a/lib-python/3/cgi.py +++ b/lib-python/3/cgi.py @@ -45,7 +45,7 @@ __all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_qs", "parse_qsl", "parse_multipart", - "parse_header", "print_exception", "print_environ", + "parse_header", "test", "print_exception", "print_environ", "print_form", "print_directory", "print_arguments", "print_environ_usage", "escape"] diff --git a/lib-python/3/code.py b/lib-python/3/code.py --- a/lib-python/3/code.py +++ b/lib-python/3/code.py @@ -186,7 +186,7 @@ """Reset the input buffer.""" self.buffer = [] - def interact(self, banner=None): + def interact(self, banner=None, exitmsg=None): """Closely emulate the interactive Python console. The optional banner argument specifies the banner to print @@ -196,6 +196,11 @@ to confuse this with the real interpreter -- since it's so close!). + The optional exitmsg argument specifies the exit message + printed when exiting. Pass the empty string to suppress + printing an exit message. If exitmsg is not given or None, + a default message is printed. + """ try: sys.ps1 @@ -230,6 +235,10 @@ self.write("\nKeyboardInterrupt\n") self.resetbuffer() more = 0 + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) def push(self, line): """Push a line to the interpreter. @@ -267,7 +276,7 @@ -def interact(banner=None, readfunc=None, local=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -279,6 +288,7 @@ banner -- passed to InteractiveConsole.interact() readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() + exitmsg -- passed to InteractiveConsole.interact() """ console = InteractiveConsole(local) @@ -289,7 +299,7 @@ import readline except ImportError: pass - console.interact(banner) + console.interact(banner, exitmsg) if __name__ == "__main__": diff --git a/lib-python/3/collections/__init__.py b/lib-python/3/collections/__init__.py --- a/lib-python/3/collections/__init__.py +++ b/lib-python/3/collections/__init__.py @@ -356,7 +356,7 @@ {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}') ''' -def namedtuple(typename, field_names, verbose=False, rename=False): +def namedtuple(typename, field_names, *, verbose=False, rename=False, module=None): """Returns a new subclass of tuple with named fields. >>> Point = namedtuple('Point', ['x', 'y']) @@ -396,7 +396,7 @@ field_names[index] = '_%d' % index seen.add(name) for name in [typename] + field_names: - if type(name) != str: + if type(name) is not str: raise TypeError('Type names and field names must be strings') if not name.isidentifier(): raise ValueError('Type names and field names must be valid ' @@ -437,11 +437,15 @@ # For pickling to work, the __module__ variable needs to be set to the frame # where the named tuple is created. Bypass this step in environments where # sys._getframe is not defined (Jython for example) or sys._getframe is not - # defined for arguments greater than 0 (IronPython). - try: - result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') - except (AttributeError, ValueError): - pass + # defined for arguments greater than 0 (IronPython), or where the user has + # specified a particular module. + if module is None: + try: + module = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + result.__module__ = module return result From pypy.commits at gmail.com Mon Mar 19 12:07:35 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 09:07:35 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Extend struct half-float test with native formats. Message-ID: <5aafe047.8aaa1c0a.97238.5a64@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r94006:e4575f135d6f Date: 2018-03-19 17:02 +0100 http://bitbucket.org/pypy/pypy/changeset/e4575f135d6f/ Log: Extend struct half-float test with native formats. diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py --- a/pypy/module/struct/test/test_struct.py +++ b/pypy/module/struct/test/test_struct.py @@ -258,12 +258,19 @@ raises(OverflowError, pack, "e", 65504.0) == b'\x7b\xff' assert unpack(">e", b'\x7b\xff') == (65504.0,) raises(OverflowError, pack, " Author: Manuel Jacob Branch: py3.6 Changeset: r94008:760c10a5b6d8 Date: 2018-03-19 17:14 +0100 http://bitbucket.org/pypy/pypy/changeset/760c10a5b6d8/ Log: Extend struct half-float test with native formats. diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py --- a/pypy/module/struct/test/test_struct.py +++ b/pypy/module/struct/test/test_struct.py @@ -258,12 +258,19 @@ raises(OverflowError, pack, "e", 65504.0) == b'\x7b\xff' assert unpack(">e", b'\x7b\xff') == (65504.0,) raises(OverflowError, pack, " Author: Manuel Jacob Branch: py3.6 Changeset: r94007:a756aa082d6b Date: 2018-03-19 17:14 +0100 http://bitbucket.org/pypy/pypy/changeset/a756aa082d6b/ Log: hg merge py3.5 diff too long, truncating to 2000 out of 2599 lines diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -223,6 +223,7 @@ that fails it falls back to the VS90COMNTOOLS env var. """ vsbase = VS_BASE % version + batfile = 'vcvarsall.bat' try: productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, "productdir") @@ -235,9 +236,14 @@ toolsdir = os.environ.get(toolskey, None) if toolsdir and os.path.isdir(toolsdir): - productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") - productdir = os.path.abspath(productdir) + if os.path.exists(os.path.join(toolsdir, 'VsDevCmd.bat')): + productdir = toolsdir + batfile = 'VsDevCmd.bat' + else: + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) return None else: @@ -245,7 +251,7 @@ if not productdir: log.debug("No productdir found") return None - vcvarsall = os.path.join(productdir, "vcvarsall.bat") + vcvarsall = os.path.join(productdir, batfile) if os.path.isfile(vcvarsall): return vcvarsall log.debug("Unable to find vcvarsall.bat") @@ -289,6 +295,7 @@ if len(result) != len(interesting): raise ValueError(str(list(result.keys()))) + log.debug('Got', result) return result # More globals diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py --- a/lib_pypy/_dbm.py +++ b/lib_pypy/_dbm.py @@ -168,7 +168,14 @@ def open(filename, flag='r', mode=0o666): "open a DBM database" if not isinstance(filename, str): - raise TypeError("expected string") + if sys.version_info < (3,) and isinstance(filename, unicode): + # unlike CPython we'll encode 'filename' with filesystemencoding + # instead of defaultencoding, because that seems like a far + # better idea. But I'm also open for saying that we should + # rather go for bug-to-bug compatibility instead. + filename = filename.encode(sys.getfilesystemencoding()) + else: + raise TypeError("expected string") filename = filename.encode(sys.getdefaultencoding()) openflag = 0 diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -3,7 +3,7 @@ import importlib.machinery -def get_hashed_dir(cfile): +def _get_hashed_filename(cfile): with open(cfile,'r') as fid: content = fid.read() # from cffi's Verifier() @@ -23,10 +23,28 @@ username = os.environ['USERNAME'] #windows except KeyError: username = os.getuid() - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + return tempfile.gettempdir() + os.path.sep + 'testcapi_%s_%s%s' % ( username, k1, k2) - if not os.path.exists(output_dir): + +def get_hashed_dir(cfile): + hashed_fn = _get_hashed_filename(cfile) + try: + with open(hashed_fn) as f: + dirname = f.read(1024) + except IOError: + dirname = '' + tmpdir = tempfile.gettempdir() + if (not dirname or '/' in dirname or '\\' in dirname or '\x00' in dirname + or not os.path.isdir(os.path.join(tmpdir, dirname))): + dirname = binascii.hexlify(os.urandom(8)) + if not isinstance(dirname, str): # Python 3 + dirname = dirname.decode('ascii') + dirname = 'testcapi_' + dirname + output_dir = os.path.join(tmpdir, dirname) + try: os.mkdir(output_dir) + except OSError: + pass return output_dir @@ -35,13 +53,12 @@ return suffixes[0] if suffixes else None -def compile_shared(csource, modulename, output_dir=None): +def compile_shared(csource, modulename, output_dir): """Compile '_testcapi.c' or '_ctypes_test.c' into an extension module, and import it. """ thisdir = os.path.dirname(__file__) - if output_dir is None: - output_dir = tempfile.mkdtemp() + assert output_dir is not None from distutils.ccompiler import new_compiler @@ -85,4 +102,16 @@ # Now import the newly created library, it will replace the original # module in sys.modules fp, filename, description = imp.find_module(modulename, path=[output_dir]) - imp.load_module(modulename, fp, filename, description) + with fp: + imp.load_module(modulename, fp, filename, description) + + # If everything went fine up to now, write the name of this new + # directory to 'hashed_fn', for future processes (and to avoid a + # growing number of temporary directories that are not completely + # obvious to clean up on Windows) + hashed_fn = _get_hashed_filename(os.path.join(thisdir, csource)) + try: + with open(hashed_fn, 'w') as f: + f.write(os.path.basename(output_dir)) + except IOError: + pass diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.11.4 +Version: 1.11.5 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI from .error import CDefError, FFIError, VerificationError, VerificationMissing -__version__ = "1.11.4" -__version_info__ = (1, 11, 4) +__version__ = "1.11.5" +__version_info__ = (1, 11, 5) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h --- a/lib_pypy/cffi/_embedding.h +++ b/lib_pypy/cffi/_embedding.h @@ -146,32 +146,6 @@ PyGILState_STATE state; PyObject *pycode=NULL, *global_dict=NULL, *x; -#if PY_MAJOR_VERSION >= 3 - /* see comments in _cffi_carefully_make_gil() about the - Python2/Python3 difference - */ -#else - /* Acquire the GIL. We have no threadstate here. If Python is - already initialized, it is possible that there is already one - existing for this thread, but it is not made current now. - */ - PyEval_AcquireLock(); - - _cffi_py_initialize(); - - /* The Py_InitializeEx() sometimes made a threadstate for us, but - not always. Indeed Py_InitializeEx() could be called and do - nothing. So do we have a threadstate, or not? We don't know, - but we can replace it with NULL in all cases. - */ - (void)PyThreadState_Swap(NULL); - - /* Now we can release the GIL and re-acquire immediately using the - logic of PyGILState(), which handles making or installing the - correct threadstate. - */ - PyEval_ReleaseLock(); -#endif state = PyGILState_Ensure(); /* Call the initxxx() function from the present module. It will @@ -247,7 +221,7 @@ if (f != NULL && f != Py_None) { PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME - "\ncompiled with cffi version: 1.11.4" + "\ncompiled with cffi version: 1.11.5" "\n_cffi_backend module: ", f); modules = PyImport_GetModuleDict(); mod = PyDict_GetItemString(modules, "_cffi_backend"); @@ -278,16 +252,14 @@ that we don't hold the GIL before (if it exists), and we don't hold it afterwards. - What it really does is completely different in Python 2 and - Python 3. + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) - Python 2 - ======== - - Initialize the GIL, without initializing the rest of Python, - by calling PyEval_InitThreads(). - - PyEval_InitThreads() must not be called concurrently at all. + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. So we use a global variable as a simple spin lock. This global variable must be from 'libpythonX.Y.so', not from this cffi-based extension module, because it must be shared from @@ -297,18 +269,6 @@ string "ENDMARKER". We change it temporarily to point to the next character in that string. (Yes, I know it's REALLY obscure.) - - Python 3 - ======== - - In Python 3, PyEval_InitThreads() cannot be called before - Py_InitializeEx() any more. So this function calls - Py_InitializeEx() first. It uses the same obscure logic to - make sure we never call it concurrently. - - Arguably, this is less good on the spinlock, because - Py_InitializeEx() takes much longer to run than - PyEval_InitThreads(). But I didn't find a way around it. */ #ifdef WITH_THREAD @@ -332,8 +292,7 @@ } #endif -#if PY_MAJOR_VERSION >= 3 - /* Python 3: call Py_InitializeEx() */ + /* call Py_InitializeEx() */ { PyGILState_STATE state = PyGILState_UNLOCKED; if (!Py_IsInitialized()) @@ -344,17 +303,6 @@ PyEval_InitThreads(); PyGILState_Release(state); } -#else - /* Python 2: call PyEval_InitThreads() */ -# ifdef WITH_THREAD - if (!PyEval_ThreadsInitialized()) { - PyEval_InitThreads(); /* makes the GIL */ - PyEval_ReleaseLock(); /* then release it */ - } - /* else: there is already a GIL, but we still needed to do the - spinlock dance to make sure that we see it as fully ready */ -# endif -#endif #ifdef WITH_THREAD /* release the lock */ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -143,6 +143,13 @@ self._libraries.append(lib) return lib + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + def _typeof_locked(self, cdecl): # call me with the lock! key = cdecl @@ -898,6 +905,9 @@ return addressof_var(name) raise AttributeError("cffi library has no function or " "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() # if libname is not None: try: diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -352,21 +352,20 @@ self.fldquals = fldquals self.build_c_name_with_marker() - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type - def enumfields(self): + def enumfields(self, expand_anonymous_struct_union=True): fldquals = self.fldquals if fldquals is None: fldquals = (0,) * len(self.fldnames) for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): # nested anonymous struct/union for result in type.enumfields(): yield result diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -836,6 +836,10 @@ def _struct_collecttype(self, tp): self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) def _struct_decl(self, tp, cname, approxname): if tp.fldtypes is None: @@ -884,7 +888,7 @@ named_ptr not in self.ffi._parser._included_declarations)): if tp.fldtypes is None: pass # opaque - elif tp.partial or tp.has_anonymous_struct_fields(): + elif tp.partial or any(tp.anonymous_struct_fields()): pass # field layout obtained silently from the C compiler else: flags.append("_CFFI_F_CHECK_FIELDS") @@ -896,7 +900,8 @@ flags = '|'.join(flags) or '0' c_fields = [] if reason_for_not_expanding is None: - enumfields = list(tp.enumfields()) + expand_anonymous_struct_union = not self.target_is_python + enumfields = list(tp.enumfields(expand_anonymous_struct_union)) for fldname, fldtype, fbitsize, fqual in enumfields: fldtype = self._field_type(tp, fldname, fldtype) self._check_not_opaque(fldtype, diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,8 +81,13 @@ it doesn't so far, creating troubles. That's why we check for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, it's better not to use py_limited_api until issue #355 + can be resolved (by having virtualenv copy PYTHON3.DLL). See also + the start of _cffi_include.h. """ - if 'py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount'): + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and sys.platform != 'win32'): import setuptools try: setuptools_major_version = int(setuptools.__version__.partition('.')[0]) @@ -143,8 +148,8 @@ def _add_py_module(dist, ffi, module_name): from distutils.dir_util import mkpath - from distutils.command.build_py import build_py - from distutils.command.build_ext import build_ext + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext from distutils import log from cffi import recompiler @@ -164,6 +169,17 @@ generate_mod(os.path.join(self.build_lib, *module_path)) dist.cmdclass['build_py'] = build_py_make_mod + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + # the following is only for "build_ext -i" base_class_2 = dist.cmdclass.get('build_ext', build_ext) class build_ext_make_mod(base_class_2): diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -567,6 +567,11 @@ versions of PyPy may have to rename the arguments if CPython starts accepting them too. +* PyPy3: ``distutils`` has been enhanced to allow finding ``VsDevCmd.bat`` in the + directory pointed to by the ``VS%0.f0COMNTOOLS`` (typically ``VS140COMNTOOLS``) + environment variable. CPython searches for ``vcvarsall.bat`` somewhere **above** + that value. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -48,3 +48,17 @@ .. branch: winapi Update _winapi and internal _winbase_cffi (via _winbase_build) for python 3 + +.. branch: refactor-slots + +Refactor cpyext slots. + + +.. branch: call-loopinvariant-into-bridges + +Speed up branchy code that does a lot of function inlining by saving one call +to read the TLS in most bridges. + +.. branch: rpython-sprint + +Refactor in rpython signatures diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py --- a/pypy/interpreter/test/test_zpy.py +++ b/pypy/interpreter/test/test_zpy.py @@ -113,6 +113,7 @@ def test_pytrace(): output = run(sys.executable, pypypath, '-S', stdin="__pytrace__ = 1\nx = 5\nx") + output = output.replace('\r\n', '\n') assert ('\t: LOAD_CONST 0 (5)\n' '\t: STORE_NAME 0 (x)\n' '\t: LOAD_CONST 1 (None)\n' diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi from rpython.rtyper.lltypesystem import rffi -VERSION = "1.11.4" +VERSION = "1.11.5" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -53,8 +53,7 @@ self.libhandle = rffi.cast(DLLHANDLE, 0) if not libhandle: - raise oefmt(self.ffi.w_FFIError, "library '%s' is already closed", - self.libname) + return self.may_unregister_rpython_finalizer(self.ffi.space) # Clear the dict to force further accesses to do cdlopen_fetch() diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -38,9 +38,16 @@ space = self.space return space.newtext("" % self.name) + def check_closed(self): + if self.handle == rffi.cast(DLLHANDLE, 0): + raise oefmt(self.space.w_ValueError, + "library '%s' has already been closed", + self.name) + @unwrap_spec(w_ctype=W_CType, name='text') def load_function(self, w_ctype, name): from pypy.module._cffi_backend import ctypeptr, ctypearray + self.check_closed() space = self.space # if not isinstance(w_ctype, ctypeptr.W_CTypePtrOrArray): @@ -60,6 +67,7 @@ @unwrap_spec(w_ctype=W_CType, name='text') def read_variable(self, w_ctype, name): + self.check_closed() space = self.space try: cdata = dlsym(self.handle, name) @@ -71,6 +79,7 @@ @unwrap_spec(w_ctype=W_CType, name='text') def write_variable(self, w_ctype, name, w_value): + self.check_closed() space = self.space try: cdata = dlsym(self.handle, name) @@ -80,6 +89,9 @@ name, self.name) w_ctype.convert_from_object(rffi.cast(rffi.CCHARP, cdata), w_value) + def close_lib(self): + self._finalize_() + W_Library.typedef = TypeDef( '_cffi_backend.Library', @@ -87,6 +99,7 @@ load_function = interp2app(W_Library.load_function), read_variable = interp2app(W_Library.read_variable), write_variable = interp2app(W_Library.write_variable), + close_lib = interp2app(W_Library.close_lib), ) W_Library.typedef.acceptable_as_base_class = False diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.11.4", ("This test_c.py file is for testing a version" +assert __version__ == "1.11.5", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -395,6 +395,10 @@ # the next one is from 'libm', not 'libc', but we assume # that it is already loaded too, so it should work assert x.load_function(BVoidP, 'sqrt') + # + x.close_lib() + py.test.raises(ValueError, x.load_function, BVoidP, 'sqrt') + x.close_lib() def test_no_len_on_nonarray(): p = new_primitive_type("int") @@ -1210,6 +1214,9 @@ ll = find_and_load_library('c') stderr = ll.read_variable(BVoidP, "stderr") assert stderr == cast(BVoidP, _testfunc(8)) + # + ll.close_lib() + py.test.raises(ValueError, ll.read_variable, BVoidP, "stderr") def test_read_variable_as_unknown_length_array(): ## FIXME: this test assumes glibc specific behavior, it's not compliant with C standard @@ -1236,6 +1243,9 @@ assert not ll.read_variable(BVoidP, "stderr") ll.write_variable(BVoidP, "stderr", stderr) assert ll.read_variable(BVoidP, "stderr") == stderr + # + ll.close_lib() + py.test.raises(ValueError, ll.write_variable, BVoidP, "stderr", stderr) def test_callback(): BInt = new_primitive_type("int") diff --git a/pypy/module/_cffi_backend/test/test_re_python.py b/pypy/module/_cffi_backend/test/test_re_python.py --- a/pypy/module/_cffi_backend/test/test_re_python.py +++ b/pypy/module/_cffi_backend/test/test_re_python.py @@ -114,12 +114,10 @@ from re_python_pysrc import ffi lib = ffi.dlopen(self.extmod) ffi.dlclose(lib) - e = raises(ffi.error, ffi.dlclose, lib) - assert str(e.value) == ( - "library '%s' is already closed" % (self.extmod,)) e = raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( "library '%s' has been closed" % (self.extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(self): self.fix_path() diff --git a/pypy/module/_posixsubprocess/test/test_subprocess.py b/pypy/module/_posixsubprocess/test/test_subprocess.py --- a/pypy/module/_posixsubprocess/test/test_subprocess.py +++ b/pypy/module/_posixsubprocess/test/test_subprocess.py @@ -1,4 +1,9 @@ from os.path import dirname +import py, sys + +if sys.platform == 'win32': + py.test.skip("not used on win32") + class AppTestSubprocess: spaceconfig = dict(usemodules=('_posixsubprocess', 'signal', diff --git a/pypy/module/_posixsubprocess/test/test_ztranslation.py b/pypy/module/_posixsubprocess/test/test_ztranslation.py --- a/pypy/module/_posixsubprocess/test/test_ztranslation.py +++ b/pypy/module/_posixsubprocess/test/test_ztranslation.py @@ -1,4 +1,8 @@ from pypy.objspace.fake.checkmodule import checkmodule +import py, sys + +if sys.platform == 'win32': + py.test.skip("not used on win32") def test_posixsubprocess_translates(): checkmodule('_posixsubprocess') diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -813,10 +813,10 @@ def test_recv_send_timeout(self): from _socket import socket, timeout, SOL_SOCKET, SO_RCVBUF, SO_SNDBUF cli = socket() + cli.settimeout(1.0) cli.connect(self.serv.getsockname()) fileno, addr = self.serv._accept() t = socket(fileno=fileno) - cli.settimeout(1.0) # test recv() timeout t.send(b'*') buf = cli.recv(100) diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -407,8 +407,11 @@ if not (last_pos == ctx.match_start == ctx.match_end and n > 0): # the above ignores empty matches on latest position + last_pos = ctx.match_end if filter_is_callable: w_match = self.getmatch(ctx, True) + # make a copy of 'ctx'; see test_sub_matches_stay_valid + ctx = ctx.fresh_copy(start) # match_start/match_end dropped w_piece = space.call_function(w_filter, w_match) if not space.is_w(w_piece, space.w_None): assert strbuilder is None and unicodebuilder is None @@ -425,7 +428,6 @@ unicodebuilder.append(filter_as_unicode) else: sublist_w.append(w_filter) - last_pos = ctx.match_end n += 1 elif last_pos >= ctx.end: break # empty match at the end: finished diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -395,6 +395,18 @@ KEYCRE = re.compile(r"%\(([^)]*)\)s|.") raises(TypeError, KEYCRE.sub, "hello", {"%(": 1}) + def test_sub_matches_stay_valid(self): + import re + matches = [] + def callback(match): + matches.append(match) + return "x" + result = re.compile(r"[ab]").sub(callback, "acb") + assert result == "xcx" + assert len(matches) == 2 + assert matches[0].group() == "a" + assert matches[1].group() == "b" + class AppTestSreScanner: diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -175,7 +175,7 @@ c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with rffi.scoped_unicode2wcharp(filename) as wide_filename: c_filename = rffi.cast(rffi.CCHARP, wide_filename) - ret = rwinreg.RegLoadKey(hkey, c_subkey, c_filename) + ret = rwinreg.RegLoadKeyW(hkey, c_subkey, c_filename) if ret != 0: raiseWindowsError(space, ret, 'RegLoadKey') @@ -196,7 +196,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(filename) as wide_filename: c_filename = rffi.cast(rffi.CCHARP, wide_filename) - ret = rwinreg.RegSaveKey(hkey, c_filename, None) + ret = rwinreg.RegSaveKeyW(hkey, c_filename, None) if ret != 0: raiseWindowsError(space, ret, 'RegSaveKey') @@ -226,7 +226,7 @@ c_subkey = rffi.cast(rffi.CCHARP, subkey) with rffi.scoped_unicode2wcharp(value) as dataptr: c_dataptr = rffi.cast(rffi.CCHARP, dataptr) - ret = rwinreg.RegSetValue(hkey, c_subkey, rwinreg.REG_SZ, + ret = rwinreg.RegSetValueW(hkey, c_subkey, rwinreg.REG_SZ, c_dataptr, len(value)) if ret != 0: raiseWindowsError(space, ret, 'RegSetValue') @@ -250,7 +250,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwin32.PLONG.TO, 1) as bufsize_p: - ret = rwinreg.RegQueryValue(hkey, c_subkey, None, bufsize_p) + ret = rwinreg.RegQueryValueW(hkey, c_subkey, None, bufsize_p) bufSize = intmask(bufsize_p[0]) if ret == rwinreg.ERROR_MORE_DATA: bufSize = 256 @@ -259,7 +259,7 @@ while True: with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as buf: - ret = rwinreg.RegQueryValue(hkey, c_subkey, buf, bufsize_p) + ret = rwinreg.RegQueryValueW(hkey, c_subkey, buf, bufsize_p) if ret == rwinreg.ERROR_MORE_DATA: print 'bufSize was %d, too small' % bufSize # Resize and retry @@ -440,7 +440,7 @@ try: with rffi.scoped_unicode2wcharp(value_name) as wide_vn: c_vn = rffi.cast(rffi.CCHARP, wide_vn) - ret = rwinreg.RegSetValueEx(hkey, c_vn, 0, typ, buf, buflen) + ret = rwinreg.RegSetValueExW(hkey, c_vn, 0, typ, buf, buflen) finally: lltype.free(buf, flavor='raw') if ret != 0: @@ -460,7 +460,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, null_dword, + ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword, null_dword, None, retDataSize) bufSize = intmask(retDataSize[0]) if ret == rwinreg.ERROR_MORE_DATA: @@ -472,7 +472,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, bufSize) as databuf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: - ret = rwinreg.RegQueryValueEx(hkey, c_subkey, null_dword, + ret = rwinreg.RegQueryValueExW(hkey, c_subkey, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry @@ -505,7 +505,7 @@ with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKey(hkey, c_subkey, rethkey) + ret = rwinreg.RegCreateKeyW(hkey, c_subkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'CreateKey') return W_HKEY(space, rethkey[0]) @@ -527,7 +527,7 @@ with rffi.scoped_unicode2wcharp(sub_key) as wide_sub_key: c_subkey = rffi.cast(rffi.CCHARP, wide_sub_key) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegCreateKeyEx(hkey, c_subkey, reserved, None, 0, + ret = rwinreg.RegCreateKeyExW(hkey, c_subkey, reserved, None, 0, access, None, rethkey, lltype.nullptr(rwin32.LPDWORD.TO)) if ret != 0: @@ -549,7 +549,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) - ret = rwinreg.RegDeleteKey(hkey, c_subkey) + ret = rwinreg.RegDeleteKeyW(hkey, c_subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteKey') @@ -562,7 +562,7 @@ hkey = hkey_w(w_hkey, space) with rffi.scoped_unicode2wcharp(subkey) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) - ret = rwinreg.RegDeleteValue(hkey, c_subkey) + ret = rwinreg.RegDeleteValueW(hkey, c_subkey) if ret != 0: raiseWindowsError(space, ret, 'RegDeleteValue') @@ -582,7 +582,7 @@ with rffi.scoped_unicode2wcharp(sub_key) as wide_subkey: c_subkey = rffi.cast(rffi.CCHARP, wide_subkey) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegOpenKeyEx(hkey, c_subkey, reserved, access, rethkey) + ret = rwinreg.RegOpenKeyExW(hkey, c_subkey, reserved, access, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegOpenKeyEx') return W_HKEY(space, rethkey[0]) @@ -607,7 +607,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retDataSize: - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyW( hkey, None, null_dword, null_dword, null_dword, null_dword, null_dword, null_dword, retValueSize, retDataSize, @@ -628,7 +628,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retType: c_valuebuf = rffi.cast(rffi.CCHARP, valuebuf) - ret = rwinreg.RegEnumValue( + ret = rwinreg.RegEnumValueW( hkey, index, c_valuebuf, retValueSize, null_dword, retType, databuf, retDataSize) if ret == rwinreg.ERROR_MORE_DATA: @@ -673,7 +673,7 @@ with lltype.scoped_alloc(rffi.CCHARP.TO, 257) as buf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: retValueSize[0] = r_uint(257) # includes NULL terminator - ret = rwinreg.RegEnumKeyEx(hkey, index, buf, retValueSize, + ret = rwinreg.RegEnumKeyExW(hkey, index, buf, retValueSize, null_dword, None, null_dword, lltype.nullptr(rwin32.PFILETIME.TO)) if ret != 0: @@ -695,7 +695,7 @@ with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as nValues: with lltype.scoped_alloc(rwin32.PFILETIME.TO, 1) as ft: null_dword = lltype.nullptr(rwin32.LPDWORD.TO) - ret = rwinreg.RegQueryInfoKey( + ret = rwinreg.RegQueryInfoKeyW( hkey, None, null_dword, null_dword, nSubKeys, null_dword, null_dword, nValues, null_dword, null_dword, @@ -722,7 +722,7 @@ machine = space.text_or_none_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: - ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) + ret = rwinreg.RegConnectRegistryW(machine, hkey, rethkey) if ret != 0: raiseWindowsError(space, ret, 'RegConnectRegistry') return W_HKEY(space, rethkey[0]) diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -207,18 +207,20 @@ except: pass - key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) - SaveKey(key, self.tmpfilename) + with OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) as key: + SaveKey(key, self.tmpfilename) def test_expand_environment_string(self): from winreg import ExpandEnvironmentStrings import nt r = ExpandEnvironmentStrings("%windir%\\test") assert isinstance(r, str) - if 'WINDIR' in list(nt.environ.keys()): + if 'WINDIR' in nt.environ: assert r == nt.environ["WINDIR"] + "\\test" + elif 'windir' in nt.environ: + assert r == nt.environ["windir"] + "\\test" else: - assert r == nt.environ["windir"] + "\\test" + skip('nt.environ not filled in for untranslated tests') def test_long_key(self): from winreg import ( diff --git a/pypy/module/cpyext/buffer.py b/pypy/module/cpyext/buffer.py --- a/pypy/module/cpyext/buffer.py +++ b/pypy/module/cpyext/buffer.py @@ -189,33 +189,6 @@ decref(space, view.c_obj) return 0 -def fill_buffer(space, view, pybuf, py_obj): - view.c_buf = cts.cast('void *', pybuf.get_raw_address()) - view.c_obj = py_obj - if py_obj: - incref(space, py_obj) - view.c_len = pybuf.getlength() - view.c_itemsize = pybuf.getitemsize() - rffi.setintfield(view, 'c_readonly', int(pybuf.readonly)) - rffi.setintfield(view, 'c_ndim', pybuf.getndim()) - view.c_format = rffi.str2charp(pybuf.getformat()) - shape = pybuf.getshape() - if not shape: - view.c_shape = lltype.nullptr(Py_ssize_tP.TO) - else: - view.c_shape = cts.cast('Py_ssize_t*', view.c__shape) - for i, n in enumerate(shape): - view.c_shape[i] = n - strides = pybuf.getstrides() - if not strides: - view.c_strides = lltype.nullptr(Py_ssize_tP.TO) - else: - view.c_strides = cts.cast('Py_ssize_t*', view.c__strides) - for i, n in enumerate(strides): - view.c_strides[i] = n - view.c_suboffsets = lltype.nullptr(Py_ssize_tP.TO) - view.c_internal = lltype.nullptr(rffi.VOIDP.TO) - DEFAULT_FMT = rffi.str2charp("B") @cpython_api([lltype.Ptr(Py_buffer), PyObject, rffi.VOIDP, Py_ssize_t, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -688,12 +688,46 @@ return space.call_args(space.get(new_fn, w_self), args) return slot_tp_new + at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], + rffi.INT_real, error=-1) +def bytes_getbuffer(space, w_str, view, flags): + from pypy.module.cpyext.bytesobject import PyBytes_AsString + from pypy.module.cpyext.buffer import PyBuffer_FillInfo + c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str)) + return PyBuffer_FillInfo(space, view, w_str, c_buf, + space.len_w(w_str), 1, flags) + +def slot_from_buffer_w(space, typedef): + name = 'bf_getbuffer' + @slot_function([PyObject, Py_bufferP, rffi.INT_real], + rffi.INT_real, error=-1) + @func_renamer("cpyext_%s_%s" % (name, typedef.name)) + def buff_w(space, w_self, c_view, flags): + w_obj = w_self + if c_view: + #like PyObject_GetBuffer + flags = widen(flags) + buf = space.buffer_w(w_obj, flags) + try: + c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) + c_view.c_obj = make_ref(space, w_obj) + except ValueError: + s = buf.as_str() + w_s = space.newbytes(s) + c_view.c_obj = make_ref(space, w_s) + c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( + s, track_allocation=False)) + rffi.setintfield(c_view, 'c_readonly', 1) + ret = fill_Py_buffer(space, buf, c_view) + return ret + return 0 + return buff_w + @slot_factory('tp_as_buffer.c_bf_getbuffer') def make_bf_getbuffer(space, typedef, name, attr): w_type = space.gettypeobject(typedef) - buff_fn = w_type.lookup('__buffer__') - if buff_fn is not None: - return slot_from___buffer__(space, typedef, buff_fn) + if space.is_w(w_type, space.w_bytes): + return bytes_getbuffer elif typedef.buffer: return slot_from_buffer_w(space, typedef) else: @@ -739,59 +773,6 @@ return slot_tp_descr_set -def slot_from___buffer__(space, typedef, buff_fn): - name = 'bf_getbuffer' - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name, typedef.name)) - def buff_w(space, w_self, c_view, flags): - args = Arguments(space, [space.newint(flags)]) - w_obj = space.call_args(space.get(buff_fn, w_self), args) - if c_view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - c_view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - c_view.c_obj = make_ref(space, w_s) - c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(c_view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, c_view) - return ret - return 0 - return buff_w - -def slot_from_buffer_w(space, typedef): - name = 'bf_getbuffer' - @slot_function([PyObject, Py_bufferP, rffi.INT_real], - rffi.INT_real, error=-1) - @func_renamer("cpyext_%s_%s" % (name, typedef.name)) - def buff_w(space, w_self, c_view, flags): - w_obj = w_self - if c_view: - #like PyObject_GetBuffer - flags = widen(flags) - buf = space.buffer_w(w_obj, flags) - try: - c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) - c_view.c_obj = make_ref(space, w_obj) - except ValueError: - s = buf.as_str() - w_s = space.newbytes(s) - c_view.c_obj = make_ref(space, w_s) - c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp( - s, track_allocation=False)) - rffi.setintfield(c_view, 'c_readonly', 1) - ret = fill_Py_buffer(space, buf, c_view) - return ret - return 0 - return buff_w - missing_wrappers = ['wrap_indexargfunc', 'wrap_del'] for name in missing_wrappers: assert name not in globals() diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -640,6 +640,33 @@ self.attr1 = 123 assert module.test_tp_getattro(C(), 123) + def test_issue_2760_getattr(self): + module = self.import_extension('foo', [ + ("get_foo", "METH_O", + ''' + char* name = "foo"; + PyTypeObject *tp = Py_TYPE(args); + PyObject *res; + if (tp->tp_getattr != NULL) { + res = (*tp->tp_getattr)(args, name); + } + else if (tp->tp_getattro != NULL) { + PyObject *w = PyUnicode_FromString(name); + res = (*tp->tp_getattro)(args, w); + Py_DECREF(w); + } + else { + res = Py_None; + } + return res; + ''')]) + class Passthrough(object): + def __getattr__(self, name): + return name + + obj = Passthrough() + assert module.get_foo(obj) == 'foo' + def test_nb_int(self): module = self.import_extension('foo', [ ("nb_int", "METH_VARARGS", diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -236,11 +236,6 @@ def update_all_slots(space, w_type, pto): # fill slots in pto - # Not very sure about it, but according to - # test_call_tp_dealloc, we should not - # overwrite slots that are already set: these ones are probably - # coming from a parent C type. - for method_name, slot_name, slot_names, slot_apifunc in slotdefs_for_tp_slots: slot_func_helper = None w_descr = w_type.dict_w.get(method_name, None) @@ -276,8 +271,7 @@ def fill_slot(space, pto, w_type, slot_names, slot_func_helper): # XXX special case wrapper-functions and use a "specific" slot func if len(slot_names) == 1: - if not getattr(pto, slot_names[0]): - setattr(pto, slot_names[0], slot_func_helper) + setattr(pto, slot_names[0], slot_func_helper) elif ((w_type is space.w_list or w_type is space.w_tuple) and slot_names[0] == 'c_tp_as_number'): # XXX hack - how can we generalize this? The problem is method @@ -312,8 +306,7 @@ struct = lltype.malloc(STRUCT_TYPE, flavor='raw', zero=True) setattr(pto, slot_names[0], struct) - if not getattr(struct, slot_names[1]): - setattr(struct, slot_names[1], slot_func_helper) + setattr(struct, slot_names[1], slot_func_helper) def add_operators(space, dict_w, pto, name): from pypy.module.cpyext.object import PyObject_HashNotImplemented @@ -526,33 +519,6 @@ realize=type_realize, dealloc=type_dealloc) - at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) -def bytes_getbuffer(space, w_str, view, flags): - from pypy.module.cpyext.bytesobject import PyBytes_AsString - from pypy.module.cpyext.buffer import PyBuffer_FillInfo - c_buf = rffi.cast(rffi.VOIDP, PyBytes_AsString(space, w_str)) - return PyBuffer_FillInfo(space, view, w_str, c_buf, - space.len_w(w_str), 1, flags) - - at slot_function([PyObject, lltype.Ptr(Py_buffer), rffi.INT_real], rffi.INT_real, error=-1) -def bf_getbuffer(space, w_obj, view, flags): - from pypy.module.cpyext.buffer import fill_buffer - buf = space.buffer_w(w_obj, rffi.cast(lltype.Signed, flags)) - fill_buffer(space, view, buf, as_pyobj(space, w_obj)) - return 0 - -def setup_buffer_procs(space, w_type, pto): - bufspec = w_type.layout.typedef.buffer - if not bufspec: - return - c_buf = lltype.malloc(PyBufferProcs, flavor='raw', zero=True) - lltype.render_immortal(c_buf) - if space.is_w(w_type, space.w_bytes): - c_buf.c_bf_getbuffer = llslot(space, bytes_getbuffer) - else: - c_buf.c_bf_getbuffer = llslot(space, bf_getbuffer) - pto.c_tp_as_buffer = c_buf - @slot_function([PyObject], lltype.Void) def type_dealloc(space, obj): from pypy.module.cpyext.object import _dealloc @@ -611,8 +577,6 @@ pto.c_tp_itemsize = 1 elif space.is_w(w_type, space.w_tuple): pto.c_tp_itemsize = rffi.sizeof(PyObject) - # buffer protocol - setup_buffer_procs(space, w_type, pto) state = space.fromcache(State) pto.c_tp_free = state.C.PyObject_Free @@ -730,7 +694,6 @@ pto.c_tp_as_buffer = base.c_tp_as_buffer if base.c_tp_as_buffer: # inherit base.c_tp_as_buffer functions not inherited from w_type - # note: builtin types are handled in setup_buffer_procs pto_as = pto.c_tp_as_buffer base_as = base.c_tp_as_buffer if not pto_as.c_bf_getbuffer: diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py --- a/pypy/module/cpyext/userslot.py +++ b/pypy/module/cpyext/userslot.py @@ -106,7 +106,7 @@ return space.getitem(w_obj1, w_obj2) @slot_function([PyObject, PyObject], PyObject) -def slot_tp_getattr(space, w_obj1, w_obj2): +def slot_tp_getattr_hook(space, w_obj1, w_obj2): return space.getattr(w_obj1, w_obj2) @slot_function([PyObject, PyObject, PyObject], PyObject) diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -82,7 +82,7 @@ import imp for suffix, mode, type in imp.get_suffixes(): if type == imp.PY_SOURCE: - assert suffix == '.py' + assert suffix in ('.py', '.pyw') assert mode == 'r' elif type == imp.PY_COMPILED: assert suffix == '.pyc' diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1092,9 +1092,11 @@ if sys.platform == 'win32': os.chmod(my_path, 0o400) assert (os.stat(my_path).st_mode & 0o600) == 0o400 + os.chmod(self.path, 0o700) else: os.chmod(my_path, 0o200) assert (os.stat(my_path).st_mode & 0o777) == 0o200 + os.chmod(self.path, 0o700) if hasattr(os, 'fchmod'): def test_fchmod(self): @@ -1406,6 +1408,22 @@ if len(e.value.args) > 2: assert e.value.args[2] == "\\foo\\bar\\baz" + @py.test.mark.skipif("sys.platform != 'win32'") + def test_rename(self): + os = self.posix + fname = self.path2 + 'rename.txt' + with open(fname, "w") as f: + f.write("this is a rename test") + unicode_name = str(self.udir) + u'/test\u03be.txt' + os.rename(fname, unicode_name) + with open(unicode_name) as f: + assert f.read() == 'this is a rename test' + os.rename(unicode_name, fname) + with open(fname) as f: + assert f.read() == 'this is a rename test' + os.unlink(fname) + + def test_device_encoding(self): import sys encoding = self.posix.device_encoding(sys.stdout.fileno()) @@ -1509,6 +1527,8 @@ def test_environ(self): import sys, os environ = os.environ + if not environ: + skip('environ not filled in for untranslated tests') for k, v in environ.items(): assert type(k) is str assert type(v) is str diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py @@ -500,3 +500,21 @@ """) m = ffi.dlopen(lib_m) assert dir(m) == ['MYE1', 'MYE2', 'MYFOO', 'myconst', 'myfunc', 'myvar'] + + def test_dlclose(self): + if self.Backend is CTypesBackend: + py.test.skip("not with the ctypes backend") + ffi = FFI(backend=self.Backend()) + ffi.cdef("int foobar(void); int foobaz;") + lib = ffi.dlopen(lib_m) + ffi.dlclose(lib) + e = py.test.raises(ValueError, getattr, lib, 'foobar') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, getattr, lib, 'foobaz') + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + e = py.test.raises(ValueError, setattr, lib, 'foobaz', 42) + assert str(e.value).startswith("library '") + assert str(e.value).endswith("' has already been closed") + ffi.dlclose(lib) # does not raise diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py @@ -115,8 +115,12 @@ if sys.platform == 'win32': import os # did we already build it? - if os.path.exists(str(udir.join('testownlib.dll'))): - cls.module = str(udir.join('testownlib.dll')) + if cls.Backend is CTypesBackend: + dll_path = str(udir) + '\\testownlib1.dll' # only ascii for the ctypes backend + else: + dll_path = str(udir) + '\\' + (u+'testownlib\u03be.dll') # non-ascii char + if os.path.exists(dll_path): + cls.module = dll_path return # try (not too hard) to find the version used to compile this python # no mingw @@ -136,8 +140,9 @@ if os.path.isfile(vcvarsall): cmd = '"%s" %s' % (vcvarsall, arch) + ' & cl.exe testownlib.c ' \ ' /LD /Fetestownlib.dll' - subprocess.check_call(cmd, cwd = str(udir), shell=True) - cls.module = str(udir.join('testownlib.dll')) + subprocess.check_call(cmd, cwd = str(udir), shell=True) + os.rename(str(udir) + '\\testownlib.dll', dll_path) + cls.module = dll_path else: subprocess.check_call( 'cc testownlib.c -shared -fPIC -o testownlib.so', diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py @@ -1,9 +1,10 @@ # Generated by pypy/tool/import_cffi.py -import sys +import sys, os import py from cffi import FFI from cffi import recompiler, ffiplatform, VerificationMissing from pypy.module.test_lib_pypy.cffi_tests.udir import udir +from pypy.module.test_lib_pypy.cffi_tests.support import u def setup_module(mod): @@ -36,6 +37,13 @@ 'globalconst42', 'globalconsthello'] ) outputfilename = ffiplatform.compile(str(tmpdir), ext) + if sys.platform == "win32": + # test with a non-ascii char + outputfn1 = outputfilename + ofn, oext = os.path.splitext(outputfn1) + outputfilename = ofn + (u+'\u03be') + oext + #print(repr(outputfn1) + ' ==> ' + repr(outputfilename)) + os.rename(outputfn1, outputfilename) mod.extmod = outputfilename mod.tmpdir = tmpdir # @@ -56,6 +64,9 @@ typedef struct bar_s { int x; signed char a[]; } bar_t; enum foo_e { AA, BB, CC }; int strlen(const char *); + struct with_union { union { int a; char b; }; }; + union with_struct { struct { int a; char b; }; }; + struct NVGcolor { union { float rgba[4]; struct { float r,g,b,a; }; }; }; """) ffi.set_source('re_python_pysrc', None) ffi.emit_python_code(str(tmpdir.join('re_python_pysrc.py'))) @@ -105,12 +116,14 @@ from re_python_pysrc import ffi lib = ffi.dlopen(extmod) ffi.dlclose(lib) - e = py.test.raises(ffi.error, ffi.dlclose, lib) - assert str(e.value).startswith( - "library '%s' is already closed" % (extmod,)) + if type(extmod) is not str: # unicode, on python 2 + str_extmod = extmod.encode('utf-8') + else: + str_extmod = extmod e = py.test.raises(ffi.error, getattr, lib, 'add42') assert str(e.value) == ( - "library '%s' has been closed" % (extmod,)) + "library '%s' has been closed" % (str_extmod,)) + ffi.dlclose(lib) # does not raise def test_constant_via_lib(): from re_python_pysrc import ffi @@ -213,3 +226,23 @@ ffi.set_source('test_partial_enum', None) py.test.raises(VerificationMissing, ffi.emit_python_code, str(tmpdir.join('test_partial_enum.py'))) + +def test_anonymous_union_inside_struct(): + # based on issue #357 + from re_python_pysrc import ffi + INT = ffi.sizeof("int") + assert ffi.offsetof("struct with_union", "a") == 0 + assert ffi.offsetof("struct with_union", "b") == 0 + assert ffi.sizeof("struct with_union") == INT + # + assert ffi.offsetof("union with_struct", "a") == 0 + assert ffi.offsetof("union with_struct", "b") == INT + assert ffi.sizeof("union with_struct") >= INT + 1 + # + FLOAT = ffi.sizeof("float") + assert ffi.sizeof("struct NVGcolor") == FLOAT * 4 + assert ffi.offsetof("struct NVGcolor", "rgba") == 0 + assert ffi.offsetof("struct NVGcolor", "r") == 0 + assert ffi.offsetof("struct NVGcolor", "g") == FLOAT + assert ffi.offsetof("struct NVGcolor", "b") == FLOAT * 2 + assert ffi.offsetof("struct NVGcolor", "a") == FLOAT * 3 diff --git a/pypy/module/test_lib_pypy/test_dbm_extra.py b/pypy/module/test_lib_pypy/test_dbm_extra.py --- a/pypy/module/test_lib_pypy/test_dbm_extra.py +++ b/pypy/module/test_lib_pypy/test_dbm_extra.py @@ -1,4 +1,4 @@ -import py +import py, os from rpython.tool.udir import udir try: from lib_pypy import dbm @@ -73,3 +73,8 @@ assert 'key_with_empty_value' in d assert d['key_with_empty_value'] == '' d.close() + +def test_unicode_filename(): + path = str(udir) + os.sep + u'test_dbm_extra.test_unicode_filename' + d = dbm.open(path, 'c') + d.close() diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,6 +1,7 @@ +import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -40,9 +41,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 - at example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,13 +96,12 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: raise AssertionError("annotationoftype(%r)" % (t,)) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -354,7 +354,7 @@ else: # this is dead code, but in case we have a gc that does # not have a write barrier and does not zero memory, we would - # need to clal it + # need to call it if op.getopnum() == rop.SETFIELD_GC: self.consider_setfield_gc(op) elif op.getopnum() == rop.SETARRAYITEM_GC: diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -30,7 +30,7 @@ class CompileData(object): memo = None log_noopt = True - + def forget_optimization_info(self): for arg in self.trace.inputargs: arg.set_forwarded(None) @@ -67,19 +67,26 @@ """ This represents label() ops jump with no extra info associated with the label """ - def __init__(self, trace, call_pure_results=None, + def __init__(self, trace, resumestorage=None, call_pure_results=None, enable_opts=None): self.trace = trace + self.resumestorage = resumestorage self.call_pure_results = call_pure_results self.enable_opts = enable_opts def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll): from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer + from rpython.jit.metainterp.optimizeopt.bridgeopt import deserialize_optimizer_knowledge #assert not unroll opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations) - return opt.propagate_all_forward(self.trace.get_iter(), - self.call_pure_results) + traceiter = self.trace.get_iter() + if self.resumestorage: + frontend_inputargs = self.trace.inputargs + deserialize_optimizer_knowledge(opt, self.resumestorage, + frontend_inputargs, + traceiter.inputargs) + return opt.propagate_all_forward(traceiter, self.call_pure_results) class BridgeCompileData(CompileData): """ This represents ops() with a jump at the end that goes to some @@ -518,7 +525,7 @@ for item in lst: item.set_forwarded(None) # XXX we should really do it, but we need to remember the values - # somehoe for ContinueRunningNormally + # somehow for ContinueRunningNormally if reset_values: item.reset_value() @@ -671,38 +678,16 @@ raise jitexc.ExitFrameWithExceptionRef(cpu, value) -class TerminatingLoopToken(JitCellToken): # FIXME: kill? - terminating = True - - def __init__(self, nargs, finishdescr): - self.finishdescr = finishdescr - -def make_done_loop_tokens(): - done_with_this_frame_descr_void = DoneWithThisFrameDescrVoid() - done_with_this_frame_descr_int = DoneWithThisFrameDescrInt() - done_with_this_frame_descr_ref = DoneWithThisFrameDescrRef() - done_with_this_frame_descr_float = DoneWithThisFrameDescrFloat() - exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() - - # pseudo loop tokens to make the life of optimize.py easier - d = {'loop_tokens_done_with_this_frame_int': [ - TerminatingLoopToken(1, done_with_this_frame_descr_int) - ], - 'loop_tokens_done_with_this_frame_ref': [ - TerminatingLoopToken(1, done_with_this_frame_descr_ref) - ], - 'loop_tokens_done_with_this_frame_float': [ - TerminatingLoopToken(1, done_with_this_frame_descr_float) - ], - 'loop_tokens_done_with_this_frame_void': [ - TerminatingLoopToken(0, done_with_this_frame_descr_void) - ], - 'loop_tokens_exit_frame_with_exception_ref': [ - TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) - ], - } - d.update(locals()) - return d +def make_and_attach_done_descrs(targets): + for name, cls in [ + ("done_with_this_frame_descr_void", DoneWithThisFrameDescrVoid), + ("done_with_this_frame_descr_int", DoneWithThisFrameDescrInt), + ("done_with_this_frame_descr_ref", DoneWithThisFrameDescrRef), + ("done_with_this_frame_descr_float", DoneWithThisFrameDescrFloat), + ("exit_frame_with_exception_descr_ref", ExitFrameWithExceptionDescrRef)]: + descr = cls() + for target in targets: + setattr(target, name, descr) class ResumeDescr(AbstractFailDescr): _attrs_ = () @@ -726,6 +711,9 @@ TY_REF = 0x04 TY_FLOAT = 0x06 + def get_resumestorage(self): + raise NotImplementedError("abstract base class") + def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): if (self.must_compile(deadframe, metainterp_sd, jitdriver_sd) and not rstack.stack_almost_full()): @@ -854,15 +842,23 @@ class ResumeGuardCopiedDescr(AbstractResumeGuardDescr): _attrs_ = ('status', 'prev') + def __init__(self, prev): + AbstractResumeGuardDescr.__init__(self) + assert isinstance(prev, ResumeGuardDescr) + self.prev = prev + def copy_all_attributes_from(self, other): assert isinstance(other, ResumeGuardCopiedDescr) self.prev = other.prev def clone(self): - cloned = ResumeGuardCopiedDescr() - cloned.copy_all_attributes_from(self) + cloned = ResumeGuardCopiedDescr(self.prev) return cloned + def get_resumestorage(self): + prev = self.prev + assert isinstance(prev, ResumeGuardDescr) + return prev class ResumeGuardDescr(AbstractResumeGuardDescr): _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals', @@ -873,8 +869,7 @@ rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) def copy_all_attributes_from(self, other): - if isinstance(other, ResumeGuardCopiedDescr): - other = other.prev + other = other.get_resumestorage() assert isinstance(other, ResumeGuardDescr) self.rd_consts = other.rd_consts self.rd_pendingfields = other.rd_pendingfields @@ -895,6 +890,9 @@ cloned.copy_all_attributes_from(self) return cloned + def get_resumestorage(self): + return self + class ResumeGuardExcDescr(ResumeGuardDescr): pass @@ -936,22 +934,22 @@ ptr = cpu.ts.cast_to_baseclass(gcref) return cast_base_ptr_to_instance(AllVirtuals, ptr) -def invent_fail_descr_for_op(opnum, optimizer, copied_guard=False): +def invent_fail_descr_for_op(opnum, optimizer, copied_from_descr=None): if opnum == rop.GUARD_NOT_FORCED or opnum == rop.GUARD_NOT_FORCED_2: - assert not copied_guard + assert copied_from_descr is None resumedescr = ResumeGuardForcedDescr() resumedescr._init(optimizer.metainterp_sd, optimizer.jitdriver_sd) elif opnum in (rop.GUARD_IS_OBJECT, rop.GUARD_SUBCLASS, rop.GUARD_GC_TYPE): # note - this only happens in tests resumedescr = ResumeAtPositionDescr() elif opnum in (rop.GUARD_EXCEPTION, rop.GUARD_NO_EXCEPTION): - if copied_guard: - resumedescr = ResumeGuardCopiedExcDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedExcDescr(copied_from_descr) else: resumedescr = ResumeGuardExcDescr() else: - if copied_guard: - resumedescr = ResumeGuardCopiedDescr() + if copied_from_descr is not None: + resumedescr = ResumeGuardCopiedDescr(copied_from_descr) else: resumedescr = ResumeGuardDescr() return resumedescr @@ -1036,6 +1034,9 @@ self.original_greenkey, jitcell_token) metainterp_sd.stats.add_jitcell_token(jitcell_token) + def get_resumestorage(self): + return None + def compile_trace(metainterp, resumekey, runtime_boxes): """Try to compile a new bridge leading from the beginning of the history @@ -1067,22 +1068,15 @@ enable_opts = jitdriver_sd.warmstate.enable_opts call_pure_results = metainterp.call_pure_results + resumestorage = resumekey.get_resumestorage() if metainterp.history.ends_with_jump: - if isinstance(resumekey, ResumeGuardCopiedDescr): - key = resumekey.prev - assert isinstance(key, ResumeGuardDescr) - elif isinstance(resumekey, ResumeFromInterpDescr): - key = None - else: - key = resumekey - assert isinstance(key, ResumeGuardDescr) - data = BridgeCompileData(trace, runtime_boxes, key, + data = BridgeCompileData(trace, runtime_boxes, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts, inline_short_preamble=inline_short_preamble) else: - data = SimpleCompileData(trace, + data = SimpleCompileData(trace, resumestorage, call_pure_results=call_pure_results, enable_opts=enable_opts) try: diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -404,7 +404,6 @@ target_tokens = None failed_states = None retraced_count = 0 - terminating = False # see TerminatingLoopToken in compile.py invalidated = False outermost_jitdriver_sd = None # and more data specified by the backend when the loop is compiled @@ -935,7 +934,7 @@ return insns def check_simple_loop(self, expected=None, **check): - """ Usefull in the simplest case when we have only one trace ending with + """ Useful in the simplest case when we have only one trace ending with a jump back to itself and possibly a few bridges. Only the operations within the loop formed by that single jump will be counted. diff --git a/rpython/jit/metainterp/optimizeopt/bridgeopt.py b/rpython/jit/metainterp/optimizeopt/bridgeopt.py --- a/rpython/jit/metainterp/optimizeopt/bridgeopt.py +++ b/rpython/jit/metainterp/optimizeopt/bridgeopt.py @@ -17,11 +17,17 @@ # # ( ) length times, if getfield(box1, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # # # ( ) length times, if getarrayitem_gc(box1, index, descr) == box2 # both boxes should be in the liveboxes +# (or constants) # +# ---- call_loopinvariant knowledge +# +# ( ) length times, if call_loopinvariant(const) == box2 +# box2 should be in liveboxes # ---- @@ -55,11 +61,11 @@ return box def serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, liveboxes_from_env, memo): + from rpython.jit.metainterp.history import ConstInt available_boxes = {} for box in liveboxes: if box is not None and box in liveboxes_from_env: available_boxes[box] = None - metainterp_sd = optimizer.metainterp_sd # class knowledge is stored as bits, true meaning the class is known, false # means unknown. on deserializing we look at the bits, and read the runtime @@ -106,7 +112,19 @@ numb_state.append_int(0) numb_state.append_int(0) + if optimizer.optrewrite: + tuples_loopinvariant = optimizer.optrewrite.serialize_optrewrite( + available_boxes) + numb_state.append_int(len(tuples_loopinvariant)) + for constarg0, box in tuples_loopinvariant: + numb_state.append_short( + tag_box(ConstInt(constarg0), liveboxes_from_env, memo)) + numb_state.append_short(tag_box(box, liveboxes_from_env, memo)) + else: + numb_state.append_int(0) + def deserialize_optimizer_knowledge(optimizer, resumestorage, frontend_boxes, liveboxes): + from rpython.jit.metainterp.history import ConstInt reader = resumecode.Reader(resumestorage.rd_numb) assert len(frontend_boxes) == len(liveboxes) metainterp_sd = optimizer.metainterp_sd @@ -131,8 +149,6 @@ optimizer.make_constant_class(box, cls) # heap knowledge - if not optimizer.optheap: - return length = reader.next_item() result_struct = [] for i in range(length): @@ -154,4 +170,19 @@ tagged = reader.next_item() box2 = decode_box(resumestorage, tagged, liveboxes, metainterp_sd.cpu) result_array.append((box1, index, descr, box2)) - optimizer.optheap.deserialize_optheap(result_struct, result_array) + if optimizer.optheap: + optimizer.optheap.deserialize_optheap(result_struct, result_array) + + # call_loopinvariant knowledge + length = reader.next_item() + result_loopinvariant = [] + for i in range(length): + tagged1 = reader.next_item() + const = decode_box(resumestorage, tagged1, liveboxes, metainterp_sd.cpu) + assert isinstance(const, ConstInt) + i = const.getint() + tagged2 = reader.next_item() + box = decode_box(resumestorage, tagged2, liveboxes, metainterp_sd.cpu) + result_loopinvariant.append((i, box)) + if optimizer.optrewrite: + optimizer.optrewrite.deserialize_optrewrite(result_loopinvariant) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -688,12 +688,10 @@ def _copy_resume_data_from(self, guard_op, last_guard_op): - descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, True) last_descr = last_guard_op.getdescr() + descr = compile.invent_fail_descr_for_op(guard_op.getopnum(), self, last_descr) assert isinstance(last_descr, compile.ResumeGuardDescr) - if isinstance(descr, compile.ResumeGuardCopiedDescr): - descr.prev = last_descr - else: + if not isinstance(descr, compile.ResumeGuardCopiedDescr): descr.copy_all_attributes_from(last_descr) guard_op.setdescr(descr) guard_op.setfailargs(last_guard_op.getfailargs()) diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -877,6 +877,18 @@ optimize_SAME_AS_R = optimize_SAME_AS_I optimize_SAME_AS_F = optimize_SAME_AS_I + def serialize_optrewrite(self, available_boxes): + res = [] + for i, box in self.loop_invariant_results.iteritems(): + box = self.get_box_replacement(box) + if box in available_boxes: + res.append((i, box)) + return res + + def deserialize_optrewrite(self, tups): + for i, box in tups: + self.loop_invariant_results[i] = box + dispatch_opt = make_dispatcher_method(OptRewrite, 'optimize_', default=OptRewrite.emit) optimize_guards = _findall(OptRewrite, 'optimize_', 'GUARD') diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -31,8 +31,8 @@ expected = convert_old_style_to_targets(exp, jump=True) call_pure_results = self._convert_call_pure_results(call_pure_results) trace = convert_loop_to_trace(loop, FakeMetaInterpStaticData(self.cpu)) - compile_data = compile.SimpleCompileData(trace, - call_pure_results) + compile_data = compile.SimpleCompileData( + trace, call_pure_results=call_pure_results) info, ops = self._do_optimize_loop(compile_data) label_op = ResOperation(rop.LABEL, info.inputargs) loop.inputargs = info.inputargs diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1854,12 +1854,7 @@ self._addr2name_keys = [] self._addr2name_values = [] - self.__dict__.update(compile.make_done_loop_tokens()) - for val in ['int', 'float', 'ref', 'void']: - fullname = 'done_with_this_frame_descr_' + val - setattr(self.cpu, fullname, getattr(self, fullname)) - d = self.exit_frame_with_exception_descr_ref - self.cpu.exit_frame_with_exception_descr_ref = d + compile.make_and_attach_done_descrs([self, cpu]) def _freeze_(self): return True @@ -1909,8 +1904,8 @@ history.REF: 'ref', history.FLOAT: 'float', history.VOID: 'void'}[jd.result_type] - tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) - jd.portal_finishtoken = tokens[0].finishdescr + token = getattr(self, 'done_with_this_frame_descr_%s' % name) + jd.portal_finishtoken = token jd.propagate_exc_descr = exc_descr # self.cpu.propagate_exception_descr = exc_descr @@ -2463,10 +2458,7 @@ def handle_guard_failure(self, resumedescr, deadframe): debug_start('jit-tracing') self.staticdata.profiler.start_tracing() - if isinstance(resumedescr, compile.ResumeGuardCopiedDescr): - key = resumedescr.prev - else: - key = resumedescr + key = resumedescr.get_resumestorage() assert isinstance(key, compile.ResumeGuardDescr) # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp @@ -2770,21 +2762,19 @@ if result_type == history.VOID: assert exitbox is None exits = [] - loop_tokens = sd.loop_tokens_done_with_this_frame_void + token = sd.done_with_this_frame_descr_void elif result_type == history.INT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_int + token = sd.done_with_this_frame_descr_int elif result_type == history.REF: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_ref + token = sd.done_with_this_frame_descr_ref elif result_type == history.FLOAT: exits = [exitbox] - loop_tokens = sd.loop_tokens_done_with_this_frame_float + token = sd.done_with_this_frame_descr_float else: assert False - # FIXME: kill TerminatingLoopToken? # FIXME: can we call compile_trace? - token = loop_tokens[0].finishdescr self.history.record(rop.FINISH, exits, None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, exits) @@ -2810,7 +2800,7 @@ def compile_exit_frame_with_exception(self, valuebox): self.store_token_in_vable() sd = self.staticdata - token = sd.loop_tokens_exit_frame_with_exception_ref[0].finishdescr + token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) self.history.trace.done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py --- a/rpython/jit/metainterp/test/test_bridgeopt.py +++ b/rpython/jit/metainterp/test/test_bridgeopt.py @@ -1,6 +1,9 @@ # tests that check that information is fed from the optimizer into the bridges +import pytest + import math + from rpython.rlib import jit from rpython.jit.metainterp.test.support import LLJitMixin from rpython.jit.metainterp.optimizeopt.bridgeopt import serialize_optimizer_knowledge @@ -27,6 +30,7 @@ class FakeOptimizer(object): metainterp_sd = None optheap = None + optrewrite = None def __init__(self, dct={}, cpu=None): self.dct = dct @@ -61,7 +65,8 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert unpack_numbering(numb_state.create_numbering()) == [1, 0b010000, 0, 0] + assert unpack_numbering(numb_state.create_numbering()) == [ + 1, 0b010000, 0, 0, 0] rbox1 = InputArgRef() rbox2 = InputArgRef() @@ -100,7 +105,7 @@ serialize_optimizer_knowledge(optimizer, numb_state, liveboxes, {}, None) - assert len(numb_state.create_numbering().code) == 3 + math.ceil(len(refboxes) / 6.0) + assert len(numb_state.create_numbering().code) == 4 + math.ceil(len(refboxes) / 6.0) From pypy.commits at gmail.com Mon Mar 19 12:23:19 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 09:23:19 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: Delete recently-introduced EOL whitespace. Message-ID: <5aafe3f7.873f1c0a.cbe5f.4f26@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r94009:2df857639086 Date: 2018-03-19 17:22 +0100 http://bitbucket.org/pypy/pypy/changeset/2df857639086/ Log: Delete recently-introduced EOL whitespace. diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -243,7 +243,7 @@ productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") productdir = os.path.abspath(productdir) if not os.path.isdir(productdir): - + log.debug("%s is not a valid directory" % productdir) return None else: From pypy.commits at gmail.com Mon Mar 19 13:24:28 2018 From: pypy.commits at gmail.com (Floris Bruynooghe) Date: Mon, 19 Mar 2018 10:24:28 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: First step, collecting and running apptest_*.py works Message-ID: <5aaff24c.88d31c0a.6dc48.665e@mx.google.com> Author: Floris Bruynooghe Branch: py3tests Changeset: r94011:eea5b6bc977d Date: 2018-03-19 11:28 +0100 http://bitbucket.org/pypy/pypy/changeset/eea5b6bc977d/ Log: First step, collecting and running apptest_*.py works diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -43,11 +43,10 @@ def py3k_skip(message): py.test.skip('[py3k] %s' % message) py.test.py3k_skip = py3k_skip + if config.getoption('runappdirect'): + config.addinivalue_line('python_files', 'apptest_*.py') def pytest_addoption(parser): - from rpython.conftest import pytest_addoption - pytest_addoption(parser) - group = parser.getgroup("pypy options") group.addoption('-A', '--runappdirect', action="store_true", default=False, dest="runappdirect", @@ -94,7 +93,8 @@ ensure_pytest_builtin_helpers() def pytest_pycollect_makemodule(path, parent): - return PyPyModule(path, parent) + if not parent.config.getoption('runappdirect'): + return PyPyModule(path, parent) def is_applevel(item): from pypy.tool.pytest.apptest import AppTestFunction @@ -193,5 +193,7 @@ appclass.obj.runappdirect = option.runappdirect -def pytest_ignore_collect(path): +def pytest_ignore_collect(path, config): + if config.getoption('runappdirect') and not path.fnmatch('apptest_*.py'): + return True return path.check(link=1) diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/apptest_coroutine.py @@ -0,0 +1,204 @@ +import pytest + + +def test_cannot_iterate(): + async def f(x): + pass + pytest.raises(TypeError, "for i in f(5): pass") + pytest.raises(TypeError, iter, f(5)) + pytest.raises(TypeError, next, f(5)) + + +def test_async_for(): + class X: + def __aiter__(self): + return MyAIter() + class MyAIter: + async def __anext__(self): + return 42 + async def f(x): + sum = 0 + async for a in x: + sum += a + if sum > 100: + break + return sum + cr = f(X()) + try: + cr.send(None) + except StopIteration as e: + assert e.value == 42 * 3 + else: + assert False, "should have raised" + + +def test_StopAsyncIteration(): + class X: + def __aiter__(self): + return MyAIter() + class MyAIter: + count = 0 + async def __anext__(self): + if self.count == 3: + raise StopAsyncIteration + self.count += 1 + return 42 + async def f(x): + sum = 0 + async for a in x: + sum += a + return sum + cr = f(X()) + try: + cr.send(None) + except StopIteration as e: + assert e.value == 42 * 3 + else: + assert False, "should have raised" + + +def test_async_for_old_style(): + class X: + def __aiter__(self): + return MyAIter() + class MyAIter: + def __await__(self): + return iter([20, 30]) + async def f(x): + sum = 0 + async for a in x: + sum += a + if sum > 100: + break + return sum + cr = f(X()) + assert next(cr.__await__()) == 20 + + +def test_set_coroutine_wrapper(): + import sys + async def f(): + pass + seen = [] + def my_wrapper(cr): + seen.append(cr) + return 42 + assert sys.get_coroutine_wrapper() is None + sys.set_coroutine_wrapper(my_wrapper) + assert sys.get_coroutine_wrapper() is my_wrapper + cr = f() + assert cr == 42 + sys.set_coroutine_wrapper(None) + assert sys.get_coroutine_wrapper() is None + + +def test_async_with(): + seen = [] + class X: + async def __aenter__(self): + seen.append('aenter') + async def __aexit__(self, *args): + seen.append('aexit') + async def f(x): + async with x: + return 42 + c = f(X()) + try: + c.send(None) + except StopIteration as e: + assert e.value == 42 + else: + assert False, "should have raised" + assert seen == ['aenter', 'aexit'] + + +def test_await(): + class X: + def __await__(self): + i1 = yield 40 + assert i1 == 82 + i2 = yield 41 + assert i2 == 93 + async def f(): + await X() + await X() + c = f() + assert c.send(None) == 40 + assert c.send(82) == 41 + assert c.send(93) == 40 + assert c.send(82) == 41 + pytest.raises(StopIteration, c.send, 93) + + +def test_await_error(): + async def f(): + await [42] + c = f() + try: + c.send(None) + except TypeError as e: + assert str(e) == "object list can't be used in 'await' expression" + else: + assert False, "should have raised" + + +def test_async_with_exception_context(): + class CM: + async def __aenter__(self): + pass + async def __aexit__(self, *e): + 1/0 + async def f(): + async with CM(): + raise ValueError + c = f() + try: + c.send(None) + except ZeroDivisionError as e: + assert e.__context__ is not None + assert isinstance(e.__context__, ValueError) + else: + assert False, "should have raised" + + +def test_runtime_warning(): + import gc, warnings + async def foobaz(): + pass + with warnings.catch_warnings(record=True) as l: + foobaz() + gc.collect() + gc.collect() + gc.collect() + + assert len(l) == 1, repr(l) + w = l[0].message + assert isinstance(w, RuntimeWarning) + assert str(w).startswith("coroutine ") + assert str(w).endswith("foobaz' was never awaited") + + +def test_async_for_with_tuple_subclass(): + class Done(Exception): pass + + class AIter(tuple): + i = 0 + def __aiter__(self): + return self + async def __anext__(self): + if self.i >= len(self): + raise StopAsyncIteration + self.i += 1 + return self[self.i - 1] + + result = [] + async def foo(): + async for i in AIter([42]): + result.append(i) + raise Done + + try: + foo().send(None) + except Done: + pass + assert result == [42] From pypy.commits at gmail.com Mon Mar 19 13:24:31 2018 From: pypy.commits at gmail.com (Floris Bruynooghe) Date: Mon, 19 Mar 2018 10:24:31 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Make untranslated app tests work on a very basic level Message-ID: <5aaff24f.83c21c0a.55edf.64f3@mx.google.com> Author: Floris Bruynooghe Branch: py3tests Changeset: r94012:6fdcb46e0b2f Date: 2018-03-19 18:21 +0100 http://bitbucket.org/pypy/pypy/changeset/6fdcb46e0b2f/ Log: Make untranslated app tests work on a very basic level This can collect a module and run test functions inside it, without any fixtures or setup. diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py --- a/_pytest/assertion/rewrite.py +++ b/_pytest/assertion/rewrite.py @@ -265,10 +265,10 @@ if (not source.startswith(BOM_UTF8) and cookie_re.match(source[0:end1]) is None and cookie_re.match(source[end1 + 1:end2]) is None): - if hasattr(state, "_indecode"): - # encodings imported us again, so don't rewrite. - return None, None - state._indecode = True + # if hasattr(state, "_indecode"): + # # encodings imported us again, so don't rewrite. + # return None, None + # state._indecode = True try: try: source.decode("ascii") @@ -293,10 +293,21 @@ except SyntaxError: # It's possible that this error is from some bug in the # assertion rewriting, but I don't know of a fast way to tell. - state.trace("failed to compile: %r" % (fn,)) + # state.trace("failed to compile: %r" % (fn,)) return None, None return stat, co + +def create_module(co): + """Hack to create a module from a code object created by _rewrite_test()""" + mod = imp.new_module(co.co_filename.split('/')[-1].split('.')[0]) + mod.__file__ = co.co_filename + # mod.__cached__ = pyc + mod.__loader__ = None + exec(co, mod.__dict__) + return mod + + def _make_rewritten_pyc(state, source_stat, pyc, co): """Try to dump rewritten code to *pyc*.""" if sys.platform.startswith("win"): diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -9,6 +9,7 @@ PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind(LOOK_FOR_PYTHON3) if PYTHON3 is not None: PYTHON3 = str(PYTHON3) +APPLEVEL_FN = 'apptest_*.py' # pytest settings rsyncdirs = ['.', '../lib-python', '../lib_pypy', '../demo'] @@ -44,7 +45,7 @@ py.test.skip('[py3k] %s' % message) py.test.py3k_skip = py3k_skip if config.getoption('runappdirect'): - config.addinivalue_line('python_files', 'apptest_*.py') + config.addinivalue_line('python_files', APPLEVEL_FN) def pytest_addoption(parser): group = parser.getgroup("pypy options") @@ -94,7 +95,11 @@ def pytest_pycollect_makemodule(path, parent): if not parent.config.getoption('runappdirect'): - return PyPyModule(path, parent) + if path.fnmatch(APPLEVEL_FN): + from pypy.tool.pytest.apptest2 import AppTestModule + return AppTestModule(path, parent) + else: + return PyPyModule(path, parent) def is_applevel(item): from pypy.tool.pytest.apptest import AppTestFunction @@ -110,7 +115,8 @@ else: item.add_marker('interplevel') -class PyPyModule(py.test.collect.Module): + +class PyPyModule(pytest.Module): """ we take care of collecting classes both at app level and at interp-level (because we need to stick a space at the class) ourselves. @@ -194,6 +200,6 @@ def pytest_ignore_collect(path, config): - if config.getoption('runappdirect') and not path.fnmatch('apptest_*.py'): + if config.getoption('runappdirect') and not path.fnmatch(APPLEVEL_FN): return True return path.check(link=1) diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/apptest2.py @@ -0,0 +1,72 @@ +import sys + +import pytest +import pypy.interpreter.function +from pypy.interpreter.error import OperationError +from pypy.tool.pytest import objspace +from pypy.tool.pytest import appsupport + + +class AppTestModule(pytest.Module): + + def collect(self): + space = objspace.gettestobjspace() + w_mod = space.appexec([] ,"""(): + import sys + sys.path.insert(0, '%s') + import _pytest.assertion.rewrite, py.path + + stat, co = _pytest.assertion.rewrite._rewrite_test(None, py.path.local('%s')) + mod = _pytest.assertion.rewrite.create_module(co) + return mod + """ % (self.config.rootdir, str(self.fspath))) + mod_dict = w_mod.getdict(space).unwrap(space) + items = [] + for name, w_obj in mod_dict.items(): + if not name.startswith('test_'): + continue + if not isinstance(w_obj, pypy.interpreter.function.Function): + continue + items.append(AppTestFunction(name, self, w_obj)) + return items + + def setup(self): + pass + + +class AppError(Exception): + + def __init__(self, excinfo): + self.excinfo = excinfo + + +class AppTestFunction(pytest.Item): + + def __init__(self, name, parent, w_obj): + super(AppTestFunction, self).__init__(name, parent) + self.w_obj = w_obj + + def runtest(self): + target = self.w_obj + space = target.space + self.execute_appex(space, target) + + def repr_failure(self, excinfo): + if excinfo.errisinstance(AppError): + excinfo = excinfo.value.excinfo + return super(AppTestFunction, self).repr_failure(excinfo) + + def execute_appex(self, space, w_func): + space.getexecutioncontext().set_sys_exc_info(None) + try: + space.call_function(w_func) + except OperationError as e: + if self.config.option.raise_operr: + raise + tb = sys.exc_info()[2] + if e.match(space, space.w_KeyboardInterrupt): + raise KeyboardInterrupt, KeyboardInterrupt(), tb + appexcinfo = appsupport.AppExceptionInfo(space, e) + if appexcinfo.traceback: + raise AppError, AppError(appexcinfo), tb + raise From pypy.commits at gmail.com Mon Mar 19 13:24:40 2018 From: pypy.commits at gmail.com (Raemi) Date: Mon, 19 Mar 2018 10:24:40 -0700 (PDT) Subject: [pypy-commit] stmgc c8-reshare-pages: fix warnings that let tests fail Message-ID: <5aaff258.98bf1c0a.64312.6664@mx.google.com> Author: Remi Meier Branch: c8-reshare-pages Changeset: r2156:322ad09f62f0 Date: 2018-03-19 18:24 +0100 http://bitbucket.org/pypy/stmgc/changeset/322ad09f62f0/ Log: fix warnings that let tests fail diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -14,7 +14,7 @@ { assert(undo->type != TYPE_POSITION_MARKER); free(undo->backup); - assert(undo->backup = (char*)0xbb); + assert((undo->backup = (char*)0xbb)); increment_total_allocated(-SLICE_SIZE(undo->slice)); } diff --git a/c8/stm/largemalloc.c b/c8/stm/largemalloc.c --- a/c8/stm/largemalloc.c +++ b/c8/stm/largemalloc.c @@ -427,8 +427,8 @@ mscan->size = msize + chunk->size; next_chunk(mscan)->prev_size = mscan->size; - assert(chunk->prev_size = (size_t)-1); - assert(chunk->size = (size_t)-1); + assert((chunk->prev_size = (size_t)-1)); + assert((chunk->size = (size_t)-1)); chunk = mscan; } From pypy.commits at gmail.com Mon Mar 19 13:40:00 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 19 Mar 2018 10:40:00 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: fix from merge Message-ID: <5aaff5f0.178bdf0a.58a91.458f@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r94013:6a7b9467958c Date: 2018-03-19 18:36 +0100 http://bitbucket.org/pypy/pypy/changeset/6a7b9467958c/ Log: fix from merge diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -111,6 +111,9 @@ def newbytes(self, obj): return bytes(obj) + def newutf8(self, obj, lgth): + return obj + def call_function(self, func, *args, **kwds): return func(*args, **kwds) From pypy.commits at gmail.com Mon Mar 19 18:20:31 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 15:20:31 -0700 (PDT) Subject: [pypy-commit] pypy default: Test and fix interpreter crash after int overflow involving __length_hint__. Message-ID: <5ab037af.4b981c0a.f743d.12cb@mx.google.com> Author: Manuel Jacob Branch: Changeset: r94016:c5cc7018cdd5 Date: 2018-03-19 23:19 +0100 http://bitbucket.org/pypy/pypy/changeset/c5cc7018cdd5/ Log: Test and fix interpreter crash after int overflow involving __length_hint__. diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -15,6 +15,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.objectmodel import ( import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.rlib.rarithmetic import ovfcheck from rpython.rlib import longlong2float from rpython.tool.sourcetools import func_with_new_name @@ -871,7 +872,12 @@ """Extend w_list from a generic iterable""" length_hint = self.space.length_hint(w_iterable, 0) if length_hint: - w_list._resize_hint(w_list.length() + length_hint) + try: + newsize_hint = ovfcheck(w_list.length() + length_hint) + except OverflowError: + pass + else: + w_list._resize_hint(newsize_hint) extended = _do_extend_from_iterable(self.space, w_list, w_iterable) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -619,6 +619,18 @@ assert l == [1.2, 2.3, 3.4, 4.5] assert l is l0 + def test_extend_iterable_length_hint_overflow(self): + import sys + class CustomIterable(object): + def __iter__(self): + if False: + yield + def __length_hint__(self): + return sys.maxsize + a = [1, 2, 3, 4] + a.extend(CustomIterable()) + assert a == [1, 2, 3, 4] + def test_sort(self): l = l0 = [1, 5, 3, 0] l.sort() From pypy.commits at gmail.com Mon Mar 19 18:23:16 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 15:23:16 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: hg merge default Message-ID: <5ab03854.8bc7df0a.9f089.17ee@mx.google.com> Author: Manuel Jacob Branch: py3.5 Changeset: r94017:d12c359516b2 Date: 2018-03-19 23:21 +0100 http://bitbucket.org/pypy/pypy/changeset/d12c359516b2/ Log: hg merge default diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -15,6 +15,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.objectmodel import ( import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.rlib.rarithmetic import ovfcheck from rpython.rlib import longlong2float from rpython.tool.sourcetools import func_with_new_name @@ -848,7 +849,12 @@ """Extend w_list from a generic iterable""" length_hint = self.space.length_hint(w_iterable, 0) if length_hint: - w_list._resize_hint(w_list.length() + length_hint) + try: + newsize_hint = ovfcheck(w_list.length() + length_hint) + except OverflowError: + pass + else: + w_list._resize_hint(newsize_hint) extended = _do_extend_from_iterable(self.space, w_list, w_iterable) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -618,6 +618,18 @@ assert l == [1.2, 2.3, 3.4, 4.5] assert l is l0 + def test_extend_iterable_length_hint_overflow(self): + import sys + class CustomIterable(object): + def __iter__(self): + if False: + yield + def __length_hint__(self): + return sys.maxsize + a = [1, 2, 3, 4] + a.extend(CustomIterable()) + assert a == [1, 2, 3, 4] + def test_sort(self): l = l0 = [1, 5, 3, 0] l.sort() From pypy.commits at gmail.com Mon Mar 19 18:23:19 2018 From: pypy.commits at gmail.com (mjacob) Date: Mon, 19 Mar 2018 15:23:19 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: hg merge py3.5 Message-ID: <5ab03857.06321c0a.dc3d5.1525@mx.google.com> Author: Manuel Jacob Branch: py3.6 Changeset: r94018:845ecbcca6b6 Date: 2018-03-19 23:22 +0100 http://bitbucket.org/pypy/pypy/changeset/845ecbcca6b6/ Log: hg merge py3.5 diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -243,7 +243,7 @@ productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") productdir = os.path.abspath(productdir) if not os.path.isdir(productdir): - + log.debug("%s is not a valid directory" % productdir) return None else: diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -15,6 +15,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.objectmodel import ( import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.rlib.rarithmetic import ovfcheck from rpython.rlib import longlong2float from rpython.tool.sourcetools import func_with_new_name @@ -848,7 +849,12 @@ """Extend w_list from a generic iterable""" length_hint = self.space.length_hint(w_iterable, 0) if length_hint: - w_list._resize_hint(w_list.length() + length_hint) + try: + newsize_hint = ovfcheck(w_list.length() + length_hint) + except OverflowError: + pass + else: + w_list._resize_hint(newsize_hint) extended = _do_extend_from_iterable(self.space, w_list, w_iterable) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -618,6 +618,18 @@ assert l == [1.2, 2.3, 3.4, 4.5] assert l is l0 + def test_extend_iterable_length_hint_overflow(self): + import sys + class CustomIterable(object): + def __iter__(self): + if False: + yield + def __length_hint__(self): + return sys.maxsize + a = [1, 2, 3, 4] + a.extend(CustomIterable()) + assert a == [1, 2, 3, 4] + def test_sort(self): l = l0 = [1, 5, 3, 0] l.sort() From pypy.commits at gmail.com Tue Mar 20 02:38:38 2018 From: pypy.commits at gmail.com (arigo) Date: Mon, 19 Mar 2018 23:38:38 -0700 (PDT) Subject: [pypy-commit] pypy default: Extend FinalizerQueue to be able to handle low-level GCREF pointers Message-ID: <5ab0ac6e.85921c0a.a80ba.631a@mx.google.com> Author: Armin Rigo Branch: Changeset: r94019:c4654fc5b220 Date: 2018-03-20 07:37 +0100 http://bitbucket.org/pypy/pypy/changeset/c4654fc5b220/ Log: Extend FinalizerQueue to be able to handle low-level GCREF pointers directly, instead of high-level instances of some class diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -385,6 +385,7 @@ # # Class: # the class (or base class) of finalized objects + # --or-- None to handle low-level GCREFs directly # # def finalizer_trigger(self): # called to notify that new items have been put in the queue @@ -397,11 +398,13 @@ def next_dead(self): if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance + from rpython.rtyper.lltypesystem.llmemory import GCREF + from rpython.rtyper.annlowlevel import cast_gcref_to_instance tag = FinalizerQueue._get_tag(self) - ptr = llop.gc_fq_next_dead(OBJECTPTR, tag) - return cast_base_ptr_to_instance(self.Class, ptr) + ptr = llop.gc_fq_next_dead(GCREF, tag) + if self.Class is not None: + ptr = cast_gcref_to_instance(self.Class, ptr) + return ptr try: return self._queue.popleft() except (AttributeError, IndexError): @@ -410,14 +413,18 @@ @specialize.arg(0) @jit.dont_look_inside def register_finalizer(self, obj): - assert isinstance(obj, self.Class) + from rpython.rtyper.lltypesystem.llmemory import GCREF + if self.Class is None: + assert lltype.typeOf(obj) == GCREF + else: + assert isinstance(obj, self.Class) if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr + from rpython.rtyper.annlowlevel import cast_instance_to_gcref tag = FinalizerQueue._get_tag(self) - ptr = cast_instance_to_base_ptr(obj) - llop.gc_fq_register(lltype.Void, tag, ptr) + if self.Class is not None: + obj = cast_instance_to_gcref(obj) + llop.gc_fq_register(lltype.Void, tag, obj) return else: self._untranslated_register_finalizer(obj) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -599,3 +599,94 @@ e = py.test.raises(TyperError, gengraph, f, []) assert str(e.value).startswith('the RPython-level __del__() method in') + + def test_translated_boehm(self): + self._test_translated(use_gc="boehm", llcase=False) + + def test_translated_boehm_ll(self): + self._test_translated(use_gc="boehm", llcase=True) + + def test_translated_incminimark(self): + self._test_translated(use_gc="incminimark", llcase=False) + + def test_translated_incminimark_ll(self): + self._test_translated(use_gc="incminimark", llcase=True) + + def _test_translated(self, use_gc, llcase): + import subprocess + from rpython.rlib import objectmodel + from rpython.translator.interactive import Translation + # + class Seen: + count = 0 + class MySimpleFQ(rgc.FinalizerQueue): + if not llcase: + Class = T_Root + else: + Class = None + def finalizer_trigger(self): + seen.count += 1 + seen = Seen() + fq = MySimpleFQ() + if not llcase: + EMPTY = None + llbuilder = T_Int + else: + from rpython.rtyper.annlowlevel import llstr + EMPTY = lltype.nullptr(llmemory.GCREF.TO) + def llbuilder(n): + return lltype.cast_opaque_ptr(llmemory.GCREF, llstr(str(n))) + + def subfunc(): + w0 = llbuilder(40); fq.register_finalizer(w0) + w1 = llbuilder(41); fq.register_finalizer(w1) + w2 = llbuilder(42); fq.register_finalizer(w2) + w3 = llbuilder(43); fq.register_finalizer(w3) + w4 = llbuilder(44); fq.register_finalizer(w4) + w5 = llbuilder(45); fq.register_finalizer(w5) + w6 = llbuilder(46); fq.register_finalizer(w6) + w7 = llbuilder(47); fq.register_finalizer(w7) + w8 = llbuilder(48); fq.register_finalizer(w8) + w9 = llbuilder(49); fq.register_finalizer(w9) + gc.collect() + assert seen.count == 0 + assert fq.next_dead() is EMPTY + objectmodel.keepalive_until_here(w0) + objectmodel.keepalive_until_here(w1) + objectmodel.keepalive_until_here(w2) + objectmodel.keepalive_until_here(w3) + objectmodel.keepalive_until_here(w4) + objectmodel.keepalive_until_here(w5) + objectmodel.keepalive_until_here(w6) + objectmodel.keepalive_until_here(w7) + objectmodel.keepalive_until_here(w8) + objectmodel.keepalive_until_here(w9) + + def main(argv): + assert fq.next_dead() is EMPTY + subfunc() + gc.collect(); gc.collect(); gc.collect() + assert seen.count > 0 + n = fq.next_dead() + while True: + if not llcase: + assert type(n) is T_Int and 40 <= n.x <= 49 + else: + from rpython.rtyper.lltypesystem.rstr import STR + assert lltype.typeOf(n) is llmemory.GCREF + p = lltype.cast_opaque_ptr(lltype.Ptr(STR), n) + assert len(p.chars) == 2 + assert p.chars[0] == "4" + assert "0" <= p.chars[1] <= "9" + n = fq.next_dead() + if n is EMPTY: + break + print "OK!" + return 0 + # + t = Translation(main, gc=use_gc) + t.disable(['backendopt']) + t.set_backend_extra_options(c_debug_defines=True) + exename = t.compile() + data = subprocess.check_output([str(exename), '.', '.', '.']) + assert data.strip().endswith('OK!') From pypy.commits at gmail.com Tue Mar 20 04:09:35 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 20 Mar 2018 01:09:35 -0700 (PDT) Subject: [pypy-commit] pypy default: FinalizerQueue works with Boehm too Message-ID: <5ab0c1bf.52bf1c0a.e907c.46b5@mx.google.com> Author: Armin Rigo Branch: Changeset: r94020:2804dc8a8645 Date: 2018-03-20 09:08 +0100 http://bitbucket.org/pypy/pypy/changeset/2804dc8a8645/ Log: FinalizerQueue works with Boehm too diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -378,8 +378,6 @@ class FinalizerQueue(object): """A finalizer queue. See pypy/doc/discussion/finalizer-order.rst. - Note: only works with the framework GCs (like minimark). It is - ignored with Boehm or with refcounting (used by tests). """ # Must be subclassed, and the subclass needs these attributes: # From pypy.commits at gmail.com Tue Mar 20 04:56:52 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 20 Mar 2018 01:56:52 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger-updated: Fix type Message-ID: <5ab0ccd4.f8aedf0a.d1592.6028@mx.google.com> Author: Armin Rigo Branch: reverse-debugger-updated Changeset: r94021:78d7729c2379 Date: 2018-03-20 09:56 +0100 http://bitbucket.org/pypy/pypy/changeset/78d7729c2379/ Log: Fix type diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -1159,7 +1159,7 @@ compilation_info=eci, _nowrapper=True) _, state.C.set_marker = rffi.CExternVariable( rffi.VOIDP, '_pypy_rawrefcount_w_marker_deallocating', - eci, _nowrapper=True, c_type='Py_ssize_t') + eci, _nowrapper=True, c_type='void *') state.C._PyPy_subtype_dealloc = rffi.llexternal( '_PyPy_subtype_dealloc', [PyObject], lltype.Void, compilation_info=eci, _nowrapper=True) From pypy.commits at gmail.com Tue Mar 20 06:10:34 2018 From: pypy.commits at gmail.com (jdb) Date: Tue, 20 Mar 2018 03:10:34 -0700 (PDT) Subject: [pypy-commit] cffi default: Show the recommended use case: API/out-of-line first, in its own section. Message-ID: <5ab0de1a.10afdf0a.4acab.987c@mx.google.com> Author: Jean-Daniel Browne Branch: Changeset: r3116:9318184d748c Date: 2018-03-20 09:09 +0100 http://bitbucket.org/cffi/cffi/changeset/9318184d748c/ Log: Show the recommended use case: API/out-of-line first, in its own section. diff --git a/doc/source/overview.rst b/doc/source/overview.rst --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -3,6 +3,131 @@ ======================================================= .. contents:: + + +This document starts, in the first section, with a simple working +example of using CFFI to call a C function from Python. CFFI is +flexible and covers several use cases presented in the second +section. Then, the next section shows how to export Python functions +to a Python interpreter embedded in a C or C++ application. The last +two sections delve deeper in the CFFI library. + +Make sure you have `cffi installed`__. + +.. __: installation.html + +.. _out-of-line-api-level: +.. _real-example: + + +Example: calling a C function from Python +----------------------------------------- + +This example is about the use case when the library sources are +available, the next section shows how use a compiled, installed +library. + +1. Make sure the sources of the library defining the useful C function + is available. For this example, create the file ``pi.c`` and ``pi.h``: + + .. code-block:: C + + /* filename: pi.c*/ + # include + # include + + /* Returns a very crude approximation of Pi + given a int: a number of iteration */ + float pi_approx(int n){ + + double i,x,y,sum=0; + + for(i=0;i=1.0.0"], + cffi_modules=["pi_extension_build:ffibuilder"], + install_requires=["cffi>=1.0.0"], + ) + +``cffi_modules`` is a list of ``:`` describing the modules to build. + + +Other CFFI modes +---------------- CFFI can be used in one of four modes: "ABI" versus "API" level, each with "in-line" or "out-of-line" preparation (or compilation). @@ -18,13 +143,9 @@ step of preparation (and possibly C compilation) that produces a module which your main program can then import. -(The examples below assume that you have `installed CFFI`__.) - -.. __: installation.html - Simple example (ABI level, in-line) ------------------------------------ ++++++++++++++++++++++++++++++++++++ .. code-block:: python @@ -59,96 +180,9 @@ also faster.) -.. _out-of-line-api-level: -.. _real-example: - -Real example (API level, out-of-line) -------------------------------------- - -.. code-block:: python - - # file "example_build.py" - - # Note: we instantiate the same 'cffi.FFI' class as in the previous - # example, but call the result 'ffibuilder' now instead of 'ffi'; - # this is to avoid confusion with the other 'ffi' object you get below - - from cffi import FFI - ffibuilder = FFI() - - ffibuilder.set_source("_example", - r""" // passed to the real C compiler, - // contains implementation of things declared in cdef() - #include - #include - - struct passwd *get_pw_for_root(void) { - return getpwuid(0); - } - """, - libraries=[]) # or a list of libraries to link with - # (more arguments like setup.py's Extension class: - # include_dirs=[..], extra_objects=[..], and so on) - - ffibuilder.cdef(""" - // declarations that are shared between Python and C - struct passwd { - char *pw_name; - ...; // literally dot-dot-dot - }; - struct passwd *getpwuid(int uid); // defined in - struct passwd *get_pw_for_root(void); // defined in set_source() - """) - - if __name__ == "__main__": - ffibuilder.compile(verbose=True) - -You need to run the ``example_build.py`` script once to generate -"source code" into the file ``_example.c`` and compile this to a -regular C extension module. (CFFI selects either Python or C for the -module to generate based on whether the second argument to -``set_source()`` is ``None`` or not.) - -*You need a C compiler for this single step. It produces a file called -e.g. _example.so or _example.pyd. If needed, it can be distributed in -precompiled form like any other extension module.* - -Then, in your main program, you use: - -.. code-block:: python - - from _example import ffi, lib - - p = lib.getpwuid(0) - assert ffi.string(p.pw_name) == b'root' - p = lib.get_pw_for_root() - assert ffi.string(p.pw_name) == b'root' - -Note that this works independently of the exact C layout of ``struct -passwd`` (it is "API level", as opposed to "ABI level"). It requires -a C compiler in order to run ``example_build.py``, but it is much more -portable than trying to get the details of the fields of ``struct -passwd`` exactly right. Similarly, in the ``cdef()`` we declared -``getpwuid()`` as taking an ``int`` argument; on some platforms this -might be slightly incorrect---but it does not matter. - -Note also that at runtime, the API mode is faster than the ABI mode. - -To integrate it inside a ``setup.py`` distribution with Setuptools: - -.. code-block:: python - - from setuptools import setup - - setup( - ... - setup_requires=["cffi>=1.0.0"], - cffi_modules=["example_build.py:ffibuilder"], - install_requires=["cffi>=1.0.0"], - ) Struct/Array Example (minimal, in-line) ---------------------------------------- ++++++++++++++++++++++++++++++++++++++++ .. code-block:: python @@ -183,17 +217,18 @@ *This example does not call any C compiler.* This example also admits an out-of-line equivalent. It is similar to -`Real example (API level, out-of-line)`_ above, but passing ``None`` as -the second argument to ``ffibuilder.set_source()``. Then in the main -program you write ``from _simple_example import ffi`` and then the same -content as the in-line example above starting from the line ``image = +the first example `Example: calling a C function from Python`_ above, +but passing ``None`` as the second argument to +``ffibuilder.set_source()``. Then in the main program you write +``from _simple_example import ffi`` and then the same content as the +in-line example above starting from the line ``image = ffi.new("pixel_t[]", 800*600)``. .. _performance: Purely for performance (API level, out-of-line) ------------------------------------------------ ++++++++++++++++++++++++++++++++++++++++++++++++ A variant of the `section above`__ where the goal is not to call an existing C library, but to compile and call some C function written @@ -244,7 +279,7 @@ .. _out-of-line-abi-level: Out-of-line, ABI level ----------------------- +++++++++++++++++++++++ The out-of-line ABI mode is a mixture of the regular (API) out-of-line mode and the in-line ABI mode. It lets you use the ABI mode, with its From pypy.commits at gmail.com Tue Mar 20 07:06:38 2018 From: pypy.commits at gmail.com (mjacob) Date: Tue, 20 Mar 2018 04:06:38 -0700 (PDT) Subject: [pypy-commit] buildbot default: Add link to py3.6 summary. Message-ID: <5ab0eb3e.dd8bdf0a.2016f.83d7@mx.google.com> Author: Manuel Jacob Branch: Changeset: r1059:d86c1810081c Date: 2018-03-20 12:06 +0100 http://bitbucket.org/pypy/buildbot/changeset/d86c1810081c/ Log: Add link to py3.6 summary. diff --git a/master/templates/layout.html b/master/templates/layout.html --- a/master/templates/layout.html +++ b/master/templates/layout.html @@ -28,6 +28,7 @@ - - Summary (trunk) - Summary (py3.5) + - Summary (py3.6) - Summary - Nightly builds From pypy.commits at gmail.com Tue Mar 20 07:53:43 2018 From: pypy.commits at gmail.com (mjacob) Date: Tue, 20 Mar 2018 04:53:43 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Remove EOL whitespace. Message-ID: <5ab0f647.158e1c0a.745d8.a03c@mx.google.com> Author: Manuel Jacob Branch: py3.6 Changeset: r94022:9f4edb1898f2 Date: 2018-03-20 12:53 +0100 http://bitbucket.org/pypy/pypy/changeset/9f4edb1898f2/ Log: Remove EOL whitespace. diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -78,25 +78,25 @@ def f(): (i for i in x) = 10 - + async def foo(a=await something()): pass - + async def foo(): await - + def foo(): await something() - + async def foo(): yield - + async def foo(): yield from [] - + async def foo(): await await fut - + """) From pypy.commits at gmail.com Tue Mar 20 08:44:52 2018 From: pypy.commits at gmail.com (Alexander Schremmer) Date: Tue, 20 Mar 2018 05:44:52 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger-updated: Make it compile, breaks identityhash of prebuilt objects. Message-ID: <5ab10244.55a81c0a.1063c.800a@mx.google.com> Author: Alexander Schremmer Branch: reverse-debugger-updated Changeset: r94026:1f98c4723883 Date: 2018-03-20 13:44 +0100 http://bitbucket.org/pypy/pypy/changeset/1f98c4723883/ Log: Make it compile, breaks identityhash of prebuilt objects. diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -410,17 +410,8 @@ RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj) { - /* Boehm only */ - if (obj->h_hash == 0) { - /* We never need to record anything: if h_hash is zero (which - is the case for all newly allocated objects), then we just - copy h_uid. This gives a stable answer. This would give - 0 for all prebuilt objects, but these should not have a - null h_hash anyway. - */ - obj->h_hash = obj->h_uid; - } - return obj->h_hash; + /* XXX This will make all prebuilt objects have id-hash 0. */ + return obj->h_uid; } RPY_EXTERN From pypy.commits at gmail.com Tue Mar 20 09:11:26 2018 From: pypy.commits at gmail.com (mjacob) Date: Tue, 20 Mar 2018 06:11:26 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Raise SyntaxError when 'return' with value is used inside an async generator. Message-ID: <5ab1087e.4dce1c0a.f881d.9b18@mx.google.com> Author: Manuel Jacob Branch: py3.6 Changeset: r94027:1938b1ea9975 Date: 2018-03-20 14:10 +0100 http://bitbucket.org/pypy/pypy/changeset/1938b1ea9975/ Log: Raise SyntaxError when 'return' with value is used inside an async generator. diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -263,6 +263,9 @@ def note_return(self, ret): if ret.value: + if self.is_coroutine and self.is_generator: + raise SyntaxError("'return' with value in async generator", + ret.lineno, ret.col_offset) self.return_with_value = True self.ret = ret diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py --- a/pypy/interpreter/test/test_syntax.py +++ b/pypy/interpreter/test/test_syntax.py @@ -94,6 +94,10 @@ async def foo(): await await fut + async def foo(): + yield + return 42 + """) From pypy.commits at gmail.com Tue Mar 20 10:23:44 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Mar 2018 07:23:44 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Inject a fake pytest module for untranslated new-style applevel tests Message-ID: <5ab11970.88d31c0a.f9170.accc@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94028:488e2edbef85 Date: 2018-03-20 13:27 +0100 http://bitbucket.org/pypy/pypy/changeset/488e2edbef85/ Log: Inject a fake pytest module for untranslated new-style applevel tests diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -263,21 +263,21 @@ raise raise oefmt(space.w_AssertionError, "DID NOT RAISE") -app_raises = gateway.interp2app_temp(pypyraises) +app_raises = gateway.interp2app(pypyraises) def pypyskip(space, w_message): """skip a test at app-level. """ msg = space.unwrap(w_message) py.test.skip(msg) -app_skip = gateway.interp2app_temp(pypyskip) +app_skip = gateway.interp2app(pypyskip) -def py3k_pypyskip(space, w_message): - """skip a test at app-level. """ - msg = space.unwrap(w_message) +def py3k_pypyskip(space, w_message): + """skip a test at app-level. """ + msg = space.unwrap(w_message) py.test.skip('[py3k] %s' % msg) -app_py3k_skip = gateway.interp2app_temp(py3k_pypyskip) +app_py3k_skip = gateway.interp2app(py3k_pypyskip) def raises_w(space, w_ExpectedException, *args, **kwds): try: diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -1,6 +1,8 @@ import sys +import os import pytest +from pypy import pypydir import pypy.interpreter.function from pypy.interpreter.error import OperationError from pypy.tool.pytest import objspace @@ -19,7 +21,7 @@ stat, co = _pytest.assertion.rewrite._rewrite_test(None, py.path.local('%s')) mod = _pytest.assertion.rewrite.create_module(co) return mod - """ % (self.config.rootdir, str(self.fspath))) + """ % (os.path.dirname(pypydir), str(self.fspath))) mod_dict = w_mod.getdict(space).unwrap(space) items = [] for name, w_obj in mod_dict.items(): diff --git a/pypy/tool/pytest/fake_pytest/__init__.py b/pypy/tool/pytest/fake_pytest/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/fake_pytest/__init__.py @@ -0,0 +1,9 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + applevel_name = 'pytest' + interpleveldefs = { + 'raises': 'interp_pytest.pypyraises', + 'skip': 'interp_pytest.pypyskip' + } + appleveldefs = {} diff --git a/pypy/tool/pytest/fake_pytest/interp_pytest.py b/pypy/tool/pytest/fake_pytest/interp_pytest.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/fake_pytest/interp_pytest.py @@ -0,0 +1,1 @@ +from pypy.tool.pytest.appsupport import pypyraises, pypyskip diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -30,6 +30,7 @@ config = make_config(option) if config.objspace.usemodules.thread: config.translation.thread = True + config.objspace.extmodules = 'pypy.tool.pytest.fake_pytest' space = make_objspace(config) space.startup() # Initialize all builtin modules space.setitem(space.builtin.w_dict, space.wrap('AssertionError'), From pypy.commits at gmail.com Tue Mar 20 10:38:52 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 20 Mar 2018 07:38:52 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: (anto, matti): start a branch in which to speed up things like PySlice_Check and similar Message-ID: <5ab11cfc.986f1c0a.cca51.32ba@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94029:3ee549601971 Date: 2018-03-20 14:30 +0100 http://bitbucket.org/pypy/pypy/changeset/3ee549601971/ Log: (anto, matti): start a branch in which to speed up things like PySlice_Check and similar From pypy.commits at gmail.com Tue Mar 20 10:38:54 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 20 Mar 2018 07:38:54 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: (anto, matti): add a passing test Message-ID: <5ab11cfe.c59adf0a.efc9c.c40d@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94030:c05e38122aa5 Date: 2018-03-20 15:09 +0100 http://bitbucket.org/pypy/pypy/changeset/c05e38122aa5/ Log: (anto, matti): add a passing test diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py --- a/pypy/module/cpyext/test/test_sliceobject.py +++ b/pypy/module/cpyext/test/test_sliceobject.py @@ -79,3 +79,14 @@ """), ]) assert module.get_ellipsis() is Ellipsis + + def test_typecheck(self): + module = self.import_extension('foo', [ + ("check", "METH_O", + """ + PySliceObject *slice = (PySliceObject *)args; + return PyLong_FromLong(PySlice_Check(slice)); + """), + ]) + s = slice(10, 20, 30) + assert module.check(s) From pypy.commits at gmail.com Tue Mar 20 10:38:56 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 20 Mar 2018 07:38:56 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: (anto, matti) introduce a new way to implement Py*_Check efficiently: we add a Message-ID: <5ab11d00.06d21c0a.4934e.d996@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94031:185b284722b0 Date: 2018-03-20 15:25 +0100 http://bitbucket.org/pypy/pypy/changeset/185b284722b0/ Log: (anto, matti) introduce a new way to implement Py*_Check efficiently: we add a new field to typeobject which containts flags for types like float, which don't have an official Py_TPFLAGS_*_SUBCLASS. Then, we can implement Py*_Check and Py*_CheckExact as a C macro, avoiding the slow roundtrip to RPython diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -133,6 +133,11 @@ 'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3 constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name) +# PyPy-specific flags +for name in ('FLOAT',): + constant_names.append('Py_TPPYPYFLAGS_%s_SUBCLASS' % name) + + for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) globals().update(rffi_platform.configure(CConfig_constants)) diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py --- a/pypy/module/cpyext/floatobject.py +++ b/pypy/module/cpyext/floatobject.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (PyObjectFields, bootstrap_function, cpython_struct, - CANNOT_FAIL, cpython_api, PyObject, build_type_checkers, CONST_STRING) + CANNOT_FAIL, cpython_api, PyObject, CONST_STRING) from pypy.module.cpyext.pyobject import ( make_typedescr, track_reference, from_ref) from pypy.interpreter.error import OperationError @@ -38,8 +38,6 @@ track_reference(space, obj, w_obj) return w_obj -PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") - @cpython_api([lltype.Float], PyObject) def PyFloat_FromDouble(space, value): return space.newfloat(value) diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h --- a/pypy/module/cpyext/include/floatobject.h +++ b/pypy/module/cpyext/include/floatobject.h @@ -32,6 +32,11 @@ return PyFloat_FromDouble(-Py_HUGE_VAL); \ } while(0) +#define PyFloat_Check(op) \ + _PyPy_Type_FastSubclass((op)->ob_type, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) +#define PyFloat_CheckExact(op) ((op)->ob_type == &PyFloat_Type) + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -228,6 +228,11 @@ #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1L<<30) #define Py_TPFLAGS_TYPE_SUBCLASS (1L<<31) +/* These are conceptually the same as the flags above, but they are + PyPy-specific and are stored inside tp_pypy_flags */ +#define Py_TPPYPYFLAGS_FLOAT_SUBCLASS (1L<<0) + + #define Py_TPFLAGS_DEFAULT_EXTERNAL ( \ Py_TPFLAGS_HAVE_GETCHARBUFFER | \ Py_TPFLAGS_HAVE_SEQUENCE_IN | \ @@ -247,6 +252,8 @@ #define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) #define PyType_FastSubclass(t,f) PyType_HasFeature(t,f) +#define _PyPy_Type_FastSubclass(t,f) (((t)->tp_pypy_flags & (f)) != 0) + #define PyType_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) #define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type) diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -311,6 +311,10 @@ /* Type attribute cache version tag. Added in version 2.6 */ unsigned int tp_version_tag; + /* PyPy specific extra fields: make sure that they are ALWAYS at the end, + for compatibility with CPython */ + long tp_pypy_flags; + } PyTypeObject; typedef struct _heaptypeobject { diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -102,9 +102,11 @@ """ PyObject* pyobj = PyFloat_FromDouble(1.0); PyFloatObject* pfo = (PyFloatObject*)pyobj; - int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) && - PyFloat_Check(pfo) && PyFloat_CheckExact(pfo); + int res = (PyFloat_Check(pyobj) + + PyFloat_CheckExact(pyobj) * 10 + + PyFloat_Check(pfo) * 100 + + PyFloat_CheckExact(pfo) * 1000); Py_DecRef(pyobj); return PyLong_FromLong(res);"""), ]) - assert module.test() == 1 + assert module.test() == 1111 diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py --- a/pypy/module/cpyext/test/test_number.py +++ b/pypy/module/cpyext/test/test_number.py @@ -11,7 +11,6 @@ PyNumber_Index, PyNumber_Coerce, PyNumber_CoerceEx, PyNumber_Add, PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute, PyNumber_Power, PyNumber_InPlacePower) -from pypy.module.cpyext.floatobject import PyFloat_Check from pypy.module.cpyext.intobject import PyInt_CheckExact from pypy.module.cpyext.longobject import PyLong_CheckExact from pypy.module.cpyext.object import PyObject_Size @@ -86,7 +85,7 @@ w_res = from_ref(space, ppl[0]) - assert PyFloat_Check(space, w_res) + assert space.isinstance_w(w_res, space.w_float) assert space.unwrap(w_res) == 123. decref(space, pl) decref(space, pf) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1537,4 +1537,29 @@ pass assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0 + def test_has_pypy_subclass_flag(self): + module = self.import_extension('foo', [ + ("test_pypy_flags", "METH_VARARGS", + ''' + long long in_flag, my_flag; + PyObject * obj; + if (!PyArg_ParseTuple(args, "OL", &obj, &in_flag)) + return NULL; + if (!PyType_Check(obj)) + { + PyErr_SetString(PyExc_ValueError, "input must be type"); + return NULL; + } + my_flag = ((PyTypeObject*)obj)->tp_pypy_flags; + if ((my_flag & in_flag) != in_flag) + return PyLong_FromLong(-1); + return PyLong_FromLong(0); + '''),]) + # copied from object.h + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + class MyFloat(float): + pass + assert module.test_pypy_flags(float, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + assert module.test_pypy_flags(MyFloat, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -22,6 +22,7 @@ Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS, Py_TPFLAGS_TYPE_SUBCLASS, Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3 + Py_TPPYPYFLAGS_FLOAT_SUBCLASS, ) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, @@ -426,6 +427,9 @@ pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS elif space.issubtype_w(w_obj, space.w_dict): pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS + # the following types are a pypy-specific extensions, using tp_pypy_flags + elif space.issubtype_w(w_obj, space.w_float): + pto.c_tp_pypy_flags |= Py_TPPYPYFLAGS_FLOAT_SUBCLASS def check_descr(space, w_self, w_type): if not space.isinstance_w(w_self, w_type): From pypy.commits at gmail.com Tue Mar 20 10:38:58 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 20 Mar 2018 07:38:58 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: implement PySlice_Check as a fast macro instead of going through the slow roundtrip Message-ID: <5ab11d02.135e1c0a.2df20.2039@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94032:4744cec39496 Date: 2018-03-20 15:31 +0100 http://bitbucket.org/pypy/pypy/changeset/4744cec39496/ Log: implement PySlice_Check as a fast macro instead of going through the slow roundtrip diff --git a/pypy/module/cpyext/include/sliceobject.h b/pypy/module/cpyext/include/sliceobject.h --- a/pypy/module/cpyext/include/sliceobject.h +++ b/pypy/module/cpyext/include/sliceobject.h @@ -17,6 +17,8 @@ PyObject *step; } PySliceObject; +#define PySlice_Check(op) ((op)->ob_type == &PySlice_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -47,7 +47,6 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) -PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @cpython_api([PyObject, PyObject, PyObject], PyObject) def PySlice_New(space, w_start, w_stop, w_step): diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py --- a/pypy/module/cpyext/test/test_sliceobject.py +++ b/pypy/module/cpyext/test/test_sliceobject.py @@ -2,14 +2,8 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP -from pypy.module.cpyext.sliceobject import PySlice_Check class TestSliceObject(BaseApiTest): - def test_slice(self, space): - w_i = space.wrap(10) - w_slice = space.newslice(w_i, w_i, w_i) - assert PySlice_Check(space, w_slice) - assert not PySlice_Check(space, w_i) def test_GetIndicesEx(self, space, api): w = space.wrap From pypy.commits at gmail.com Tue Mar 20 10:38:59 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 20 Mar 2018 07:38:59 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: (anto, matti): fix the tests broken by the previous commit: we are no longer Message-ID: <5ab11d03.42ae1c0a.65e0.a0e6@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94033:403e8744b785 Date: 2018-03-20 15:37 +0100 http://bitbucket.org/pypy/pypy/changeset/403e8744b785/ Log: (anto, matti): fix the tests broken by the previous commit: we are no longer able to call PySlice_Check directly, but the isinstance check should be enough diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -74,9 +74,8 @@ normal slices. Returns 0 on success and -1 on error with exception set.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \ w_slice.indices4(space, length) return 0 @@ -96,9 +95,8 @@ objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, in the source of your extension.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0] = \ w_slice.indices3(space, length) return 0 From pypy.commits at gmail.com Tue Mar 20 10:54:20 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Mar 2018 07:54:20 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: anto, matti) implement PyBool_Check as a fast macro Message-ID: <5ab1209c.06bbdf0a.f5a26.aaa6@mx.google.com> Author: Matti Picus Branch: cpyext-fast-typecheck Changeset: r94034:833681add062 Date: 2018-03-20 16:53 +0200 http://bitbucket.org/pypy/pypy/changeset/833681add062/ Log: anto, matti) implement PyBool_Check as a fast macro diff --git a/pypy/module/cpyext/boolobject.py b/pypy/module/cpyext/boolobject.py --- a/pypy/module/cpyext/boolobject.py +++ b/pypy/module/cpyext/boolobject.py @@ -1,9 +1,5 @@ -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, - build_type_checkers) - -# Inheriting from bool isn't actually possible. -PyBool_Check = build_type_checkers("Bool")[1] +from rpython.rtyper.lltypesystem import rffi +from pypy.module.cpyext.api import cpython_api, PyObject @cpython_api([rffi.LONG], PyObject) def PyBool_FromLong(space, value): diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -16,6 +16,8 @@ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py --- a/pypy/module/cpyext/test/test_boolobject.py +++ b/pypy/module/cpyext/test/test_boolobject.py @@ -1,7 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong -from pypy.module.cpyext.floatobject import PyFloat_FromDouble +from pypy.module.cpyext.boolobject import PyBool_FromLong class TestBoolObject(BaseApiTest): def test_fromlong(self, space): @@ -12,12 +11,6 @@ else: assert obj is space.w_False - def test_check(self, space): - assert PyBool_Check(space, space.w_True) - assert PyBool_Check(space, space.w_False) - assert not PyBool_Check(space, space.w_None) - assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0)) - class AppTestBoolMacros(AppTestCpythonExtensionBase): def test_macros(self): module = self.import_extension('foo', [ @@ -42,4 +35,14 @@ assert module.to_int(False) == 0 assert module.to_int(True) == 1 - + def test_check(self): + module = self.import_extension('foo', [ + ("type_check", "METH_O", + ''' + return PyLong_FromLong(PyBool_Check(args)); + ''')]) + assert module.type_check(True) + assert module.type_check(False) + assert not module.type_check(None) + assert not module.type_check(1.0) + From pypy.commits at gmail.com Tue Mar 20 10:57:23 2018 From: pypy.commits at gmail.com (Raemi) Date: Tue, 20 Mar 2018 07:57:23 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: (arigo, remi) fix test Message-ID: <5ab12153.a6a0df0a.9ccbb.7a5d@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94035:c13441bb1f3b Date: 2018-03-20 15:56 +0100 http://bitbucket.org/pypy/pypy/changeset/c13441bb1f3b/ Log: (arigo, remi) fix test diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -64,7 +64,8 @@ def register_quasi_immut_field(self, op, optimizer): from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr - assert optimizer.ensure_ptr_info_arg0(op)._compatibility_conditions is self + if optimizer is not None: # for test + assert optimizer.ensure_ptr_info_arg0(op)._compatibility_conditions is self descr = op.getdescr() assert isinstance(descr, QuasiImmutDescr) self.last_quasi_immut_field_descr = descr diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py --- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py @@ -538,7 +538,8 @@ # call with quasi-immut box = InputArgRef() ccond.register_quasi_immut_field( - ResOperation(rop.QUASIIMMUT_FIELD, [box], self.quasiimmutdescr)) + ResOperation(rop.QUASIIMMUT_FIELD, [box], self.quasiimmutdescr), + optimizer=None) getfield_op = ResOperation( rop.GETFIELD_GC_I, [box], self.quasifielddescr) op = ResOperation( From pypy.commits at gmail.com Tue Mar 20 12:19:17 2018 From: pypy.commits at gmail.com (Raemi) Date: Tue, 20 Mar 2018 09:19:17 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: (arigo, remi) test and fix a case of too many bridges Message-ID: <5ab13485.d1581c0a.238d5.ec6f@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94036:931f5eaed82f Date: 2018-03-20 17:18 +0100 http://bitbucket.org/pypy/pypy/changeset/931f5eaed82f/ Log: (arigo, remi) test and fix a case of too many bridges see test for explanation diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -1,6 +1,5 @@ from rpython.jit.codewriter import heaptracker, longlong from rpython.jit.codewriter.jitcode import JitCode, SwitchDictDescr -from rpython.jit.metainterp.compile import ResumeAtPositionDescr from rpython.jit.metainterp.jitexc import get_llexception, reraise from rpython.jit.metainterp import jitexc from rpython.jit.metainterp.history import MissingValue diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -692,6 +692,8 @@ class ResumeDescr(AbstractFailDescr): _attrs_ = () + resume_at_loop_start = False + def clone(self): return self @@ -901,7 +903,7 @@ pass class ResumeAtPositionDescr(ResumeGuardDescr): - pass + resume_at_loop_start = True class CompileLoopVersionDescr(ResumeGuardDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): @@ -1062,7 +1064,7 @@ metainterp_sd.jitlog.start_new_trace(metainterp_sd, faildescr=resumekey, entry_bridge=False, jd_name=jd_name) # - if isinstance(resumekey, ResumeAtPositionDescr): + if resumekey.resume_at_loop_start: inline_short_preamble = False else: inline_short_preamble = True @@ -1109,6 +1111,7 @@ metainterp.retrace_needed(new_trace, info) return None + class GuardCompatibleDescr(ResumeGuardDescr): """ A descr for guard_compatible. All the conditions that a value should fulfil need to be attached to this descr by optimizeopt. """ @@ -1214,6 +1217,10 @@ return [] +class ResumeAtPositionForCompatibleDescr(GuardCompatibleDescr): + resume_at_loop_start = True + + # ____________________________________________________________ memory_error = MemoryError() diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -542,8 +542,7 @@ raise InvalidLoop('A GUARD_CLASS (%s) was proven to always fail' % r) old_guard_op = info.get_last_guard(self.optimizer) - if old_guard_op and not isinstance(old_guard_op.getdescr(), - compile.ResumeAtPositionDescr): + if old_guard_op and not old_guard_op.getdescr().resume_at_loop_start: # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value. if old_guard_op.getopnum() == rop.GUARD_NONNULL: @@ -566,8 +565,8 @@ expectedclassbox = op.getarg(1) info = self.getptrinfo(op.getarg(0)) old_guard_op = info.get_last_guard(self.optimizer) - update_last_guard = not old_guard_op or isinstance( - old_guard_op.getdescr(), compile.ResumeAtPositionDescr) + update_last_guard = (not old_guard_op or + old_guard_op.getdescr().resume_at_loop_start) self.make_constant_class(op.getarg(0), expectedclassbox, update_last_guard) def optimize_GUARD_NONNULL_CLASS(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -342,7 +342,7 @@ if isinstance(guard, GuardResOp): guard.rd_resume_position = patchguardop.rd_resume_position if guard.opnum == rop.GUARD_COMPATIBLE: - guard.setdescr(compile.GuardCompatibleDescr()) + guard.setdescr(compile.ResumeAtPositionForCompatibleDescr()) else: guard.setdescr(compile.ResumeAtPositionDescr()) self.send_extra_operation(guard) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2496,7 +2496,7 @@ self.current_merge_points = [] self.resumekey = resumedescr self.seen_loop_header_for_jdindex = -1 - if isinstance(key, compile.ResumeAtPositionDescr): + if key.resume_at_loop_start: self.seen_loop_header_for_jdindex = self.jitdriver_sd.index self.prepare_resume_from_failure(deadframe, inputargs, resumedescr) if self.resumekey_original_loop_token is None: # very rare case diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -2,7 +2,6 @@ from rpython.rlib import jit from rpython.rtyper.lltypesystem import lltype, rffi - class TestCompatible(LLJitMixin): def test_simple(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) @@ -613,6 +612,59 @@ self.check_resops(call_i=0) + def test_short_preamble_resume_at_loop_start(self): + """check that an inserted guard_compatible in the short preamble will always + resume at the loop start and not create a bridge (which contains + another loop iteration with guards, causing a lot of bridges to + appear). Basically do the same as for guard_value in short preambles. + """ + from rpython.rlib.objectmodel import we_are_translated + + class C(object): + def __init__(self, x): + self.x = x + + p1 = C(1) + p1.link = None + + p2 = C(2) + p2.link = C(1) + p2.link.link = None + + p2a = C(2) + p2a.link = C(2) + p2a.link.link = C(1) + + driver = jit.JitDriver(greens=[], reds=['n', 'x']) + + @jit.elidable_compatible() + def g(o): + return o.x + + def f(n, x): + res = 0 + while n > 0: + driver.can_enter_jit(n=n, x=x) + driver.jit_merge_point(n=n, x=x) + x = jit.hint(x, promote_compatible=True) + res = g(x) + if res == 2: + x = x.link + n -= res + return res + + def main(x): + jit.set_param(driver, 'trace_eagerness', 1) + res = f(100, p1) + res += f(100, p2) + res += f(100, p2a) + return res + + x = self.meta_interp(main, [False]) + self.check_trace_count(4) + self.check_resops(int_sub=3) # not 4! + + def test_like_objects(self): from rpython.rlib.objectmodel import we_are_translated class Map(object): From pypy.commits at gmail.com Tue Mar 20 12:30:05 2018 From: pypy.commits at gmail.com (mjacob) Date: Tue, 20 Mar 2018 09:30:05 -0700 (PDT) Subject: [pypy-commit] pypy py3.6: Add gc_collect() to make it crash more reliably. Yay! Message-ID: <5ab1370d.03b6df0a.62f5a.84c0@mx.google.com> Author: Manuel Jacob Branch: py3.6 Changeset: r94037:02930c62826c Date: 2018-03-20 17:29 +0100 http://bitbucket.org/pypy/pypy/changeset/02930c62826c/ Log: Add gc_collect() to make it crash more reliably. Yay! diff --git a/lib-python/3/test/test_asyncgen.py b/lib-python/3/test/test_asyncgen.py --- a/lib-python/3/test/test_asyncgen.py +++ b/lib-python/3/test/test_asyncgen.py @@ -5,7 +5,7 @@ from unittest import mock -from test.support import import_module +from test.support import import_module, gc_collect asyncio = import_module("asyncio") @@ -623,6 +623,7 @@ await g.__anext__() await g.__anext__() del g + gc_collect() await asyncio.sleep(0.1, loop=self.loop) From pypy.commits at gmail.com Tue Mar 20 13:46:45 2018 From: pypy.commits at gmail.com (arigo) Date: Tue, 20 Mar 2018 10:46:45 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: (remi, arigo) Message-ID: <5ab14905.177a1c0a.65609.be05@mx.google.com> Author: Armin Rigo Branch: guard-compatible Changeset: r94038:ec6d67e6b1cb Date: 2018-03-20 18:45 +0100 http://bitbucket.org/pypy/pypy/changeset/ec6d67e6b1cb/ Log: (remi, arigo) Fix what looks like a bug by staring at the diff with default diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -665,8 +665,9 @@ self._last_guard_op) else: op = self.store_final_boxes_in_guard(guard_op, pendingfields) + self._last_guard_op = op if opnum == rop.GUARD_COMPATIBLE: - self._last_guard_op = op # XXX don't share the next one either + self._last_guard_op = None # XXX don't share the next one either # for unrolling for farg in op.getfailargs(): if farg: From pypy.commits at gmail.com Tue Mar 20 17:51:56 2018 From: pypy.commits at gmail.com (rlamy) Date: Tue, 20 Mar 2018 14:51:56 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Revert changes to _pytest/ made by this branch Message-ID: <5ab1827c.c7a3df0a.50e72.05cb@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94042:46d7ec4d97a8 Date: 2018-03-20 22:51 +0100 http://bitbucket.org/pypy/pypy/changeset/46d7ec4d97a8/ Log: Revert changes to _pytest/ made by this branch diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py --- a/_pytest/assertion/rewrite.py +++ b/_pytest/assertion/rewrite.py @@ -265,10 +265,10 @@ if (not source.startswith(BOM_UTF8) and cookie_re.match(source[0:end1]) is None and cookie_re.match(source[end1 + 1:end2]) is None): - # if hasattr(state, "_indecode"): - # # encodings imported us again, so don't rewrite. - # return None, None - # state._indecode = True + if hasattr(state, "_indecode"): + # encodings imported us again, so don't rewrite. + return None, None + state._indecode = True try: try: source.decode("ascii") @@ -293,21 +293,10 @@ except SyntaxError: # It's possible that this error is from some bug in the # assertion rewriting, but I don't know of a fast way to tell. - # state.trace("failed to compile: %r" % (fn,)) + state.trace("failed to compile: %r" % (fn,)) return None, None return stat, co - -def create_module(co): - """Hack to create a module from a code object created by _rewrite_test()""" - mod = imp.new_module(co.co_filename.split('/')[-1].split('.')[0]) - mod.__file__ = co.co_filename - # mod.__cached__ = pyc - mod.__loader__ = None - exec(co, mod.__dict__) - return mod - - def _make_rewritten_pyc(state, source_stat, pyc, co): """Try to dump rewritten code to *pyc*.""" if sys.platform.startswith("win"): From pypy.commits at gmail.com Tue Mar 20 19:20:02 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 20 Mar 2018 16:20:02 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: more debugging of finding cl.exe, 226287429316 did not print anything Message-ID: <5ab19722.51951c0a.31ec9.a3d5@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94043:3da473ecc0a8 Date: 2018-03-21 01:19 +0200 http://bitbucket.org/pypy/pypy/changeset/3da473ecc0a8/ Log: more debugging of finding cl.exe, 226287429316 did not print anything diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -288,6 +288,9 @@ value = value[:-1] result[key] = removeDuplicates(value) + except Exception as e: + log.debug(repr(e)) + raise finally: popen.stdout.close() popen.stderr.close() From pypy.commits at gmail.com Wed Mar 21 05:01:25 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Mar 2018 02:01:25 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: more msvc debugging for buildbot failure Message-ID: <5ab21f65.c7581c0a.c0153.816d@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94046:83c9f6e4e93b Date: 2018-03-21 11:00 +0200 http://bitbucket.org/pypy/pypy/changeset/83c9f6e4e93b/ Log: more msvc debugging for buildbot failure diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -266,7 +266,7 @@ if vcvarsall is None: raise DistutilsPlatformError("Unable to find vcvarsall.bat") - log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + log.debug("Calling 'vcvarsall.bat %s' (version=%s), python=%s", arch, version, sys.executable) popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py --- a/lib_pypy/_audioop_build.py +++ b/lib_pypy/_audioop_build.py @@ -656,4 +656,6 @@ ffi.set_source("_audioop_cffi", C_SOURCE) if __name__ == "__main__": + import sys + print('using python from', sys.executable) ffi.compile(verbose=2) From pypy.commits at gmail.com Wed Mar 21 06:25:48 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 21 Mar 2018 03:25:48 -0700 (PDT) Subject: [pypy-commit] pypy rpython-sprint: Close branch rpython-sprint Message-ID: <5ab2332c.88d31c0a.f9170.6144@mx.google.com> Author: Ronan Lamy Branch: rpython-sprint Changeset: r94047:cba6b37e11fc Date: 2018-03-21 10:25 +0000 http://bitbucket.org/pypy/pypy/changeset/cba6b37e11fc/ Log: Close branch rpython-sprint From pypy.commits at gmail.com Wed Mar 21 06:26:18 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 21 Mar 2018 03:26:18 -0700 (PDT) Subject: [pypy-commit] pypy default: Merged in rpython-sprint (pull request #598) Message-ID: <5ab2334a.0fbcdf0a.6de6.d186@mx.google.com> Author: Ronan Lamy Branch: Changeset: r94048:e4136ea7f1b6 Date: 2018-03-21 10:25 +0000 http://bitbucket.org/pypy/pypy/changeset/e4136ea7f1b6/ Log: Merged in rpython-sprint (pull request #598) Cleanup + document enforceargs() diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -103,7 +103,7 @@ elif bookkeeper and not hasattr(t, '_freeze_'): return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: - raise AssertionError("annotationoftype(%r)" % (t,)) + raise TypeError("Annotation of type %r not supported" % (t,)) class Sig(object): diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -259,6 +259,26 @@ intmask(). +Type Enforcing and Checking +--------------------------- + +RPython provides a helper decorator to force RPython-level types on function +arguments. The decorator, called ``enforceargs()``, accepts as parameters the +types expected to match the arguments of the function. + +Functions decorated with ``enforceargs()`` have their function signature +analyzed and their RPython-level type inferred at import time (for further +details about the flavor of translation performed in RPython, see the +`Annotation pass documentation`_). Encountering types not supported by RPython +will raise a ``TypeError``. + +``enforceargs()`` by default also performs type checking of parameter types +each time the function is invoked. To disable this behavior, it's possible to +pass the ``typecheck=False`` parameter to the decorator. + +.. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator + + Exception rules --------------- diff --git a/rpython/doc/translation.rst b/rpython/doc/translation.rst --- a/rpython/doc/translation.rst +++ b/rpython/doc/translation.rst @@ -48,7 +48,7 @@ be present in memory as a form that is "static enough" in the sense of :doc:`RPython `. -2. The Annotator_ performs a global analysis starting from an specified +2. The Annotator_ performs a global analysis starting from a specified entry point to deduce type and other information about what each variable can contain at run-time, :ref:`building flow graphs ` as it encounters them. diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -120,7 +120,7 @@ """ Decorate a function with forcing of RPython-level types on arguments. None means no enforcing. - When not translated, the type of the actual arguments are checked against + When not translated, the type of the actual arguments is checked against the enforced types every time the function is called. You can disable the typechecking by passing ``typecheck=False`` to @enforceargs. """ @@ -147,8 +147,7 @@ # they are already homogeneous, so we only check the first # item. The case of empty list/dict is handled inside typecheck() if isinstance(arg, list): - item = arg[0] - return [get_type_descr_of_argument(item)] + return [get_type_descr_of_argument(arg[0])] elif isinstance(arg, dict): key, value = next(arg.iteritems()) return {get_type_descr_of_argument(key): get_type_descr_of_argument(value)} From pypy.commits at gmail.com Wed Mar 21 06:44:49 2018 From: pypy.commits at gmail.com (Raemi) Date: Wed, 21 Mar 2018 03:44:49 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: fix missing part of a previous commit 931f5eaed82f Message-ID: <5ab237a1.56b4df0a.11db1.5222@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94049:84015df039c9 Date: 2018-03-21 11:43 +0100 http://bitbucket.org/pypy/pypy/changeset/84015df039c9/ Log: fix missing part of a previous commit 931f5eaed82f Apparently we missed another place where ResumeAtPosition..() was required. This second part of the previous commit 931f5eaed82f reduces the number of bridges created in the richards benchmark from twice as many bridges to slightly less bridges compared to default- pypy. However, there are tests failing that probably just need updating. diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -420,7 +420,7 @@ arglist = self._map_args(mapping, sop.getarglist()) if sop.is_guard(): if sop.opnum == rop.GUARD_COMPATIBLE: - descr = compile.GuardCompatibleDescr() + descr = compile.ResumeAtPositionForCompatibleDescr() else: descr = compile.ResumeAtPositionDescr() op = sop.copy_and_change(sop.getopnum(), arglist, From pypy.commits at gmail.com Wed Mar 21 07:05:36 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Mar 2018 04:05:36 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: debug printing fixed?, revert previous commits Message-ID: <5ab23c80.94101c0a.d8eae.89e5@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94050:6c7ff533e018 Date: 2018-03-21 13:03 +0200 http://bitbucket.org/pypy/pypy/changeset/6c7ff533e018/ Log: debug printing fixed?, revert previous commits diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -266,7 +266,7 @@ if vcvarsall is None: raise DistutilsPlatformError("Unable to find vcvarsall.bat") - log.debug("Calling 'vcvarsall.bat %s' (version=%s), python=%s", arch, version, sys.executable) + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -288,9 +288,6 @@ value = value[:-1] result[key] = removeDuplicates(value) - except Exception as e: - log.debug(repr(e)) - raise finally: popen.stdout.close() popen.stderr.close() @@ -298,7 +295,7 @@ if len(result) != len(interesting): raise ValueError(str(list(result.keys()))) - log.debug('Got', result) + log.debug('Got %s', str(result)) return result # More globals From pypy.commits at gmail.com Wed Mar 21 08:02:50 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Mar 2018 05:02:50 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: re-enable support for immutability tracking of attributes of user-defined Message-ID: <5ab249ea.ccc4df0a.2a51d.7639@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r94051:7825277c4910 Date: 2018-03-20 10:51 +0100 http://bitbucket.org/pypy/pypy/changeset/7825277c4910/ Log: re-enable support for immutability tracking of attributes of user- defined instances. This works by piggybacking the elidability on the map's version. When an attribute goes from immutable to mutable (ie a lot during startup) this approach changes versions a bit too much: for all children of the terminator of the involved attribute, a new version is generated. Could be improved a little bit. diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -76,23 +76,22 @@ # XXX can improve the devolved case terminator = self._get_terminator() return terminator._read_terminator(obj, name, index) - #if ( # XXX in the guard_compatible world the following isconstant may never be true? - # jit.isconstant(attr.storageindex) and - # jit.isconstant(obj) and - # not attr.ever_mutated - #): - # return self._pure_mapdict_read_storage(obj, attr.storageindex) - #else: - return obj._mapdict_read_storage(storageindex) - - @jit.elidable - def _pure_mapdict_read_storage(self, obj, storageindex): + if ( + jit.isconstant(name) and + jit.isconstant(index) and + jit.isconstant(obj) + ): + can_constfold = self.can_constfold_attr_read(name, index) + if can_constfold: + return _pure_mapdict_read_storage(obj, storageindex) return obj._mapdict_read_storage(storageindex) def write(self, obj, name, index, w_value): storageindex = self.find_map_storageindex(name, index) if storageindex < 0: return self._get_terminator()._write_terminator(obj, name, index, w_value) + if self.can_constfold_attr_read(name, index): + self.invalidate_ever_mutated(name, index) obj._mapdict_write_storage(storageindex, w_value) return True @@ -112,6 +111,24 @@ return NOATTR return attr.storageindex + @jit.elidable_compatible(quasi_immut_field_name_for_second_arg="version") + def can_constfold_attr_read(self, version, name, index): + if version is None: + return False + attr = self.find_map_attr(name, index) + if attr is None: + return False + return not attr.ever_mutated + + def invalidate_ever_mutated(self, name, index): + attr = self.find_map_attr(name, index) + assert not attr.ever_mutated + attr.ever_mutated = True + # XXX is the next line too expensive? + # it's necessary for the version to change, to make + # can_constfold_attr_read actually elidable + self.terminator.mutated(self.version is not None) + @jit.elidable_compatible() def find_map_attr(self, name, index): # attr cache @@ -384,14 +401,14 @@ self.w_cls = w_cls self.all_children = None - def mutated_w_cls_version(self, version): - if version is None: + def mutated(self, have_version): + if not have_version: self.version = None else: self.version = Version() if self.all_children is not None: for map in self.all_children: - if version is None: + if not have_version: map.version = None else: map.version = Version() @@ -437,9 +454,9 @@ Terminator.__init__(self, space, w_cls) self.devolved_dict_terminator = DevolvedDictTerminator(space, w_cls) - def mutated_w_cls_version(self, version): - self.devolved_dict_terminator.mutated_w_cls_version(version) - Terminator.mutated_w_cls_version(self, version) + def mutated(self, have_version): + self.devolved_dict_terminator.mutated(have_version) + Terminator.mutated(self, have_version) def materialize_r_dict(self, space, obj, dict_w): return self._make_devolved(space) @@ -500,7 +517,7 @@ return Terminator.set_terminator(self, obj, terminator) class PlainAttribute(AbstractAttribute): - _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order'] + _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated', 'order'] def __init__(self, name, index, back): AbstractAttribute.__init__(self, back.space, back.terminator) @@ -509,7 +526,7 @@ self.storageindex = back.length() self.back = back self._size_estimate = self.length() * NUM_DIGITS_POW2 - #self.ever_mutated = False # XXX XXX XXX immutability is disabled for now + self.ever_mutated = False self.order = len(back.cache_attrs) if back.cache_attrs else 0 def _copy_attr(self, obj, new_obj): @@ -518,9 +535,9 @@ def delete(self, obj, name, index): if index == self.index and name == self.name: + if not self.ever_mutated: + self.invalidate_ever_mutated(name, index) # ok, attribute is deleted - #if not self.ever_mutated: - # self.ever_mutated = True return self.back.copy(obj) new_obj = self.back.delete(obj, name, index) if new_obj is not None: @@ -602,6 +619,13 @@ INVALID = 2 SLOTS_STARTING_FROM = 3 + + + at jit.elidable +def _pure_mapdict_read_storage(obj, storageindex): + return obj._mapdict_read_storage(storageindex) + + # a little bit of a mess of mixin classes that implement various pieces of # objspace user object functionality in terms of mapdict diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -17,7 +17,7 @@ class Class(object): def __init__(self, hasdict=True): self.hasdict = hasdict - self._version_tag = None + self._version_tag = object() if hasdict: self.terminator = DictTerminator(space, self) else: @@ -33,6 +33,9 @@ result.user_setup(sp, self) return result + def mutated(self, _): + self._version_tag = object() + class ObjectWithoutDict(ObjectWithoutDict): class typedef: hasdict = False @@ -291,7 +294,7 @@ def test_attr_immutability(monkeypatch): - pytest.skip("disabled for now") + from pypy.objspace.std import mapdict cls = Class() obj = cls.instantiate() obj.setdictvalue(space, "a", 10) @@ -308,7 +311,7 @@ indices.append(storageindex) return obj._mapdict_read_storage(storageindex) - obj.map._pure_mapdict_read_storage = _pure_mapdict_read_storage + monkeypatch.setattr(mapdict, "_pure_mapdict_read_storage", _pure_mapdict_read_storage) monkeypatch.setattr(jit, "isconstant", lambda c: True) assert obj.getdictvalue(space, "a") == 10 @@ -329,7 +332,6 @@ assert obj2.map is obj.map def test_attr_immutability_delete(): - pytest.skip("disabled for now") cls = Class() obj = cls.instantiate() obj.setdictvalue(space, "a", 10) @@ -1031,14 +1033,14 @@ a.x = 42 def f(): return a.x - # + res = self.check(f, 'x') assert res == (1, 0, 0) res = self.check(f, 'x') assert res == (0, 1, 0) res = self.check(f, 'x') assert res == (0, 1, 0) - # + a.y = "foo" # changes the map res = self.check(f, 'x') assert res == (1, 0, 0) @@ -1046,8 +1048,18 @@ assert res == (0, 1, 0) res = self.check(f, 'x') assert res == (0, 1, 0) - # - a.y = "bar" # does not change the map any more + + # the following does not change the map, but changes the version since + # y goes from immutable to mutable + a.y = "bar" + res = self.check(f, 'x') + assert res == (1, 0, 0) + res = self.check(f, 'x') + assert res == (0, 1, 0) + res = self.check(f, 'x') + assert res == (0, 1, 0) + + a.y = "baz" # now everything works res = self.check(f, 'x') assert res == (0, 1, 0) res = self.check(f, 'x') diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -255,7 +255,7 @@ def _set_version_tag(self, version_tag): self._version_tag = version_tag - self.terminator.mutated_w_cls_version(version_tag) + self.terminator.mutated(version_tag is not None) def getattribute_if_not_from_object(self): """ this method returns the applevel __getattribute__ if that is not From pypy.commits at gmail.com Wed Mar 21 08:02:52 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Mar 2018 05:02:52 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: fix typo Message-ID: <5ab249ec.6291df0a.66f67.4721@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r94052:689868846245 Date: 2018-03-21 12:36 +0100 http://bitbucket.org/pypy/pypy/changeset/689868846245/ Log: fix typo diff --git a/rpython/jit/metainterp/compatible.py b/rpython/jit/metainterp/compatible.py --- a/rpython/jit/metainterp/compatible.py +++ b/rpython/jit/metainterp/compatible.py @@ -32,7 +32,7 @@ def __init__(self, ptr): self.known_valid = ptr self.conditions = [] - self.last_quasi_immut_field_op = None + self.last_quasi_immut_field_descr = None # -1 means "stay on the original trace" self.jump_target = -1 self.frozen = False From pypy.commits at gmail.com Wed Mar 21 08:02:54 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 21 Mar 2018 05:02:54 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: change the test slightly to show the real problem: calls aren't getting removed in bridges Message-ID: <5ab249ee.89c5df0a.58e15.9ee4@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: guard-compatible Changeset: r94053:d8319beb1fe4 Date: 2018-03-21 13:01 +0100 http://bitbucket.org/pypy/pypy/changeset/d8319beb1fe4/ Log: change the test slightly to show the real problem: calls aren't getting removed in bridges diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -531,11 +531,11 @@ main(False) x = self.meta_interp(main, [True]) - assert x < 30 - x = self.meta_interp(main, [False]) - assert x < 30 + x += self.meta_interp(main, [False]) self.check_trace_count(7) + self.check_resops(call_i=0) + assert x < 60 def test_quasi_immutable_merge_short_preamble(self): from rpython.rlib.objectmodel import we_are_translated From pypy.commits at gmail.com Wed Mar 21 12:46:24 2018 From: pypy.commits at gmail.com (standy66) Date: Wed, 21 Mar 2018 09:46:24 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-py3-instancemethod-attributes: [cpyext] Add missing attributes to instancemethod Message-ID: <5ab28c60.8d6d1c0a.790ed.3e44@mx.google.com> Author: Andrew Stepanov Branch: cpyext-py3-instancemethod-attributes Changeset: r94054:a178c81744b1 Date: 2018-03-21 15:37 +0300 http://bitbucket.org/pypy/pypy/changeset/a178c81744b1/ Log: [cpyext] Add missing attributes to instancemethod diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -3,7 +3,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.function import Method -from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w +from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w, GetSetProperty from pypy.interpreter.gateway import interp2app @@ -14,6 +14,15 @@ def __init__(self, w_function): self.w_function = w_function + def fget_name(self, space): + return space.getattr(self.w_function, space.newtext("__name__")) + + def fget_module(self, space): + return space.getattr(self.w_function, space.newtext("__module__")) + + def fget_docstring(self, space): + return space.getattr(self.w_function, space.newtext("__doc__")) + @staticmethod def descr_new(space, w_subtype, w_function): # instancemethod is not subclassable @@ -38,7 +47,10 @@ __get__ = interp2app(InstanceMethod.descr_get), __repr__ = interp2app(InstanceMethod.descr_repr, descrmismatch='__repr__'), - __func__= interp_attrproperty_w('w_function', cls=InstanceMethod), + __func__ = interp_attrproperty_w('w_function', cls=InstanceMethod), + __name__ = GetSetProperty(InstanceMethod.fget_name, cls=InstanceMethod), + __module__ = GetSetProperty(InstanceMethod.fget_module, cls=InstanceMethod), + __doc__ = GetSetProperty(InstanceMethod.fget_docstring, cls=InstanceMethod), ) InstanceMethod.typedef.acceptable_as_base_class = False diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -1,5 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class AppTestInstanceMethod(AppTestCpythonExtensionBase): def test_instancemethod(self): module = self.import_extension('foo', [ @@ -27,3 +28,29 @@ InstanceMethod.testmethod.attribute = "test" assert testfunction.attribute == "test" raises(AttributeError, setattr, inst.testmethod, "attribute", "test") + + def test_instancemethod_cpyext_attributes(self): + module = self.import_extension('foo', [ + ("instancemethod_get_doc", "METH_O", + """ + PyObject* instancemethod = PyInstanceMethod_New(args); + return PyObject_GetAttrString(instancemethod, "__doc__"); + """), + ("instancemethod_get_name", "METH_O", + """ + PyObject* instancemethod = PyInstanceMethod_New(args); + return PyObject_GetAttrString(instancemethod, "__name__"); + """), + ("instancemethod_get_module", "METH_O", + """ + PyObject* instancemethod = PyInstanceMethod_New(args); + return PyObject_GetAttrString(instancemethod, "__module__"); + """) + ]) + + def testfunction(self): + """some doc""" + return self + assert(module.instancemethod_get_doc(testfunction) == testfunction.__doc__) + assert(module.instancemethod_get_module(testfunction) == testfunction.__module__) + assert(module.instancemethod_get_name(testfunction) == testfunction.__name__) \ No newline at end of file From pypy.commits at gmail.com Wed Mar 21 12:46:28 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Mar 2018 09:46:28 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge branch which adds missing dunder-atributes to instancemethods Message-ID: <5ab28c64.2495df0a.a304f.5afc@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94056:93baaf2e069a Date: 2018-03-21 18:45 +0200 http://bitbucket.org/pypy/pypy/changeset/93baaf2e069a/ Log: merge branch which adds missing dunder-atributes to instancemethods diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -13,4 +13,7 @@ Update winreg module to use unicode, wide-strings +.. branch: cpyext-py3-instancemethod-attributes +Add missing ``__doc__``, ``__module__``, ``__name__`` attributes to +``instancemethod`` diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -3,7 +3,7 @@ from pypy.module.cpyext.pyobject import PyObject from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.function import Method -from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w +from pypy.interpreter.typedef import TypeDef, interp_attrproperty_w, GetSetProperty from pypy.interpreter.gateway import interp2app @@ -14,6 +14,15 @@ def __init__(self, w_function): self.w_function = w_function + def fget_name(self, space): + return space.getattr(self.w_function, space.newtext("__name__")) + + def fget_module(self, space): + return space.getattr(self.w_function, space.newtext("__module__")) + + def fget_docstring(self, space): + return space.getattr(self.w_function, space.newtext("__doc__")) + @staticmethod def descr_new(space, w_subtype, w_function): # instancemethod is not subclassable @@ -38,7 +47,10 @@ __get__ = interp2app(InstanceMethod.descr_get), __repr__ = interp2app(InstanceMethod.descr_repr, descrmismatch='__repr__'), - __func__= interp_attrproperty_w('w_function', cls=InstanceMethod), + __func__ = interp_attrproperty_w('w_function', cls=InstanceMethod), + __name__ = GetSetProperty(InstanceMethod.fget_name, cls=InstanceMethod), + __module__ = GetSetProperty(InstanceMethod.fget_module, cls=InstanceMethod), + __doc__ = GetSetProperty(InstanceMethod.fget_docstring, cls=InstanceMethod), ) InstanceMethod.typedef.acceptable_as_base_class = False diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -1,5 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase + class AppTestInstanceMethod(AppTestCpythonExtensionBase): def test_instancemethod(self): module = self.import_extension('foo', [ @@ -27,3 +28,29 @@ InstanceMethod.testmethod.attribute = "test" assert testfunction.attribute == "test" raises(AttributeError, setattr, inst.testmethod, "attribute", "test") + + def test_instancemethod_cpyext_attributes(self): + module = self.import_extension('foo', [ + ("instancemethod_get_doc", "METH_O", + """ + PyObject* instancemethod = PyInstanceMethod_New(args); + return PyObject_GetAttrString(instancemethod, "__doc__"); + """), + ("instancemethod_get_name", "METH_O", + """ + PyObject* instancemethod = PyInstanceMethod_New(args); + return PyObject_GetAttrString(instancemethod, "__name__"); + """), + ("instancemethod_get_module", "METH_O", + """ + PyObject* instancemethod = PyInstanceMethod_New(args); + return PyObject_GetAttrString(instancemethod, "__module__"); + """) + ]) + + def testfunction(self): + """some doc""" + return self + assert(module.instancemethod_get_doc(testfunction) == testfunction.__doc__) + assert(module.instancemethod_get_module(testfunction) == testfunction.__module__) + assert(module.instancemethod_get_name(testfunction) == testfunction.__name__) \ No newline at end of file From pypy.commits at gmail.com Wed Mar 21 12:46:26 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Mar 2018 09:46:26 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-py3-instancemethod-attributes: document, close branch to be merged Message-ID: <5ab28c62.96e81c0a.32b7f.27e0@mx.google.com> Author: Matti Picus Branch: cpyext-py3-instancemethod-attributes Changeset: r94055:4800349b3d58 Date: 2018-03-21 18:43 +0200 http://bitbucket.org/pypy/pypy/changeset/4800349b3d58/ Log: document, close branch to be merged diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -13,4 +13,7 @@ Update winreg module to use unicode, wide-strings +.. branch: cpyext-py3-instancemethod-attributes +Add missing ``__doc__``, ``__module__``, ``__name__`` attributes to +``instancemethod`` From pypy.commits at gmail.com Wed Mar 21 13:45:56 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Mar 2018 10:45:56 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: running vcvarsall.bat is not changing path, try to figure out why Message-ID: <5ab29a54.c592df0a.3f9c2.8bd8@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94057:016ce1e2ce8b Date: 2018-03-21 19:45 +0200 http://bitbucket.org/pypy/pypy/changeset/016ce1e2ce8b/ Log: running vcvarsall.bat is not changing path, try to figure out why diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -266,7 +266,7 @@ if vcvarsall is None: raise DistutilsPlatformError("Unable to find vcvarsall.bat") - log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + log.debug("Calling '%s %s' (version=%s)", vcvarsall, arch, version) popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -276,6 +276,10 @@ raise DistutilsPlatformError(stderr.decode("mbcs")) stdout = stdout.decode("mbcs") + log.debug('-'*30) + log.debug(stderr.decode('mbcs')) + log.debug(stdout.decode('mbcs')) + log.debug('-'*30) for line in stdout.split("\n"): line = Reg.convert_mbcs(line) if '=' not in line: From pypy.commits at gmail.com Wed Mar 21 16:40:11 2018 From: pypy.commits at gmail.com (mattip) Date: Wed, 21 Mar 2018 13:40:11 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix Message-ID: <5ab2c32b.0eaddf0a.850f5.3904@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94058:b13c7b3ad6ee Date: 2018-03-21 22:39 +0200 http://bitbucket.org/pypy/pypy/changeset/b13c7b3ad6ee/ Log: fix diff --git a/lib-python/3/distutils/msvc9compiler.py b/lib-python/3/distutils/msvc9compiler.py --- a/lib-python/3/distutils/msvc9compiler.py +++ b/lib-python/3/distutils/msvc9compiler.py @@ -278,7 +278,7 @@ stdout = stdout.decode("mbcs") log.debug('-'*30) log.debug(stderr.decode('mbcs')) - log.debug(stdout.decode('mbcs')) + log.debug(stdout) log.debug('-'*30) for line in stdout.split("\n"): line = Reg.convert_mbcs(line) From pypy.commits at gmail.com Wed Mar 21 18:09:06 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 21 Mar 2018 15:09:06 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Move app-level AST rewriting to some unimportable location and remove more imports Message-ID: <5ab2d802.89c4df0a.d6351.1a2b@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94059:47c8f77a9d6f Date: 2018-03-21 23:08 +0100 http://bitbucket.org/pypy/pypy/changeset/47c8f77a9d6f/ Log: Move app-level AST rewriting to some unimportable location and remove more imports diff --git a/pypy/tool/pytest/app_rewrite.py b/pypy/tool/pytest/app_rewrite.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/app_rewrite.py @@ -0,0 +1,39 @@ +import re + +ASCII_IS_DEFAULT_ENCODING = False + +cookie_re = re.compile(r"^[ \t\f]*#.*coding[:=][ \t]*[-\w.]+") +BOM_UTF8 = '\xef\xbb\xbf' + +def _prepare_source(fn): + """Read the source code for re-writing.""" + try: + stat = fn.stat() + source = fn.read("rb") + except EnvironmentError: + return None, None + if ASCII_IS_DEFAULT_ENCODING: + # ASCII is the default encoding in Python 2. Without a coding + # declaration, Python 2 will complain about any bytes in the file + # outside the ASCII range. Sadly, this behavior does not extend to + # compile() or ast.parse(), which prefer to interpret the bytes as + # latin-1. (At least they properly handle explicit coding cookies.) To + # preserve this error behavior, we could force ast.parse() to use ASCII + # as the encoding by inserting a coding cookie. Unfortunately, that + # messes up line numbers. Thus, we have to check ourselves if anything + # is outside the ASCII range in the case no encoding is explicitly + # declared. For more context, see issue #269. Yay for Python 3 which + # gets this right. + end1 = source.find("\n") + end2 = source.find("\n", end1 + 1) + if (not source.startswith(BOM_UTF8) and + cookie_re.match(source[0:end1]) is None and + cookie_re.match(source[end1 + 1:end2]) is None): + try: + source.decode("ascii") + except UnicodeDecodeError: + # Let it fail in real import. + return None, None + # On Python versions which are not 2.7 and less than or equal to 3.1, the + # parser expects *nix newlines. + return stat, source diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -4,7 +4,7 @@ import pytest from pypy import pypydir import pypy.interpreter.function -import pypy.tool.pytest.rewrite +from pypy.tool.pytest import app_rewrite from pypy.interpreter.error import OperationError from pypy.tool.pytest import objspace from pypy.tool.pytest import appsupport @@ -13,19 +13,20 @@ class AppTestModule(pytest.Module): def collect(self): - _, source = pypy.tool.pytest.rewrite._prepare_source(self.fspath) + _, source = app_rewrite._prepare_source(self.fspath) space = objspace.gettestobjspace() - w_rootdir = space.newtext(os.path.dirname(pypydir)) + w_rootdir = space.newtext( + os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) w_source = space.newtext(source) w_fname = space.newtext(str(self.fspath)) w_mod = space.appexec([w_rootdir, w_source, w_fname], """(rootdir, source, fname): import sys sys.path.insert(0, rootdir) - from pypy.tool.pytest import rewrite + from ast_rewrite import rewrite_asserts, create_module - co = rewrite.rewrite_asserts(source, fname) - mod = rewrite.create_module(fname, co) + co = rewrite_asserts(source, fname) + mod = create_module(fname, co) return mod """) mod_dict = w_mod.getdict(space).unwrap(space) diff --git a/pypy/tool/pytest/rewrite.py b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py rename from pypy/tool/pytest/rewrite.py rename to pypy/tool/pytest/ast-rewriter/ast_rewrite.py --- a/pypy/tool/pytest/rewrite.py +++ b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py @@ -2,35 +2,19 @@ from __future__ import absolute_import, division, print_function import ast import itertools -import imp import marshal -import os -import re import struct import sys -import types -from . import util +from ast_util import assertrepr_compare, format_explanation as _format_explanation # pytest caches rewritten pycs in __pycache__. -if hasattr(imp, "get_tag"): - PYTEST_TAG = imp.get_tag() + "-PYTEST" -else: - if hasattr(sys, "pypy_version_info"): - impl = "pypy" - elif sys.platform == "java": - impl = "jython" - else: - impl = "cpython" - ver = sys.version_info - PYTEST_TAG = "%s-%s%s-PYTEST" % (impl, ver[0], ver[1]) - del ver, impl +PYTEST_TAG = sys.implementation.cache_tag + "-PYTEST" PYC_EXT = ".py" + (__debug__ and "c" or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT -ASCII_IS_DEFAULT_ENCODING = sys.version_info[0] < 3 if sys.version_info >= (3, 5): ast_Call = ast.Call @@ -65,60 +49,6 @@ return True -RN = "\r\n".encode("utf-8") -N = "\n".encode("utf-8") - -cookie_re = re.compile(r"^[ \t\f]*#.*coding[:=][ \t]*[-\w.]+") -BOM_UTF8 = '\xef\xbb\xbf' - - -def _prepare_source(fn): - """Read the source code for re-writing.""" - try: - stat = fn.stat() - source = fn.read("rb") - except EnvironmentError: - return None, None - if ASCII_IS_DEFAULT_ENCODING: - # ASCII is the default encoding in Python 2. Without a coding - # declaration, Python 2 will complain about any bytes in the file - # outside the ASCII range. Sadly, this behavior does not extend to - # compile() or ast.parse(), which prefer to interpret the bytes as - # latin-1. (At least they properly handle explicit coding cookies.) To - # preserve this error behavior, we could force ast.parse() to use ASCII - # as the encoding by inserting a coding cookie. Unfortunately, that - # messes up line numbers. Thus, we have to check ourselves if anything - # is outside the ASCII range in the case no encoding is explicitly - # declared. For more context, see issue #269. Yay for Python 3 which - # gets this right. - end1 = source.find("\n") - end2 = source.find("\n", end1 + 1) - if (not source.startswith(BOM_UTF8) and - cookie_re.match(source[0:end1]) is None and - cookie_re.match(source[end1 + 1:end2]) is None): - try: - source.decode("ascii") - except UnicodeDecodeError: - # Let it fail in real import. - return None, None - # On Python versions which are not 2.7 and less than or equal to 3.1, the - # parser expects *nix newlines. - return stat, source - - -def _rewrite_test(config, fn): - """Try to read and rewrite *fn* and return the code object.""" - state = config._assertstate - stat, source = _prepare_source(fn) - try: - co = rewrite_asserts(source, fn.strpath) - except SyntaxError: - # Let this pop up again in the real import. - state.trace("failed to parse/compile: %r" % (fn,)) - return None, None - return stat, co - - def rewrite_asserts(source, filename): """Parse the source code and rewrite asserts statements @@ -132,7 +62,7 @@ def create_module(filename, co, pyc=None): """Create a module from a code object created by rewrite_asserts()""" - mod = imp.new_module(filename) + mod = type(sys)(filename) mod.__file__ = co.co_filename if pyc is not None: mod.__cached__ = pyc @@ -143,6 +73,7 @@ def _make_rewritten_pyc(state, source_stat, pyc, co): """Try to dump rewritten code to *pyc*.""" + import os if sys.platform.startswith("win"): # Windows grants exclusive access to open files and doesn't have atomic # rename, so just write into the final file. @@ -202,9 +133,6 @@ return repr(obj).replace('\n', '\\n') -from .util import format_explanation as _format_explanation # noqa - - def _format_assertmsg(obj): """Format the custom assertion message given. @@ -233,7 +161,7 @@ done = True if done: break - custom = util.assertrepr_compare(ops[i], each_obj[i], each_obj[i + 1]) + custom = assertrepr_compare(ops[i], each_obj[i], each_obj[i + 1]) if custom is not None: return custom return expl @@ -365,7 +293,7 @@ else: builtin_name = '__builtin__' aliases = [ast.alias(builtin_name, "@py_builtins"), - ast.alias("pypy.tool.pytest.rewrite", "@pytest_ar")] + ast.alias("ast_rewrite", "@pytest_ar")] doc = getattr(mod, "docstring", None) expect_docstring = doc is None if doc is not None and self.is_rewrite_disabled(doc): diff --git a/pypy/tool/pytest/util.py b/pypy/tool/pytest/ast-rewriter/ast_util.py rename from pypy/tool/pytest/util.py rename to pypy/tool/pytest/ast-rewriter/ast_util.py From pypy.commits at gmail.com Wed Mar 21 18:50:20 2018 From: pypy.commits at gmail.com (rlamy) Date: Wed, 21 Mar 2018 15:50:20 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Remove costly collections import from ast_util Message-ID: <5ab2e1ac.01b9df0a.fdb52.6a72@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94060:86424558bdc9 Date: 2018-03-21 23:49 +0100 http://bitbucket.org/pypy/pypy/changeset/86424558bdc9/ Log: Remove costly collections import from ast_util diff --git a/pypy/tool/pytest/ast-rewriter/ast_util.py b/pypy/tool/pytest/ast-rewriter/ast_util.py --- a/pypy/tool/pytest/ast-rewriter/ast_util.py +++ b/pypy/tool/pytest/ast-rewriter/ast_util.py @@ -1,8 +1,6 @@ """Utilities for assertion debugging""" from __future__ import absolute_import, division, print_function -from collections import Sequence - u = str # The _reprcompare attribute on the util module is used by the new assertion @@ -102,7 +100,7 @@ summary = u('%s %s %s') % (left_repr, op, right_repr) def issequence(x): - return isinstance(x, Sequence) and not isinstance(x, basestring) + return hasattr(x, '__iter__') and not isinstance(x, basestring) def istext(x): return isinstance(x, basestring) From pypy.commits at gmail.com Thu Mar 22 04:08:47 2018 From: pypy.commits at gmail.com (Raemi) Date: Thu, 22 Mar 2018 01:08:47 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: (arigato, remi) cleanup test and make the bridge not contain a call_pure Message-ID: <5ab3648f.84d31c0a.edbb8.0c82@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94061:aab085ce1480 Date: 2018-03-22 09:08 +0100 http://bitbucket.org/pypy/pypy/changeset/aab085ce1480/ Log: (arigato, remi) cleanup test and make the bridge not contain a call_pure By preventing to share descrs with the GUARD_NOT_INVALIDATED coming from the elidable_compatible decorator, a later guard will not share the descr and therefore attach the bridge where it failed itself. With that, we don't start the trace at the GUARD_NOT_INVALIDATED, which is right after a guard_compatible. (Previously that meant that the bridge didn't contain all the information to remove a call_pure) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -658,6 +658,8 @@ # if opnum == rop.GUARD_COMPATIBLE: # XXX don't share that for now self._last_guard_op = None + # + # can resume data be shared with previous guard(s): if (self._last_guard_op and guard_op.getdescr() is None): self.metainterp_sd.profiler.count_ops(opnum, jitprof.Counters.OPT_GUARDS_SHARED) @@ -672,7 +674,11 @@ for farg in op.getfailargs(): if farg: self.force_box(farg) - if op.getopnum() == rop.GUARD_EXCEPTION: + + if opnum == rop.GUARD_NOT_INVALIDATED: + self._last_guard_op = None # XXX don't share the next one either + + if opnum == rop.GUARD_EXCEPTION: self._last_guard_op = None return op diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -485,8 +485,8 @@ c = Counter() c.count = 0 - @jit.elidable_compatible() - def g(cls, v): + @jit.elidable_compatible(quasi_immut_field_name_for_second_arg='version') + def g(cls, version): if we_are_translated(): c.count += 1 return cls.x @@ -497,8 +497,7 @@ driver.can_enter_jit(n=n, x=x) driver.jit_merge_point(n=n, x=x) x = jit.hint(x, promote_compatible=True) - v = x.version - res = g(x, x.version) + res = g(x) n -= res if n % 11 == 5: n -= 1 @@ -562,8 +561,8 @@ c = Counter() c.count = 0 - @jit.elidable_compatible() - def g(cls, v): + @jit.elidable_compatible(quasi_immut_field_name_for_second_arg='version') + def g(cls, version): if we_are_translated(): c.count += 1 return cls.x @@ -580,8 +579,7 @@ driver.can_enter_jit(n=n) driver.jit_merge_point(n=n) x = jit.hint(glob_b.x, promote_compatible=True) - v = x.version - res = g(x, v) + res = g(x) n -= res if n % 11 == 5: n -= 1 From pypy.commits at gmail.com Thu Mar 22 04:41:28 2018 From: pypy.commits at gmail.com (Raemi) Date: Thu, 22 Mar 2018 01:41:28 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: (arigato, remi) it seems to us that the current output of the test is correct Message-ID: <5ab36c38.04b3df0a.6d9b3.11d1@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94062:f55580d6252b Date: 2018-03-22 09:40 +0100 http://bitbucket.org/pypy/pypy/changeset/f55580d6252b/ Log: (arigato, remi) it seems to us that the current output of the test is correct and better diff --git a/rpython/jit/metainterp/test/test_compatible.py b/rpython/jit/metainterp/test/test_compatible.py --- a/rpython/jit/metainterp/test/test_compatible.py +++ b/rpython/jit/metainterp/test/test_compatible.py @@ -606,7 +606,7 @@ x = self.meta_interp(main, [False]) assert x < 70 - self.check_trace_count(5) + self.check_trace_count(6) self.check_resops(call_i=0) From pypy.commits at gmail.com Thu Mar 22 07:57:34 2018 From: pypy.commits at gmail.com (mjacob) Date: Thu, 22 Mar 2018 04:57:34 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger-updated: Fix replaying by adding back hash field to GC header. Message-ID: <5ab39a2e.83c21c0a.d0672.2409@mx.google.com> Author: Manuel Jacob Branch: reverse-debugger-updated Changeset: r94063:0365344dd2f6 Date: 2018-03-22 12:56 +0100 http://bitbucket.org/pypy/pypy/changeset/0365344dd2f6/ Log: Fix replaying by adding back hash field to GC header. diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -28,7 +28,7 @@ ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length ll_malloc_varsize = mh.ll_malloc_varsize - fields = [] + fields = [("hash", lltype.Signed)] if translator and translator.config.translation.reverse_debugger: fields.append(("uid", lltype.SignedLongLong)) hints = {'hints': {'gcheader': True}} diff --git a/rpython/translator/revdb/src-revdb/revdb.c b/rpython/translator/revdb/src-revdb/revdb.c --- a/rpython/translator/revdb/src-revdb/revdb.c +++ b/rpython/translator/revdb/src-revdb/revdb.c @@ -410,8 +410,17 @@ RPY_EXTERN Signed rpy_reverse_db_identityhash(struct pypy_header0 *obj) { - /* XXX This will make all prebuilt objects have id-hash 0. */ - return obj->h_uid; + /* Boehm only */ + if (obj->h_hash == 0) { + /* We never need to record anything: if h_hash is zero (which + is the case for all newly allocated objects), then we just + copy h_uid. This gives a stable answer. This would give + 0 for all prebuilt objects, but these should not have a + null h_hash anyway. + */ + obj->h_hash = obj->h_uid; + } + return obj->h_hash; } RPY_EXTERN From pypy.commits at gmail.com Thu Mar 22 08:04:46 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 05:04:46 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: add a passing test Message-ID: <5ab39bde.03b01c0a.125e.5456@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94064:b8745d3e1669 Date: 2018-03-22 12:23 +0100 http://bitbucket.org/pypy/pypy/changeset/b8745d3e1669/ Log: add a passing test diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -796,6 +796,34 @@ assert module.tp_init(list, x, ("hi",)) is None assert x == ["h", "i"] + def test_mp_subscript(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static PyObject* + mp_subscript(PyObject *self, PyObject *key) + { + return Py_BuildValue("i", 42); + } + PyMappingMethods tp_as_mapping; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init = ''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_mapping = &tp_as_mapping; + tp_as_mapping.mp_subscript = (binaryfunc)mp_subscript; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + assert obj[100] == 42 + def test_mp_ass_subscript(self): module = self.import_extension('foo', [ ("new_obj", "METH_NOARGS", From pypy.commits at gmail.com Thu Mar 22 08:04:48 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 05:04:48 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: move this logic inside W_PyCWrapperObject.descr_call, for consistency with the other classes, and allow the jit to look inside it Message-ID: <5ab39be0.8bc7df0a.9f089.0ed9@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94065:a0cad484bfaf Date: 2018-03-22 12:24 +0100 http://bitbucket.org/pypy/pypy/changeset/a0cad484bfaf/ Log: move this logic inside W_PyCWrapperObject.descr_call, for consistency with the other classes, and allow the jit to look inside it diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -229,6 +229,14 @@ assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type + def descr_call(self, space, w_self, __args__): + args_w, kw_w = __args__.unpack() + w_args = space.newtuple(args_w) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.newtext(key), w_obj) + return self.call(space, w_self, w_args, w_kw) + def call(self, space, w_self, w_args, w_kw): func_to_call = self.func if self.offset: @@ -261,16 +269,6 @@ (self.method_name, self.w_objclass.name)) - at jit.dont_look_inside -def cwrapper_descr_call(space, w_self, __args__): - self = space.interp_w(W_PyCWrapperObject, w_self) - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w[1:]) - w_self = args_w[0] - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - return self.call(space, w_self, w_args, w_kw) def cmethod_descr_get(space, w_function, w_obj, w_cls=None): asking_for_bound = (space.is_none(w_cls) or @@ -323,7 +321,7 @@ W_PyCWrapperObject.typedef = TypeDef( 'wrapper_descriptor', - __call__ = interp2app(cwrapper_descr_call), + __call__ = interp2app(W_PyCWrapperObject.descr_call), __get__ = interp2app(cmethod_descr_get), __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject, wrapfn="newtext_or_none"), From pypy.commits at gmail.com Thu Mar 22 08:04:50 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 05:04:50 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: WIP: modify the W_PyCWrapperObject call logic: instead of using a very generic logic which always build a tuple containing the arguments, we specify the expected arity and go through a specialized fast path Message-ID: <5ab39be2.c2b11c0a.3f6d1.2e80@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94066:ba7b2efa9585 Date: 2018-03-22 13:02 +0100 http://bitbucket.org/pypy/pypy/changeset/ba7b2efa9585/ Log: WIP: modify the W_PyCWrapperObject call logic: instead of using a very generic logic which always build a tuple containing the arguments, we specify the expected arity and go through a specialized fast path diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -213,12 +213,14 @@ (self.name, self.w_objclass.getname(self.space))) - class W_PyCWrapperObject(W_Root): - def __init__(self, space, pto, method_name, wrapper_func, + _immutable_fields_ = ["arity"] + + def __init__(self, space, pto, method_name, arity, wrapper_func, wrapper_func_kwds, doc, func, offset=None): self.space = space self.method_name = method_name + self.arity = arity self.wrapper_func = wrapper_func self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc @@ -229,18 +231,10 @@ assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type - def descr_call(self, space, w_self, __args__): - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - return self.call(space, w_self, w_args, w_kw) - - def call(self, space, w_self, w_args, w_kw): + def _get_func_to_call(self): func_to_call = self.func if self.offset: - pto = as_pyobj(space, self.w_objclass) + pto = as_pyobj(self.space, self.w_objclass) # make ptr the equivalent of this, using the offsets #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) if pto: @@ -254,6 +248,36 @@ assert False, "failed to convert w_type %s to PyObject" % str( self.w_objclass) assert func_to_call + return func_to_call + + def descr_call(self, space, w_self, __args__): + if self.arity == -1: + # slow, fallback logic: eventually, this should be killed + args_w, kw_w = __args__.unpack() + w_args = space.newtuple(args_w) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.newtext(key), w_obj) + return self.call(space, w_self, w_args, w_kw) + # + # new logic + # XXX: check for keywords + length = len(__args__.arguments_w) + if length != self.arity: + raise oefmt(space.w_TypeError, "expected %d arguments, got %d", + self.arity, length) + if self.arity == 1: + return self.call_1(space, w_self, __args__) + + assert False, 'should not arrive here' + + def call_1(self, space, w_self, __args__): + func = self._get_func_to_call() + w_o = __args__.arguments_w[0] + return self.wrapper_func(space, func, w_self, w_o) + + def call(self, space, w_self, w_args, w_kw): + func_to_call = self._get_func_to_call() if self.wrapper_func is None: assert self.wrapper_func_kwds is not None return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -84,11 +84,9 @@ check_num_args(space, w_args, 0) return generic_cpy_call(space, func_unary, w_self) -def wrap_binaryfunc(space, w_self, w_args, func): +def wrap_binaryfunc(space, func, w_self, w_x): func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_binary, w_self, args_w[0]) + return generic_cpy_call(space, func_binary, w_self, w_x) def _get_ob_type(space, w_obj): # please ensure that w_obj stays alive diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -338,7 +338,15 @@ continue if wrapper_func is None and wrapper_func_kwds is None: continue - w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, + + arity = -1 + from pypy.module.cpyext.slotdefs import wrap_binaryfunc + if wrapper_func is wrap_binaryfunc: + # XXX: this is just a quick hack, we need an official way to + # specify the arity + arity = 1 + + w_obj = W_PyCWrapperObject(space, pto, method_name, arity, wrapper_func, wrapper_func_kwds, doc, func_voidp, offset=offset) dict_w[method_name] = w_obj if pto.c_tp_doc: From pypy.commits at gmail.com Thu Mar 22 09:11:24 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 06:11:24 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: (anto, ronan): fix this test, because gcc (correctly :)) complains that the statement had no effect Message-ID: <5ab3ab7c.8aaa1c0a.8e006.d717@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94067:a3b4293a2b1b Date: 2018-03-22 13:10 +0000 http://bitbucket.org/pypy/pypy/changeset/a3b4293a2b1b/ Log: (anto, ronan): fix this test, because gcc (correctly :)) complains that the statement had no effect diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -636,7 +636,8 @@ Py_ssize_t refcnt_after; Py_INCREF(true_obj); Py_INCREF(true_obj); - PyBool_Check(true_obj); + if (!PyBool_Check(true_obj)) + Py_RETURN_NONE; refcnt_after = true_obj->ob_refcnt; Py_DECREF(true_obj); Py_DECREF(true_obj); From pypy.commits at gmail.com Thu Mar 22 09:31:30 2018 From: pypy.commits at gmail.com (mjacob) Date: Thu, 22 Mar 2018 06:31:30 -0700 (PDT) Subject: [pypy-commit] pypy reverse-debugger-updated: Change 'hash' field to be there only for revdb. Message-ID: <5ab3b032.10afdf0a.4acab.268f@mx.google.com> Author: Manuel Jacob Branch: reverse-debugger-updated Changeset: r94068:899d967fe147 Date: 2018-03-22 14:30 +0100 http://bitbucket.org/pypy/pypy/changeset/899d967fe147/ Log: Change 'hash' field to be there only for revdb. diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -28,8 +28,9 @@ ll_malloc_varsize_no_length = mh.ll_malloc_varsize_no_length ll_malloc_varsize = mh.ll_malloc_varsize - fields = [("hash", lltype.Signed)] + fields = [] if translator and translator.config.translation.reverse_debugger: + fields.append(("hash", lltype.Signed)) fields.append(("uid", lltype.SignedLongLong)) hints = {'hints': {'gcheader': True}} self.HDR = lltype.Struct("header", *fields, **hints) From pypy.commits at gmail.com Thu Mar 22 09:57:46 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 22 Mar 2018 06:57:46 -0700 (PDT) Subject: [pypy-commit] pypy vtune: updates for 2018 Message-ID: <5ab3b65a.d3badf0a.c9523.5de4@mx.google.com> Author: Armin Rigo Branch: vtune Changeset: r94069:2b28145a01f9 Date: 2018-03-22 14:56 +0100 http://bitbucket.org/pypy/pypy/changeset/2b28145a01f9/ Log: updates for 2018 diff --git a/rpython/jit/backend/x86/vtune.py b/rpython/jit/backend/x86/vtune.py --- a/rpython/jit/backend/x86/vtune.py +++ b/rpython/jit/backend/x86/vtune.py @@ -10,9 +10,10 @@ post_include_bits=[""" RPY_EXTERN void rpy_vtune_register(char *, long, long); """], - include_dirs=["/opt/intel/vtune_amplifier_xe/include"], + include_dirs=["/opt/intel/system_studio_2018/vtune_amplifier/include"], + libraries=["dl"], # otherwise, iJIT_IsProfilingActive() just returns 0 separate_module_sources=[""" -#include "/opt/intel/vtune_amplifier_xe/sdk/src/ittnotify/jitprofiling.c" +#include "/opt/intel/system_studio_2018/vtune_amplifier/sdk/src/ittnotify/jitprofiling.c" RPY_EXTERN void rpy_vtune_register(char *funcname, Signed addr, Signed size) { From pypy.commits at gmail.com Thu Mar 22 10:04:41 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 07:04:41 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: WIP, refactor the previous commit: since wrap_binaryfunc has a different signature, we can no longer mix it with the others, else the annotator (correctly) complains. Solve it by writing a specialized version of the class: eventually, we will have only specialized classes, and kill the generic one Message-ID: <5ab3b7f9.a6a0df0a.9ccbb.b3f4@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94070:5ef7b9a78fc4 Date: 2018-03-22 15:04 +0100 http://bitbucket.org/pypy/pypy/changeset/5ef7b9a78fc4/ Log: WIP, refactor the previous commit: since wrap_binaryfunc has a different signature, we can no longer mix it with the others, else the annotator (correctly) complains. Solve it by writing a specialized version of the class: eventually, we will have only specialized classes, and kill the generic one diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -214,15 +214,10 @@ class W_PyCWrapperObject(W_Root): - _immutable_fields_ = ["arity"] - - def __init__(self, space, pto, method_name, arity, wrapper_func, - wrapper_func_kwds, doc, func, offset=None): + + def __init__(self, space, pto, method_name, doc, func, offset): self.space = space self.method_name = method_name - self.arity = arity - self.wrapper_func = wrapper_func - self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func self.offset = offset @@ -231,7 +226,10 @@ assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type - def _get_func_to_call(self): + def descr_call(self, space, w_self, __args__): + return self.call(space, w_self, __args__) + + def get_func_to_call(self): func_to_call = self.func if self.offset: pto = as_pyobj(self.space, self.w_objclass) @@ -250,34 +248,51 @@ assert func_to_call return func_to_call - def descr_call(self, space, w_self, __args__): - if self.arity == -1: - # slow, fallback logic: eventually, this should be killed - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - return self.call(space, w_self, w_args, w_kw) - # - # new logic + def check_args(self, __args__, arity): # XXX: check for keywords length = len(__args__.arguments_w) - if length != self.arity: + if length != arity: raise oefmt(space.w_TypeError, "expected %d arguments, got %d", - self.arity, length) - if self.arity == 1: - return self.call_1(space, w_self, __args__) + arity, length) - assert False, 'should not arrive here' + def descr_method_repr(self): + return self.space.newtext("" % + (self.method_name, + self.w_objclass.name)) - def call_1(self, space, w_self, __args__): - func = self._get_func_to_call() + +class W_PyCWrapperObjectBinary(W_PyCWrapperObject): + + def __init__(self, space, pto, method_name, wrapper_func, doc, func, offset): + W_PyCWrapperObject.__init__(self, space, pto, method_name, doc, func, offset) + self.wrap_binaryfunc = wrapper_func + + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() w_o = __args__.arguments_w[0] - return self.wrapper_func(space, func, w_self, w_o) + return self.wrap_binaryfunc(space, func, w_self, w_o) - def call(self, space, w_self, w_args, w_kw): - func_to_call = self._get_func_to_call() + +class W_PyCWrapperObjectGeneric(W_PyCWrapperObject): + """ + slow generic implementation, it should die eventually + """ + + def __init__(self, space, pto, method_name, wrapper_func, + wrapper_func_kwds, doc, func, offset=None): + W_PyCWrapperObject.__init__(self, space, pto, method_name, doc, func, offset) + self.wrapper_func = wrapper_func + self.wrapper_func_kwds = wrapper_func_kwds + + def call(self, space, w_self, __args__): + args_w, kw_w = __args__.unpack() + w_args = space.newtuple(args_w) + w_kw = space.newdict() + for key, w_obj in kw_w.items(): + space.setitem(w_kw, space.newtext(key), w_obj) + # + func_to_call = self.get_func_to_call() if self.wrapper_func is None: assert self.wrapper_func_kwds is not None return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, @@ -288,10 +303,6 @@ self.method_name) return self.wrapper_func(space, w_self, w_args, func_to_call) - def descr_method_repr(self): - return self.space.newtext("" % - (self.method_name, - self.w_objclass.name)) def cmethod_descr_get(space, w_function, w_obj, w_cls=None): diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -25,8 +25,9 @@ Py_TPPYPYFLAGS_FLOAT_SUBCLASS, ) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, - W_PyCMethodObject, W_PyCFunctionObject) + PyCFunction_NewEx, PyCFunction, PyMethodDef, + W_PyCMethodObject, W_PyCFunctionObject, + W_PyCWrapperObjectGeneric, W_PyCWrapperObjectBinary) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, get_typedescr, make_typedescr, @@ -339,15 +340,16 @@ if wrapper_func is None and wrapper_func_kwds is None: continue - arity = -1 from pypy.module.cpyext.slotdefs import wrap_binaryfunc if wrapper_func is wrap_binaryfunc: # XXX: this is just a quick hack, we need an official way to - # specify the arity - arity = 1 - - w_obj = W_PyCWrapperObject(space, pto, method_name, arity, wrapper_func, - wrapper_func_kwds, doc, func_voidp, offset=offset) + # specify specialization + w_obj = W_PyCWrapperObjectBinary(space, pto, method_name, wrap_binaryfunc, + doc, func_voidp, offset=offset) + else: + w_obj = W_PyCWrapperObjectGeneric(space, pto, method_name, wrapper_func, + wrapper_func_kwds, doc, + func_voidp, offset=offset) dict_w[method_name] = w_obj if pto.c_tp_doc: dict_w['__doc__'] = space.newtext( From pypy.commits at gmail.com Thu Mar 22 10:18:16 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 22 Mar 2018 07:18:16 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Collect apptest_*.py files also when untranslated Message-ID: <5ab3bb28.4bb6df0a.dd448.7366@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94071:e703f1b77506 Date: 2018-03-22 12:56 +0100 http://bitbucket.org/pypy/pypy/changeset/e703f1b77506/ Log: Collect apptest_*.py files also when untranslated diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -44,8 +44,7 @@ def py3k_skip(message): py.test.skip('[py3k] %s' % message) py.test.py3k_skip = py3k_skip - if config.getoption('runappdirect'): - config.addinivalue_line('python_files', APPLEVEL_FN) + config.addinivalue_line('python_files', APPLEVEL_FN) def pytest_addoption(parser): group = parser.getgroup("pypy options") From pypy.commits at gmail.com Thu Mar 22 10:18:18 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 22 Mar 2018 07:18:18 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Add option to enable app-level assert rewriting; turn it off by default Message-ID: <5ab3bb2a.08c41c0a.55881.0427@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94072:aaa43023ad0d Date: 2018-03-22 15:16 +0100 http://bitbucket.org/pypy/pypy/changeset/aaa43023ad0d/ Log: Add option to enable app-level assert rewriting; turn it off by default diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -60,6 +60,9 @@ group.addoption('--raise-operr', action="store_true", default=False, dest="raise_operr", help="Show the interp-level OperationError in app-level tests") + group.addoption('--applevel-rewrite', action="store_true", + default=False, dest="applevel_rewrite", + help="Use assert rewriting in app-level test files (slow)") @pytest.fixture(scope='function') def space(request): @@ -96,7 +99,8 @@ if not parent.config.getoption('runappdirect'): if path.fnmatch(APPLEVEL_FN): from pypy.tool.pytest.apptest2 import AppTestModule - return AppTestModule(path, parent) + rewrite = parent.config.getoption('applevel_rewrite') + return AppTestModule(path, parent, rewrite_asserts=rewrite) else: return PyPyModule(path, parent) diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -11,6 +11,9 @@ class AppTestModule(pytest.Module): + def __init__(self, path, parent, rewrite_asserts=False): + super(AppTestModule, self).__init__(path, parent) + self.rewrite_asserts = rewrite_asserts def collect(self): _, source = app_rewrite._prepare_source(self.fspath) @@ -19,16 +22,26 @@ os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) w_source = space.newtext(source) w_fname = space.newtext(str(self.fspath)) - w_mod = space.appexec([w_rootdir, w_source, w_fname], - """(rootdir, source, fname): - import sys - sys.path.insert(0, rootdir) - from ast_rewrite import rewrite_asserts, create_module + if self.rewrite_asserts: + w_mod = space.appexec([w_rootdir, w_source, w_fname], + """(rootdir, source, fname): + import sys + sys.path.insert(0, rootdir) + from ast_rewrite import rewrite_asserts, create_module - co = rewrite_asserts(source, fname) - mod = create_module(fname, co) - return mod - """) + co = rewrite_asserts(source, fname) + mod = create_module(fname, co) + return mod + """) + else: + w_mod = space.appexec([w_rootdir, w_source, w_fname], + """(rootdir, source, fname): + import sys + sys.path.insert(0, rootdir) + from ast_rewrite import create_module + co = compile(source, fname, 'exec') + return create_module(fname, co) + """) mod_dict = w_mod.getdict(space).unwrap(space) items = [] for name, w_obj in mod_dict.items(): From pypy.commits at gmail.com Thu Mar 22 10:53:17 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 22 Mar 2018 07:53:17 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: hg merge vtune Message-ID: <5ab3c35d.04421c0a.ec86.a128@mx.google.com> Author: Armin Rigo Branch: guard-compatible Changeset: r94073:c4f28bd30a85 Date: 2018-03-22 15:14 +0100 http://bitbucket.org/pypy/pypy/changeset/c4f28bd30a85/ Log: hg merge vtune diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -177,7 +177,8 @@ self.pop_gcmap(mc) # cancel the push_gcmap(store=True) in the caller self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) mc.RET() - self._frame_realloc_slowpath = mc.materialize(self.cpu, []) + self._frame_realloc_slowpath = self.materialize(mc, [], + "frame_realloc") def _build_cond_call_slowpath(self, supports_floats, callee_only): """ This builds a general call slowpath, for whatever call happens to @@ -215,7 +216,7 @@ self._pop_all_regs_from_frame(mc, [eax], supports_floats, callee_only) mc.RET() self.flush_pending_slowpaths(mc) - return mc.materialize(self.cpu, []) + return self.materialize(mc, [], "cond_call") def _build_malloc_slowpath(self, kind): """ While arriving on slowpath, we have a gcpattern on stack 0. @@ -305,7 +306,7 @@ mc.JMP(imm(self.propagate_exception_path)) self.flush_pending_slowpaths(mc) # - rawstart = mc.materialize(self.cpu, []) + rawstart = self.materialize(mc, [], "malloc") return rawstart def _build_propagate_exception_path(self): @@ -323,7 +324,7 @@ self.mc.MOV(RawEbpLoc(ofs), imm(propagate_exception_descr)) # self._call_footer() - rawstart = self.mc.materialize(self.cpu, []) + rawstart = self.materialize(self.mc, [], "propagate_exception") self.propagate_exception_path = rawstart self.mc = None @@ -368,7 +369,7 @@ mc.ADD_ri(esp.value, WORD) mc.JMP(imm(self.propagate_exception_path)) # - rawstart = mc.materialize(self.cpu, []) + rawstart = self.materialize(mc, [], "stack_check") self.stack_check_slowpath = rawstart def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): @@ -469,7 +470,7 @@ mc.LEA_rs(esp.value, 7 * WORD) mc.RET() - rawstart = mc.materialize(self.cpu, []) + rawstart = self.materialize(mc, [], "write_barrier") if for_frame: self.wb_slowpath[4] = rawstart else: @@ -567,6 +568,7 @@ ops_offset=ops_offset) self.fixup_target_tokens(rawstart) + self.materialize_done(rawstart, full_size, "loop%d" % looptoken.number) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -644,6 +646,8 @@ self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) + self.materialize_done(rawstart, fullsize, + "loop%d" % original_loop_token.number) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -710,11 +714,14 @@ self.mc.JMP_l(0) self.mc.writeimm32(0) self.mc.force_frame_size(DEFAULT_FRAME_BYTES) + fullsize = self.mc.get_relative_pos() rawstart = self.materialize_loop(looptoken) # update the jump (above) to the real trace self._patch_jump_to(rawstart + offset, asminfo.rawstart) # update the guard to jump right to this custom piece of assembler self.patch_jump_for_descr(faildescr, rawstart) + self.materialize_done(rawstart, fullsize, + "loop%d" % looptoken.number) def _patch_jump_to(self, adr_jump_offset, adr_new_target): assert adr_jump_offset != 0 @@ -892,13 +899,30 @@ mc.writeimm32(allocated_depth) mc.copy_to_raw_memory(adr) + def get_asmmemmgr_blocks(self, looptoken): + clt = looptoken.compiled_loop_token + if clt.asmmemmgr_blocks is None: + clt.asmmemmgr_blocks = [] + return clt.asmmemmgr_blocks + + def materialize_done(self, rawstart, size, funcname): + from rpython.jit.backend.x86.vtune import rpy_vtune_register + with rffi.scoped_str2charp("rpyjit." + funcname) as p: + rpy_vtune_register(p, rawstart, size) + + def materialize(self, mc, allblocks, funcname, gcrootmap=None): + size = mc.get_relative_pos() + rawstart = mc.materialize(self.cpu, allblocks, gcrootmap=gcrootmap) + self.materialize_done(rawstart, size, funcname) + return rawstart + def materialize_loop(self, looptoken): self.datablockwrapper.done() # finish using cpu.asmmemmgr self.datablockwrapper = None allblocks = self.get_asmmemmgr_blocks(looptoken) size = self.mc.get_relative_pos() - res = self.mc.materialize(self.cpu, allblocks, - self.cpu.gc_ll_descr.gcrootmap) + res = self.materialize(self.mc, allblocks, + gcrootmap=self.cpu.gc_ll_descr.gcrootmap) if self.cpu.HAS_CODEMAP: self.cpu.codemap.register_codemap( self.codemap_builder.get_final_bytecode(res, size)) @@ -2072,7 +2096,7 @@ # now we return from the complete frame, which starts from # _call_header_with_stack_check(). The _call_footer below does it. self._call_footer() - rawstart = mc.materialize(self.cpu, []) + rawstart = self.materialize(mc, [], "failure_recovery") self.failure_recovery_code[exc + 2 * withfloats] = rawstart self.mc = None diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py --- a/rpython/jit/backend/x86/guard_compat.py +++ b/rpython/jit/backend/x86/guard_compat.py @@ -288,7 +288,7 @@ mc.JMP_s(0) assembler.flush_pending_slowpaths(mc) - assembler.guard_compat_search_tree = mc.materialize(assembler.cpu, []) + assembler.guard_compat_search_tree = assembler.materialize(mc, []) def build_once_guard_compat_recovery(assembler): @@ -303,7 +303,7 @@ target = assembler.get_target_for_failure_recovery_of_guard_compat() mc.JMP(regloc.imm(target)) - assembler.guard_compat_recovery = mc.materialize(assembler.cpu, []) + assembler.guard_compat_recovery = assembler.materialize(mc, []) def generate_recovery_stub(assembler, guard_token): diff --git a/rpython/jit/backend/x86/vtune.py b/rpython/jit/backend/x86/vtune.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/vtune.py @@ -0,0 +1,43 @@ +""" +Support for VTune Amplifier +""" + +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +eci = ExternalCompilationInfo( + post_include_bits=[""" +RPY_EXTERN void rpy_vtune_register(char *, long, long); +"""], + include_dirs=["/opt/intel/system_studio_2018/vtune_amplifier/include"], + libraries=["dl"], # otherwise, iJIT_IsProfilingActive() just returns 0 + separate_module_sources=[""" +#include "/opt/intel/system_studio_2018/vtune_amplifier/sdk/src/ittnotify/jitprofiling.c" + +RPY_EXTERN void rpy_vtune_register(char *funcname, Signed addr, Signed size) +{ + iJIT_Method_Load_V2 jmethod = {0}; + + if (iJIT_IsProfilingActive() != iJIT_SAMPLING_ON) { + return; + } + + jmethod.method_id = iJIT_GetNewMethodID(); + jmethod.method_name = funcname; + jmethod.method_load_address = (void *)addr; + jmethod.method_size = size; + jmethod.module_name = "rpyjit"; + + iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2, + (void*)&jmethod); +} +"""]) + +rpy_vtune_register = rffi.llexternal( + "rpy_vtune_register", + [rffi.CCHARP, lltype.Signed, lltype.Signed], + lltype.Void, + compilation_info=eci, + _nowrapper=True, + sandboxsafe=True) From pypy.commits at gmail.com Thu Mar 22 10:53:19 2018 From: pypy.commits at gmail.com (arigo) Date: Thu, 22 Mar 2018 07:53:19 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: fixes Message-ID: <5ab3c35f.06321c0a.dc3d5.7e1e@mx.google.com> Author: Armin Rigo Branch: guard-compatible Changeset: r94074:40ed7f642faf Date: 2018-03-22 15:29 +0100 http://bitbucket.org/pypy/pypy/changeset/40ed7f642faf/ Log: fixes diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -917,11 +917,12 @@ return rawstart def materialize_loop(self, looptoken): + """Must be followed by materialize_done() later""" self.datablockwrapper.done() # finish using cpu.asmmemmgr self.datablockwrapper = None allblocks = self.get_asmmemmgr_blocks(looptoken) size = self.mc.get_relative_pos() - res = self.materialize(self.mc, allblocks, + res = self.mc.materialize(self.cpu, allblocks, gcrootmap=self.cpu.gc_ll_descr.gcrootmap) if self.cpu.HAS_CODEMAP: self.cpu.codemap.register_codemap( diff --git a/rpython/jit/backend/x86/guard_compat.py b/rpython/jit/backend/x86/guard_compat.py --- a/rpython/jit/backend/x86/guard_compat.py +++ b/rpython/jit/backend/x86/guard_compat.py @@ -288,7 +288,8 @@ mc.JMP_s(0) assembler.flush_pending_slowpaths(mc) - assembler.guard_compat_search_tree = assembler.materialize(mc, []) + assembler.guard_compat_search_tree = assembler.materialize(mc, [], + "guard_compat_search_tree") def build_once_guard_compat_recovery(assembler): @@ -303,7 +304,8 @@ target = assembler.get_target_for_failure_recovery_of_guard_compat() mc.JMP(regloc.imm(target)) - assembler.guard_compat_recovery = assembler.materialize(mc, []) + assembler.guard_compat_recovery = assembler.materialize(mc, [], + "guard_compat_recovery") def generate_recovery_stub(assembler, guard_token): From pypy.commits at gmail.com Thu Mar 22 11:00:45 2018 From: pypy.commits at gmail.com (Raemi) Date: Thu, 22 Mar 2018 08:00:45 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: add a fairer targetcompatible Message-ID: <5ab3c51d.85c9df0a.9315d.6ee2@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94075:26fa11e33f71 Date: 2018-03-22 15:59 +0100 http://bitbucket.org/pypy/pypy/changeset/26fa11e33f71/ Log: add a fairer targetcompatible diff --git a/rpython/jit/tl/targetcompatible.py b/rpython/jit/tl/targetcompatible.py --- a/rpython/jit/tl/targetcompatible.py +++ b/rpython/jit/tl/targetcompatible.py @@ -91,6 +91,8 @@ elif select == 2: run3(n, objects, results) + print "run..." + start = time.clock() if select == 0: run1(n) diff --git a/rpython/jit/tl/targetcompatible2.py b/rpython/jit/tl/targetcompatible2.py new file mode 100644 --- /dev/null +++ b/rpython/jit/tl/targetcompatible2.py @@ -0,0 +1,118 @@ +import time +import py +py.path.local(__file__) +from rpython.jit.backend.hlinfo import highleveljitinfo + +from rpython.rlib.jit import JitDriver, elidable_compatible, elidable, promote + + +""" +Run as +$ ./targetcompatible-c $guard 100000000 $objs $res + +where $objs is the number of object instances that are created and passed to +the pure function, and $res is the maximum number of different results (one per +object). $guard can be 1 for testing guard_compatible and 2 for guard_value. """ + + +driver1 = JitDriver(greens=[], reds=['n']) +def run1(n): + while n > 0: + driver1.can_enter_jit(n=n) + driver1.jit_merge_point(n=n) + n -= 1 + + +# "representative case with guard_compatible": +driver2 = JitDriver(greens=[], reds=['n', 's', 'xs'], is_recursive=True) + +class A: + def __init__(self, i): + self.i = i + + at elidable_compatible() +def ec_f2(x): + return x.i + +def run2(n, xs): + s = 0 + while n > 0: + driver2.can_enter_jit(n=n, s=s, xs=xs) + driver2.jit_merge_point(n=n, s=s, xs=xs) + x = xs[n % len(xs)] + s += ec_f2(x) + n -= 1 + return s + + + +# "representative case with guard_value": +driver3 = JitDriver(greens=[], reds=['n', 's', 'xs'], is_recursive=True) + +class A: + def __init__(self, i): + self.i = i + + at elidable +def ec_f3(x): + return x.i + +def run3(n, xs): + s = 0 + while n > 0: + driver3.can_enter_jit(n=n, s=s, xs=xs) + driver3.jit_merge_point(n=n, s=s, xs=xs) + x = xs[n % len(xs)] + promote(x) + s += ec_f3(x) + n -= 1 + return s + + +def entry_point(args): + # store args[0] in a place where the JIT log can find it (used by + # viewcode.py to know the executable whose symbols it should display) + exe = args[0] + args = args[1:] + highleveljitinfo.sys_executable = exe + + select = int(args[0]) + n = int(args[1]) + objects = int(args[2]) + results = int(args[3]) + + xs = [A(i % results) for i in range(objects)] + + print "warming up..." + if select == 0: + run1(n) + elif select == 1: + run2(n, xs) + elif select == 2: + run3(n, xs) + + print "run..." + + start = time.clock() + if select == 0: + run1(n) + elif select == 1: + run2(n, xs) + elif select == 2: + run3(n, xs) + stop = time.clock() + print 'Warmup jitted: (%f seconds)' % (stop - start) + + + return 0 + + + +def target(driver, args): + return entry_point + +# ____________________________________________________________ + +if __name__ == '__main__': + import sys + sys.exit(entry_point(sys.argv)) From pypy.commits at gmail.com Thu Mar 22 11:04:39 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 22 Mar 2018 08:04:39 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Reimplement create_module at interp-level for the rewrite_asserts=False case Message-ID: <5ab3c607.d1b81c0a.380b1.e462@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94076:90e138c50fe2 Date: 2018-03-22 16:03 +0100 http://bitbucket.org/pypy/pypy/changeset/90e138c50fe2/ Log: Reimplement create_module at interp-level for the rewrite_asserts=False case diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -6,6 +6,7 @@ import pypy.interpreter.function from pypy.tool.pytest import app_rewrite from pypy.interpreter.error import OperationError +from pypy.interpreter.module import Module from pypy.tool.pytest import objspace from pypy.tool.pytest import appsupport @@ -21,7 +22,8 @@ w_rootdir = space.newtext( os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) w_source = space.newtext(source) - w_fname = space.newtext(str(self.fspath)) + fname = str(self.fspath) + w_fname = space.newtext(fname) if self.rewrite_asserts: w_mod = space.appexec([w_rootdir, w_source, w_fname], """(rootdir, source, fname): @@ -34,14 +36,7 @@ return mod """) else: - w_mod = space.appexec([w_rootdir, w_source, w_fname], - """(rootdir, source, fname): - import sys - sys.path.insert(0, rootdir) - from ast_rewrite import create_module - co = compile(source, fname, 'exec') - return create_module(fname, co) - """) + w_mod = create_module(space, w_fname, fname, source) mod_dict = w_mod.getdict(space).unwrap(space) items = [] for name, w_obj in mod_dict.items(): @@ -55,6 +50,13 @@ def setup(self): pass +def create_module(space, w_name, filename, source): + w_mod = Module(space, w_name) + w_dict = w_mod.getdict(space) + space.setitem(w_dict, space.newtext('__file__'), space.newtext(filename)) + space.exec_(source, w_dict, w_dict, filename=filename) + return w_mod + class AppError(Exception): From pypy.commits at gmail.com Thu Mar 22 11:27:36 2018 From: pypy.commits at gmail.com (Raemi) Date: Thu, 22 Mar 2018 08:27:36 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: make vtune integration work on my system Message-ID: <5ab3cb68.3388df0a.1e0e5.f024@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94077:fb4fbff6cb90 Date: 2018-03-22 16:26 +0100 http://bitbucket.org/pypy/pypy/changeset/fb4fbff6cb90/ Log: make vtune integration work on my system diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -39,6 +39,7 @@ from rpython.jit.codewriter import longlong from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import compute_unique_id +from rpython.jit.backend.x86 import vtune class Assembler386(BaseAssembler, VectorAssemblerMixin): @@ -906,9 +907,7 @@ return clt.asmmemmgr_blocks def materialize_done(self, rawstart, size, funcname): - from rpython.jit.backend.x86.vtune import rpy_vtune_register - with rffi.scoped_str2charp("rpyjit." + funcname) as p: - rpy_vtune_register(p, rawstart, size) + vtune.register_vtune_symbol("rpyjit." + funcname, rawstart, size) def materialize(self, mc, allblocks, funcname, gcrootmap=None): size = mc.get_relative_pos() diff --git a/rpython/jit/backend/x86/vtune.py b/rpython/jit/backend/x86/vtune.py --- a/rpython/jit/backend/x86/vtune.py +++ b/rpython/jit/backend/x86/vtune.py @@ -1,7 +1,7 @@ """ Support for VTune Amplifier """ - +from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -10,19 +10,31 @@ post_include_bits=[""" RPY_EXTERN void rpy_vtune_register(char *, long, long); """], - include_dirs=["/opt/intel/system_studio_2018/vtune_amplifier/include"], - libraries=["dl"], # otherwise, iJIT_IsProfilingActive() just returns 0 - separate_module_sources=[""" -#include "/opt/intel/system_studio_2018/vtune_amplifier/sdk/src/ittnotify/jitprofiling.c" + libraries=["dl", "jitprofiling"], + library_dirs=["/opt/intel/system_studio_2018/vtune_amplifier/lib64/"], + separate_module_sources=[r""" +#include "/opt/intel/system_studio_2018/vtune_amplifier/include/jitprofiling.h" + +RPY_EXTERN void rpy_make_dlopen_strong(char *funcname, Signed addr, Signed size) +{ + // make *really* sure that dlopen&Co are linked so that vtune is happy + dlopen(NULL, 0); + dlsym(NULL, NULL); + dlclose(NULL); +} RPY_EXTERN void rpy_vtune_register(char *funcname, Signed addr, Signed size) { iJIT_Method_Load_V2 jmethod = {0}; + fprintf(stderr, "call vtune register\n"); + if (iJIT_IsProfilingActive() != iJIT_SAMPLING_ON) { return; } + fprintf(stderr, "actually vtune register\n"); + jmethod.method_id = iJIT_GetNewMethodID(); jmethod.method_name = funcname; jmethod.method_load_address = (void *)addr; @@ -34,10 +46,25 @@ } """]) -rpy_vtune_register = rffi.llexternal( + + +try: + rffi_platform.verify_eci(eci) + + rpy_vtune_register = rffi.llexternal( "rpy_vtune_register", [rffi.CCHARP, lltype.Signed, lltype.Signed], lltype.Void, compilation_info=eci, _nowrapper=True, sandboxsafe=True) + + def register_vtune_symbol(name, start_addr, size): + with rffi.scoped_str2charp("JIT: " + name) as loopname: + rpy_vtune_register(loopname, start_addr, size) + +except rffi_platform.CompilationError as e: + print "WARNING: not using VTune integration", e + def register_vtune_symbol(name, start_addr, size): + pass + From pypy.commits at gmail.com Thu Mar 22 11:37:54 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 08:37:54 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: WIP: refactor again: there is no point in having tons of wrappers and a different class for each wrapper. It is cleaner and easier to inline the content of the wrapper directly inside each subclass Message-ID: <5ab3cdd2.04e61c0a.6d5ee.bce6@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94078:430955eada2e Date: 2018-03-22 16:36 +0100 http://bitbucket.org/pypy/pypy/changeset/430955eada2e/ Log: WIP: refactor again: there is no point in having tons of wrappers and a different class for each wrapper. It is cleaner and easier to inline the content of the wrapper directly inside each subclass diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -261,18 +261,6 @@ self.w_objclass.name)) -class W_PyCWrapperObjectBinary(W_PyCWrapperObject): - - def __init__(self, space, pto, method_name, wrapper_func, doc, func, offset): - W_PyCWrapperObject.__init__(self, space, pto, method_name, doc, func, offset) - self.wrap_binaryfunc = wrapper_func - - def call(self, space, w_self, __args__): - self.check_args(__args__, 1) - func = self.get_func_to_call() - w_o = __args__.arguments_w[0] - return self.wrap_binaryfunc(space, func, w_self, w_o) - class W_PyCWrapperObjectGeneric(W_PyCWrapperObject): """ diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -19,6 +19,7 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext import userslot from pypy.module.cpyext.buffer import CBuffer, CPyBuffer, fq +from pypy.module.cpyext.methodobject import W_PyCWrapperObject from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable @@ -84,9 +85,13 @@ check_num_args(space, w_args, 0) return generic_cpy_call(space, func_unary, w_self) -def wrap_binaryfunc(space, func, w_self, w_x): - func_binary = rffi.cast(binaryfunc, func) - return generic_cpy_call(space, func_binary, w_self, w_x) +class W_WrapBinaryFunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_x = __args__.arguments_w[0] + return generic_cpy_call(space, func_binary, w_self, w_x) def _get_ob_type(space, w_obj): # please ensure that w_obj stays alive @@ -915,7 +920,7 @@ static slotdef slotdefs[] = { SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, "x.__len__() <==> len(x)"), - SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc, + SQSLOT("__add__", sq_concat, slot_sq_concat, W_WrapBinaryFunc, "x.__add__(y) <==> x+y"), SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, "x.__mul__(n) <==> x*n"), @@ -943,14 +948,14 @@ SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, "x.__contains__(y) <==> y in x"), SQSLOT("__iadd__", sq_inplace_concat, NULL, - wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), + W_WrapBinaryFunc, "x.__iadd__(y) <==> x+=y"), SQSLOT("__imul__", sq_inplace_repeat, NULL, wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, "x.__len__() <==> len(x)"), MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, - wrap_binaryfunc, + W_WrapBinaryFunc, "x.__getitem__(y) <==> x[y]"), MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, wrap_objobjargproc, @@ -1019,35 +1024,35 @@ NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, "x[y:z] <==> x[y.__index__():z.__index__()]"), IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, - wrap_binaryfunc, "+"), + W_WrapBinaryFunc, "+"), IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract, - wrap_binaryfunc, "-"), + W_WrapBinaryFunc, "-"), IBSLOT("__imul__", nb_inplace_multiply, slot_nb_inplace_multiply, - wrap_binaryfunc, "*"), + W_WrapBinaryFunc, "*"), IBSLOT("__idiv__", nb_inplace_divide, slot_nb_inplace_divide, - wrap_binaryfunc, "/"), + W_WrapBinaryFunc, "/"), IBSLOT("__imod__", nb_inplace_remainder, slot_nb_inplace_remainder, - wrap_binaryfunc, "%"), + W_WrapBinaryFunc, "%"), IBSLOT("__ipow__", nb_inplace_power, slot_nb_inplace_power, - wrap_binaryfunc, "**"), + W_WrapBinaryFunc, "**"), IBSLOT("__ilshift__", nb_inplace_lshift, slot_nb_inplace_lshift, - wrap_binaryfunc, "<<"), + W_WrapBinaryFunc, "<<"), IBSLOT("__irshift__", nb_inplace_rshift, slot_nb_inplace_rshift, - wrap_binaryfunc, ">>"), + W_WrapBinaryFunc, ">>"), IBSLOT("__iand__", nb_inplace_and, slot_nb_inplace_and, - wrap_binaryfunc, "&"), + W_WrapBinaryFunc, "&"), IBSLOT("__ixor__", nb_inplace_xor, slot_nb_inplace_xor, - wrap_binaryfunc, "^"), + W_WrapBinaryFunc, "^"), IBSLOT("__ior__", nb_inplace_or, slot_nb_inplace_or, - wrap_binaryfunc, "|"), + W_WrapBinaryFunc, "|"), BINSLOT("__floordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), RBINSLOT("__rfloordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), IBSLOT("__ifloordiv__", nb_inplace_floor_divide, - slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), + slot_nb_inplace_floor_divide, W_WrapBinaryFunc, "//"), IBSLOT("__itruediv__", nb_inplace_true_divide, - slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), + slot_nb_inplace_true_divide, W_WrapBinaryFunc, "/"), TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, "x.__str__() <==> str(x)"), @@ -1062,7 +1067,7 @@ FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, - wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), + W_WrapBinaryFunc, "x.__getattribute__('name') <==> x.name"), TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""), TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -27,7 +27,7 @@ from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, W_PyCMethodObject, W_PyCFunctionObject, - W_PyCWrapperObjectGeneric, W_PyCWrapperObjectBinary) + W_PyCWrapperObject, W_PyCWrapperObjectGeneric) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, get_typedescr, make_typedescr, @@ -340,12 +340,14 @@ if wrapper_func is None and wrapper_func_kwds is None: continue - from pypy.module.cpyext.slotdefs import wrap_binaryfunc - if wrapper_func is wrap_binaryfunc: - # XXX: this is just a quick hack, we need an official way to - # specify specialization - w_obj = W_PyCWrapperObjectBinary(space, pto, method_name, wrap_binaryfunc, - doc, func_voidp, offset=offset) + # XXX: this is just a quick hack to distinguish the old wrappers from + # the new ones: eventually, all of them will be subclasses of + # W_PyCWrapperObject + if type(wrapper_func) is type and issubclass(wrapper_func, W_PyCWrapperObject): + # new style + wrapper_class = wrapper_func + w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, + offset=offset) else: w_obj = W_PyCWrapperObjectGeneric(space, pto, method_name, wrapper_func, wrapper_func_kwds, doc, From pypy.commits at gmail.com Thu Mar 22 11:39:19 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 08:39:19 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: test and fix typo Message-ID: <5ab3ce27.480f1c0a.77363.828e@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94079:624c476bc7cd Date: 2018-03-22 14:12 +0000 http://bitbucket.org/pypy/pypy/changeset/624c476bc7cd/ Log: test and fix typo diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -252,7 +252,7 @@ # XXX: check for keywords length = len(__args__.arguments_w) if length != arity: - raise oefmt(space.w_TypeError, "expected %d arguments, got %d", + raise oefmt(self.space.w_TypeError, "expected %d arguments, got %d", arity, length) def descr_method_repr(self): diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -823,6 +823,7 @@ ''') obj = module.new_obj() assert obj[100] == 42 + raises(TypeError, "obj.__getitem__(100, 101)") def test_mp_ass_subscript(self): module = self.import_extension('foo', [ From pypy.commits at gmail.com Thu Mar 22 11:46:54 2018 From: pypy.commits at gmail.com (Raemi) Date: Thu, 22 Mar 2018 08:46:54 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: cleanup and fix for passing non-code data to vtune Message-ID: <5ab3cfee.91bddf0a.bfc84.627f@mx.google.com> Author: Remi Meier Branch: guard-compatible Changeset: r94080:ad7291e0a4d3 Date: 2018-03-22 16:45 +0100 http://bitbucket.org/pypy/pypy/changeset/ad7291e0a4d3/ Log: cleanup and fix for passing non-code data to vtune diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -569,7 +569,8 @@ ops_offset=ops_offset) self.fixup_target_tokens(rawstart) - self.materialize_done(rawstart, full_size, "loop%d" % looptoken.number) + self.materialize_done(rawstart + functionpos, full_size - functionpos, + "loop%d" % looptoken.number) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -647,8 +648,9 @@ self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) - self.materialize_done(rawstart, fullsize, - "loop%d" % original_loop_token.number) + self.materialize_done(rawstart + startpos, fullsize - startpos, + "bridge_%d_0x%x" % (original_loop_token.number, + r_uint(descr_number))) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -722,7 +724,7 @@ # update the guard to jump right to this custom piece of assembler self.patch_jump_for_descr(faildescr, rawstart) self.materialize_done(rawstart, fullsize, - "loop%d" % looptoken.number) + "stiched_bridge%d" % looptoken.number) def _patch_jump_to(self, adr_jump_offset, adr_new_target): assert adr_jump_offset != 0 diff --git a/rpython/jit/backend/x86/vtune.py b/rpython/jit/backend/x86/vtune.py --- a/rpython/jit/backend/x86/vtune.py +++ b/rpython/jit/backend/x86/vtune.py @@ -13,6 +13,7 @@ libraries=["dl", "jitprofiling"], library_dirs=["/opt/intel/system_studio_2018/vtune_amplifier/lib64/"], separate_module_sources=[r""" +#include "dlfcn.h" #include "/opt/intel/system_studio_2018/vtune_amplifier/include/jitprofiling.h" RPY_EXTERN void rpy_make_dlopen_strong(char *funcname, Signed addr, Signed size) @@ -27,14 +28,10 @@ { iJIT_Method_Load_V2 jmethod = {0}; - fprintf(stderr, "call vtune register\n"); - if (iJIT_IsProfilingActive() != iJIT_SAMPLING_ON) { return; } - fprintf(stderr, "actually vtune register\n"); - jmethod.method_id = iJIT_GetNewMethodID(); jmethod.method_name = funcname; jmethod.method_load_address = (void *)addr; From pypy.commits at gmail.com Thu Mar 22 11:47:47 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 08:47:47 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: make sure to raise TypeError if you pass keywords to a slot which does not expect them Message-ID: <5ab3d023.818bdf0a.2fa30.608a@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94081:8b23d59cea09 Date: 2018-03-22 16:47 +0100 http://bitbucket.org/pypy/pypy/changeset/8b23d59cea09/ Log: make sure to raise TypeError if you pass keywords to a slot which does not expect them diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -214,6 +214,9 @@ class W_PyCWrapperObject(W_Root): + """ + Abstract class; for concrete subclasses, see slotdefs.py + """ def __init__(self, space, pto, method_name, doc, func, offset): self.space = space @@ -229,6 +232,9 @@ def descr_call(self, space, w_self, __args__): return self.call(space, w_self, __args__) + def call(self, w_self, __args__): + raise NotImplementedError + def get_func_to_call(self): func_to_call = self.func if self.offset: @@ -248,12 +254,15 @@ assert func_to_call return func_to_call - def check_args(self, __args__, arity): - # XXX: check for keywords + def check_args(self, __args__, arity, accept_kw=False): length = len(__args__.arguments_w) if length != arity: raise oefmt(self.space.w_TypeError, "expected %d arguments, got %d", arity, length) + if not accept_kw and __args__.keywords: + raise oefmt(self.space.w_TypeError, + "wrapper %s doesn't take any keyword arguments", + self.method_name) def descr_method_repr(self): return self.space.newtext("" % diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -824,6 +824,7 @@ obj = module.new_obj() assert obj[100] == 42 raises(TypeError, "obj.__getitem__(100, 101)") + raises(TypeError, "obj.__getitem__(100, a=42)") def test_mp_ass_subscript(self): module = self.import_extension('foo', [ From pypy.commits at gmail.com Thu Mar 22 11:51:57 2018 From: pypy.commits at gmail.com (rlamy) Date: Thu, 22 Mar 2018 08:51:57 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Implement AppTestFunction.reportinfo() to get correct failure reports Message-ID: <5ab3d11d.88d31c0a.f9170.9648@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94082:bc7db4a511cc Date: 2018-03-22 16:51 +0100 http://bitbucket.org/pypy/pypy/changeset/bc7db4a511cc/ Log: Implement AppTestFunction.reportinfo() to get correct failure reports diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -94,3 +94,8 @@ if appexcinfo.traceback: raise AppError, AppError(appexcinfo), tb raise + + def reportinfo(self): + """Must return a triple (fspath, lineno, test_name)""" + lineno = self.w_obj.code.co_firstlineno + return self.parent.fspath, lineno, self.w_obj.name From pypy.commits at gmail.com Thu Mar 22 12:00:41 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 09:00:41 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: bah Message-ID: <5ab3d329.15741c0a.298f5.20a8@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94083:8febb6bdae2a Date: 2018-03-22 16:00 +0000 http://bitbucket.org/pypy/pypy/changeset/8febb6bdae2a/ Log: bah diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -232,7 +232,7 @@ def descr_call(self, space, w_self, __args__): return self.call(space, w_self, __args__) - def call(self, w_self, __args__): + def call(self, space, w_self, __args__): raise NotImplementedError def get_func_to_call(self): From pypy.commits at gmail.com Thu Mar 22 13:03:06 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:06 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: convert wrap_objobjproc to the new style Message-ID: <5ab3e1ca.d1b81c0a.380b1.f9f2@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94084:0e0dc6fb5d4c Date: 2018-03-22 17:08 +0100 http://bitbucket.org/pypy/pypy/changeset/0e0dc6fb5d4c/ Log: convert wrap_objobjproc to the new style diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -271,15 +271,17 @@ space.fromcache(State).check_and_raise_exception(always=True) # Warning, confusing function name (like CPython). Used only for sq_contains. -def wrap_objobjproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjproc, func) - check_num_args(space, w_args, 1) - w_value, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_value) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class W_WrapObjObjProc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjproc, func) + w_value = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_value) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) def wrap_objobjargproc(space, w_self, w_args, func): func_target = rffi.cast(objobjargproc, func) @@ -945,7 +947,7 @@ "x.__delslice__(i, j) <==> del x[i:j]\n\ \n\ Use of negative indices is not supported."), - SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, + SQSLOT("__contains__", sq_contains, slot_sq_contains, W_WrapObjObjProc, "x.__contains__(y) <==> y in x"), SQSLOT("__iadd__", sq_inplace_concat, NULL, W_WrapBinaryFunc, "x.__iadd__(y) <==> x+=y"), From pypy.commits at gmail.com Thu Mar 22 13:03:10 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:10 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port wrap_call to the new style, reusing most of the logic we use also W_PyCFunction.call_keywords Message-ID: <5ab3e1ce.8e821c0a.e75df.8fba@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94086:edf304aa6652 Date: 2018-03-22 17:33 +0100 http://bitbucket.org/pypy/pypy/changeset/edf304aa6652/ Log: port wrap_call to the new style, reusing most of the logic we use also W_PyCFunction.call_keywords diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -45,6 +45,18 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) +def w_kwargs_from_args(space, __args__): + w_kwargs = None + if __args__.keywords: + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) + return w_kwargs + class W_PyCFunctionObject(W_Root): _immutable_fields_ = ["flags"] @@ -103,15 +115,7 @@ def call_keywords(self, space, w_self, __args__): func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) py_args = tuple_from_args_w(space, __args__.arguments_w) - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + w_kwargs = w_kwargs_from_args(space, __args__) try: return generic_cpy_call(space, func, w_self, py_args, w_kwargs) finally: @@ -283,6 +287,7 @@ self.wrapper_func_kwds = wrapper_func_kwds def call(self, space, w_self, __args__): + #xxx args_w, kw_w = __args__.unpack() w_args = space.newtuple(args_w) w_kw = space.newdict() diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -19,7 +19,8 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext import userslot from pypy.module.cpyext.buffer import CBuffer, CPyBuffer, fq -from pypy.module.cpyext.methodobject import W_PyCWrapperObject +from pypy.module.cpyext.methodobject import (W_PyCWrapperObject, tuple_from_args_w, + w_kwargs_from_args) from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable @@ -223,9 +224,13 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) -def wrap_call(space, w_self, w_args, func, w_kwds): - func_target = rffi.cast(ternaryfunc, func) - return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) +class wrap_call(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(ternaryfunc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) def wrap_ssizessizeobjargproc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeobjargproc, func) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -343,6 +343,9 @@ # XXX: this is just a quick hack to distinguish the old wrappers from # the new ones: eventually, all of them will be subclasses of # W_PyCWrapperObject + if type(wrapper_func_kwds) is type: + assert wrapper_func is None + wrapper_func = wrapper_func_kwds if type(wrapper_func) is type and issubclass(wrapper_func, W_PyCWrapperObject): # new style wrapper_class = wrapper_func From pypy.commits at gmail.com Thu Mar 22 13:03:08 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:08 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: rewrite wrap_unaryfunc in the new style Message-ID: <5ab3e1cc.52bf1c0a.e907c.9bb5@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94085:dc0945ef4c26 Date: 2018-03-22 17:14 +0100 http://bitbucket.org/pypy/pypy/changeset/dc0945ef4c26/ Log: rewrite wrap_unaryfunc in the new style diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -80,10 +80,12 @@ space.fromcache(State).check_and_raise_exception(always=True) return None -def wrap_unaryfunc(space, w_self, w_args, func): - func_unary = rffi.cast(unaryfunc, func) - check_num_args(space, w_args, 0) - return generic_cpy_call(space, func_unary, w_self) +class W_WrapUnaryFunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_unary = rffi.cast(unaryfunc, func) + return generic_cpy_call(space, func_unary, w_self) class W_WrapBinaryFunc(W_PyCWrapperObject): def call(self, space, w_self, __args__): @@ -994,13 +996,13 @@ "x.__pow__(y[, z]) <==> pow(x, y[, z])"), NBSLOT("__rpow__", nb_power, slot_nb_power, wrap_ternaryfunc_r, "y.__rpow__(x[, z]) <==> pow(x, y[, z])"), - UNSLOT("__neg__", nb_negative, slot_nb_negative, wrap_unaryfunc, "-x"), - UNSLOT("__pos__", nb_positive, slot_nb_positive, wrap_unaryfunc, "+x"), - UNSLOT("__abs__", nb_absolute, slot_nb_absolute, wrap_unaryfunc, + UNSLOT("__neg__", nb_negative, slot_nb_negative, W_WrapUnaryFunc, "-x"), + UNSLOT("__pos__", nb_positive, slot_nb_positive, W_WrapUnaryFunc, "+x"), + UNSLOT("__abs__", nb_absolute, slot_nb_absolute, W_WrapUnaryFunc, "abs(x)"), UNSLOT("__nonzero__", nb_nonzero, slot_nb_nonzero, wrap_inquirypred, "x != 0"), - UNSLOT("__invert__", nb_invert, slot_nb_invert, wrap_unaryfunc, "~x"), + UNSLOT("__invert__", nb_invert, slot_nb_invert, W_WrapUnaryFunc, "~x"), BINSLOT("__lshift__", nb_lshift, slot_nb_lshift, "<<"), RBINSLOT("__rlshift__", nb_lshift, slot_nb_lshift, "<<"), BINSLOT("__rshift__", nb_rshift, slot_nb_rshift, ">>"), @@ -1013,17 +1015,17 @@ RBINSLOT("__ror__", nb_or, slot_nb_or, "|"), NBSLOT("__coerce__", nb_coerce, slot_nb_coerce, wrap_coercefunc, "x.__coerce__(y) <==> coerce(x, y)"), - UNSLOT("__int__", nb_int, slot_nb_int, wrap_unaryfunc, + UNSLOT("__int__", nb_int, slot_nb_int, W_WrapUnaryFunc, "int(x)"), - UNSLOT("__long__", nb_long, slot_nb_long, wrap_unaryfunc, + UNSLOT("__long__", nb_long, slot_nb_long, W_WrapUnaryFunc, "long(x)"), - UNSLOT("__float__", nb_float, slot_nb_float, wrap_unaryfunc, + UNSLOT("__float__", nb_float, slot_nb_float, W_WrapUnaryFunc, "float(x)"), - UNSLOT("__oct__", nb_oct, slot_nb_oct, wrap_unaryfunc, + UNSLOT("__oct__", nb_oct, slot_nb_oct, W_WrapUnaryFunc, "oct(x)"), - UNSLOT("__hex__", nb_hex, slot_nb_hex, wrap_unaryfunc, + UNSLOT("__hex__", nb_hex, slot_nb_hex, W_WrapUnaryFunc, "hex(x)"), - NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, + NBSLOT("__index__", nb_index, slot_nb_index, W_WrapUnaryFunc, "x[y:z] <==> x[y.__index__():z.__index__()]"), IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, W_WrapBinaryFunc, "+"), @@ -1056,10 +1058,10 @@ IBSLOT("__itruediv__", nb_inplace_true_divide, slot_nb_inplace_true_divide, W_WrapBinaryFunc, "/"), - TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, + TPSLOT("__str__", tp_str, slot_tp_str, W_WrapUnaryFunc, "x.__str__() <==> str(x)"), TPSLOT("__str__", tp_print, NULL, NULL, ""), - TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, + TPSLOT("__repr__", tp_repr, slot_tp_repr, W_WrapUnaryFunc, "x.__repr__() <==> repr(x)"), TPSLOT("__repr__", tp_print, NULL, NULL, ""), TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, @@ -1090,7 +1092,7 @@ "x.__gt__(y) <==> x>y"), TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, "x.__ge__(y) <==> x>=y"), - TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, + TPSLOT("__iter__", tp_iter, slot_tp_iter, W_WrapUnaryFunc, "x.__iter__() <==> iter(x)"), TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, "x.next() -> the next value, or raise StopIteration"), From pypy.commits at gmail.com Thu Mar 22 13:03:14 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:14 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port wrap_init to the new style Message-ID: <5ab3e1d2.c30c1c0a.11fdc.586e@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94088:259e93d2de92 Date: 2018-03-22 17:48 +0100 http://bitbucket.org/pypy/pypy/changeset/259e93d2de92/ Log: port wrap_init to the new style diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -79,12 +79,16 @@ # CPython code copy&pasted inside slotdefs_str, and thus we need to keep the # same names as they are used in C. -def wrap_init(space, w_self, w_args, func, w_kwargs): - func_init = rffi.cast(initproc, func) - res = generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return None +class wrap_init(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_init = rffi.cast(initproc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return None class wrap_unaryfunc(W_PyCWrapperObject): def call(self, space, w_self, __args__): @@ -859,7 +863,8 @@ self.slot_names = tuple(("c_" + slot_name).split(".")) self.slot_func = function self.wrapper_func = wrapper1 - self.wrapper_func_kwds = wrapper2 + assert wrapper2 is None + self.wrapper_func_kwds = None # eventually kill this self.doc = doc # adapted from typeobject.c @@ -880,13 +885,7 @@ function = getattr(userslot, FUNCTION or '!missing', None) assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS - if FLAGS: - wrapper1 = None - wrapper2 = wrapper - else: - wrapper1 = wrapper - wrapper2 = None - return TypeSlot(NAME, SLOT, function, wrapper1, wrapper2, DOC) + return TypeSlot(NAME, SLOT, function, wrapper, None, DOC) def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC): return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0) From pypy.commits at gmail.com Thu Mar 22 13:03:16 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:16 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: kill wrapper_func_kwds, since it was used only by wrap_init and wrap_call, which are now new-style; this simplify a bit of stuff around Message-ID: <5ab3e1d4.06c0df0a.dfd23.04cb@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94089:0e2721692722 Date: 2018-03-22 17:54 +0100 http://bitbucket.org/pypy/pypy/changeset/0e2721692722/ Log: kill wrapper_func_kwds, since it was used only by wrap_init and wrap_call, which are now new-style; this simplify a bit of stuff around diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -857,14 +857,12 @@ PyWrapperFlag_KEYWORDS = 1 class TypeSlot: - def __init__(self, method_name, slot_name, function, wrapper1, wrapper2, doc): + def __init__(self, method_name, slot_name, function, wrapper, doc): self.method_name = method_name self.slot_name = slot_name self.slot_names = tuple(("c_" + slot_name).split(".")) self.slot_func = function - self.wrapper_func = wrapper1 - assert wrapper2 is None - self.wrapper_func_kwds = None # eventually kill this + self.wrapper_class = wrapper self.doc = doc # adapted from typeobject.c @@ -885,7 +883,7 @@ function = getattr(userslot, FUNCTION or '!missing', None) assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS - return TypeSlot(NAME, SLOT, function, wrapper, None, DOC) + return TypeSlot(NAME, SLOT, function, wrapper, DOC) def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC): return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0) @@ -1174,7 +1172,7 @@ x.slot_func.api_func if x.slot_func else None) for x in slotdefs]) slotdefs_for_wrappers = unrolling_iterable( - [(x.method_name, x.slot_names, x.wrapper_func, x.wrapper_func_kwds, x.doc) + [(x.method_name, x.slot_names, x.wrapper_class, x.doc) for x in slotdefs]) if __name__ == "__main__": diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -312,7 +312,7 @@ def add_operators(space, dict_w, pto): from pypy.module.cpyext.object import PyObject_HashNotImplemented hash_not_impl = llslot(space, PyObject_HashNotImplemented) - for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: + for method_name, slot_names, wrapper_class, doc in slotdefs_for_wrappers: if method_name in dict_w: continue offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] @@ -337,21 +337,20 @@ func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue - if wrapper_func is None and wrapper_func_kwds is None: + if wrapper_class is None: continue # XXX: this is just a quick hack to distinguish the old wrappers from # the new ones: eventually, all of them will be subclasses of # W_PyCWrapperObject - if type(wrapper_func_kwds) is type: - assert wrapper_func is None - wrapper_func = wrapper_func_kwds - if type(wrapper_func) is type and issubclass(wrapper_func, W_PyCWrapperObject): + if type(wrapper_class) is type and issubclass(wrapper_class, W_PyCWrapperObject): # new style - wrapper_class = wrapper_func w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset) else: + # old style + wrapper_func = wrapper_class + wrapper_func_kwds = None w_obj = W_PyCWrapperObjectGeneric(space, pto, method_name, wrapper_func, wrapper_func_kwds, doc, func_voidp, offset=offset) From pypy.commits at gmail.com Thu Mar 22 13:03:18 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:18 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port wrap_setattr and wrap_delattr to the new style Message-ID: <5ab3e1d6.7a86df0a.fa44b.69bd@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94090:1525006c89ef Date: 2018-03-22 18:02 +0100 http://bitbucket.org/pypy/pypy/changeset/1525006c89ef/ Log: port wrap_setattr and wrap_delattr to the new style diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -180,23 +180,28 @@ args_w = space.fixedview(w_args) return generic_cpy_call(space, func_target, w_self, args_w[0]) -def wrap_setattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 2) - w_name, w_value = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_setattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_delattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 1) - w_name, = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_delattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) def wrap_descr_get(space, w_self, w_args, func): func_target = rffi.cast(descrgetfunc, func) From pypy.commits at gmail.com Thu Mar 22 13:03:12 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 10:03:12 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: go back to the previous naming convention, with a comment explaining why Message-ID: <5ab3e1d0.cb98df0a.70afc.ffe5@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94087:de9fbc68ef97 Date: 2018-03-22 17:37 +0100 http://bitbucket.org/pypy/pypy/changeset/de9fbc68ef97/ Log: go back to the previous naming convention, with a comment explaining why diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -73,6 +73,11 @@ get_llhelper = v_func.value.api_func.get_llhelper return ctx.appcall(get_llhelper, v_space) +# NOTE: the following wrap_* are subclasses of W_PyCWrapperObject, even if +# they don't follow the usual W_* naming convention for subclasses of W_Root: +# we do this because we automatically generate most of the slots from the +# CPython code copy&pasted inside slotdefs_str, and thus we need to keep the +# same names as they are used in C. def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) @@ -81,14 +86,14 @@ space.fromcache(State).check_and_raise_exception(always=True) return None -class W_WrapUnaryFunc(W_PyCWrapperObject): +class wrap_unaryfunc(W_PyCWrapperObject): def call(self, space, w_self, __args__): self.check_args(__args__, 0) func = self.get_func_to_call() func_unary = rffi.cast(unaryfunc, func) return generic_cpy_call(space, func_unary, w_self) -class W_WrapBinaryFunc(W_PyCWrapperObject): +class wrap_binaryfunc(W_PyCWrapperObject): def call(self, space, w_self, __args__): self.check_args(__args__, 1) func = self.get_func_to_call() @@ -278,7 +283,7 @@ space.fromcache(State).check_and_raise_exception(always=True) # Warning, confusing function name (like CPython). Used only for sq_contains. -class W_WrapObjObjProc(W_PyCWrapperObject): +class wrap_objobjproc(W_PyCWrapperObject): def call(self, space, w_self, __args__): self.check_args(__args__, 1) func = self.get_func_to_call() @@ -929,7 +934,7 @@ static slotdef slotdefs[] = { SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, "x.__len__() <==> len(x)"), - SQSLOT("__add__", sq_concat, slot_sq_concat, W_WrapBinaryFunc, + SQSLOT("__add__", sq_concat, slot_sq_concat, wrap_binaryfunc, "x.__add__(y) <==> x+y"), SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, "x.__mul__(n) <==> x*n"), @@ -954,17 +959,17 @@ "x.__delslice__(i, j) <==> del x[i:j]\n\ \n\ Use of negative indices is not supported."), - SQSLOT("__contains__", sq_contains, slot_sq_contains, W_WrapObjObjProc, + SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, "x.__contains__(y) <==> y in x"), SQSLOT("__iadd__", sq_inplace_concat, NULL, - W_WrapBinaryFunc, "x.__iadd__(y) <==> x+=y"), + wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), SQSLOT("__imul__", sq_inplace_repeat, NULL, wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, "x.__len__() <==> len(x)"), MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, - W_WrapBinaryFunc, + wrap_binaryfunc, "x.__getitem__(y) <==> x[y]"), MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, wrap_objobjargproc, @@ -1001,13 +1006,13 @@ "x.__pow__(y[, z]) <==> pow(x, y[, z])"), NBSLOT("__rpow__", nb_power, slot_nb_power, wrap_ternaryfunc_r, "y.__rpow__(x[, z]) <==> pow(x, y[, z])"), - UNSLOT("__neg__", nb_negative, slot_nb_negative, W_WrapUnaryFunc, "-x"), - UNSLOT("__pos__", nb_positive, slot_nb_positive, W_WrapUnaryFunc, "+x"), - UNSLOT("__abs__", nb_absolute, slot_nb_absolute, W_WrapUnaryFunc, + UNSLOT("__neg__", nb_negative, slot_nb_negative, wrap_unaryfunc, "-x"), + UNSLOT("__pos__", nb_positive, slot_nb_positive, wrap_unaryfunc, "+x"), + UNSLOT("__abs__", nb_absolute, slot_nb_absolute, wrap_unaryfunc, "abs(x)"), UNSLOT("__nonzero__", nb_nonzero, slot_nb_nonzero, wrap_inquirypred, "x != 0"), - UNSLOT("__invert__", nb_invert, slot_nb_invert, W_WrapUnaryFunc, "~x"), + UNSLOT("__invert__", nb_invert, slot_nb_invert, wrap_unaryfunc, "~x"), BINSLOT("__lshift__", nb_lshift, slot_nb_lshift, "<<"), RBINSLOT("__rlshift__", nb_lshift, slot_nb_lshift, "<<"), BINSLOT("__rshift__", nb_rshift, slot_nb_rshift, ">>"), @@ -1020,53 +1025,53 @@ RBINSLOT("__ror__", nb_or, slot_nb_or, "|"), NBSLOT("__coerce__", nb_coerce, slot_nb_coerce, wrap_coercefunc, "x.__coerce__(y) <==> coerce(x, y)"), - UNSLOT("__int__", nb_int, slot_nb_int, W_WrapUnaryFunc, + UNSLOT("__int__", nb_int, slot_nb_int, wrap_unaryfunc, "int(x)"), - UNSLOT("__long__", nb_long, slot_nb_long, W_WrapUnaryFunc, + UNSLOT("__long__", nb_long, slot_nb_long, wrap_unaryfunc, "long(x)"), - UNSLOT("__float__", nb_float, slot_nb_float, W_WrapUnaryFunc, + UNSLOT("__float__", nb_float, slot_nb_float, wrap_unaryfunc, "float(x)"), - UNSLOT("__oct__", nb_oct, slot_nb_oct, W_WrapUnaryFunc, + UNSLOT("__oct__", nb_oct, slot_nb_oct, wrap_unaryfunc, "oct(x)"), - UNSLOT("__hex__", nb_hex, slot_nb_hex, W_WrapUnaryFunc, + UNSLOT("__hex__", nb_hex, slot_nb_hex, wrap_unaryfunc, "hex(x)"), - NBSLOT("__index__", nb_index, slot_nb_index, W_WrapUnaryFunc, + NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, "x[y:z] <==> x[y.__index__():z.__index__()]"), IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, - W_WrapBinaryFunc, "+"), + wrap_binaryfunc, "+"), IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract, - W_WrapBinaryFunc, "-"), + wrap_binaryfunc, "-"), IBSLOT("__imul__", nb_inplace_multiply, slot_nb_inplace_multiply, - W_WrapBinaryFunc, "*"), + wrap_binaryfunc, "*"), IBSLOT("__idiv__", nb_inplace_divide, slot_nb_inplace_divide, - W_WrapBinaryFunc, "/"), + wrap_binaryfunc, "/"), IBSLOT("__imod__", nb_inplace_remainder, slot_nb_inplace_remainder, - W_WrapBinaryFunc, "%"), + wrap_binaryfunc, "%"), IBSLOT("__ipow__", nb_inplace_power, slot_nb_inplace_power, - W_WrapBinaryFunc, "**"), + wrap_binaryfunc, "**"), IBSLOT("__ilshift__", nb_inplace_lshift, slot_nb_inplace_lshift, - W_WrapBinaryFunc, "<<"), + wrap_binaryfunc, "<<"), IBSLOT("__irshift__", nb_inplace_rshift, slot_nb_inplace_rshift, - W_WrapBinaryFunc, ">>"), + wrap_binaryfunc, ">>"), IBSLOT("__iand__", nb_inplace_and, slot_nb_inplace_and, - W_WrapBinaryFunc, "&"), + wrap_binaryfunc, "&"), IBSLOT("__ixor__", nb_inplace_xor, slot_nb_inplace_xor, - W_WrapBinaryFunc, "^"), + wrap_binaryfunc, "^"), IBSLOT("__ior__", nb_inplace_or, slot_nb_inplace_or, - W_WrapBinaryFunc, "|"), + wrap_binaryfunc, "|"), BINSLOT("__floordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), RBINSLOT("__rfloordiv__", nb_floor_divide, slot_nb_floor_divide, "//"), BINSLOT("__truediv__", nb_true_divide, slot_nb_true_divide, "/"), RBINSLOT("__rtruediv__", nb_true_divide, slot_nb_true_divide, "/"), IBSLOT("__ifloordiv__", nb_inplace_floor_divide, - slot_nb_inplace_floor_divide, W_WrapBinaryFunc, "//"), + slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), IBSLOT("__itruediv__", nb_inplace_true_divide, - slot_nb_inplace_true_divide, W_WrapBinaryFunc, "/"), + slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), - TPSLOT("__str__", tp_str, slot_tp_str, W_WrapUnaryFunc, + TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, "x.__str__() <==> str(x)"), TPSLOT("__str__", tp_print, NULL, NULL, ""), - TPSLOT("__repr__", tp_repr, slot_tp_repr, W_WrapUnaryFunc, + TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, "x.__repr__() <==> repr(x)"), TPSLOT("__repr__", tp_print, NULL, NULL, ""), TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, @@ -1076,7 +1081,7 @@ FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, - W_WrapBinaryFunc, "x.__getattribute__('name') <==> x.name"), + wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), TPSLOT("__getattr__", tp_getattro, slot_tp_getattr, NULL, ""), TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, @@ -1097,7 +1102,7 @@ "x.__gt__(y) <==> x>y"), TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, "x.__ge__(y) <==> x>=y"), - TPSLOT("__iter__", tp_iter, slot_tp_iter, W_WrapUnaryFunc, + TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, "x.__iter__() <==> iter(x)"), TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, "x.next() -> the next value, or raise StopIteration"), From pypy.commits at gmail.com Thu Mar 22 17:54:30 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 14:54:30 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port some of the buffer-related slots, enough to pass the array tests Message-ID: <5ab42616.aab6df0a.ad693.0ccb@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94099:567c1b64d3ec Date: 2018-03-22 22:50 +0100 http://bitbucket.org/pypy/pypy/changeset/567c1b64d3ec/ Log: port some of the buffer-related slots, enough to pass the array tests diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -351,13 +351,14 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.w_None -def wrap_ssizessizeargfunc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeargfunc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - start = space.int_w(args_w[0]) - end = space.int_w(args_w[1]) - return generic_cpy_call(space, func_target, w_self, start, end) +class wrap_ssizessizeargfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeargfunc, func) + start = space.int_w(__args__.arguments_w[0]) + end = space.int_w(__args__.arguments_w[1]) + return generic_cpy_call(space, func_target, w_self, start, end) class wrap_next(W_PyCWrapperObject): def call(self, space, w_self, __args__): @@ -380,21 +381,23 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.newint(res) -def wrap_getreadbuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getreadbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) def wrap_getwritebuffer(space, w_self, w_args, func): func_target = rffi.cast(readbufferproc, func) @@ -412,45 +415,48 @@ fq.register_finalizer(view) return space.newbuffer(CBuffer(view)) -def wrap_getbuffer(space, w_self, w_args, func): - func_target = rffi.cast(getbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(Py_buffer) as pybuf: - _flags = 0 - if space.len_w(w_args) > 0: - _flags = space.int_w(space.listview(w_args)[0]) - flags = rffi.cast(rffi.INT_real,_flags) - size = generic_cpy_call(space, func_target, w_self, pybuf, flags) - if widen(size) < 0: - space.fromcache(State).check_and_raise_exception(always=True) - ptr = pybuf.c_buf - size = pybuf.c_len - ndim = widen(pybuf.c_ndim) - shape = None - if pybuf.c_shape: - shape = [pybuf.c_shape[i] for i in range(ndim)] - strides = None - if pybuf.c_strides: - strides = [pybuf.c_strides[i] for i in range(ndim)] - if pybuf.c_format: - format = rffi.charp2str(pybuf.c_format) - else: - format = 'B' - # the CPython docs mandates that you do an incref whenever you call - # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't - # leak we release the buffer: - # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer - buf = CPyBuffer(space, ptr, size, w_self, format=format, - ndim=ndim, shape=shape, strides=strides, - itemsize=pybuf.c_itemsize, - readonly=widen(pybuf.c_readonly), - needs_decref=True, - releasebufferproc = rbp) - fq.register_finalizer(buf) - return buf.wrap(space) + +class wrap_getbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(getbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(Py_buffer) as pybuf: + _flags = 0 + if len(__args__.arguments_w) > 0: + _flags = space.int_w(__args__.arguments_w[0]) + flags = rffi.cast(rffi.INT_real,_flags) + size = generic_cpy_call(space, func_target, w_self, pybuf, flags) + if widen(size) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + ptr = pybuf.c_buf + size = pybuf.c_len + ndim = widen(pybuf.c_ndim) + shape = None + if pybuf.c_shape: + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None + if pybuf.c_strides: + strides = [pybuf.c_strides[i] for i in range(ndim)] + if pybuf.c_format: + format = rffi.charp2str(pybuf.c_format) + else: + format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer + buf = CPyBuffer(space, ptr, size, w_self, format=format, + ndim=ndim, shape=shape, strides=strides, + itemsize=pybuf.c_itemsize, + readonly=widen(pybuf.c_readonly), + needs_decref=True, + releasebufferproc = rbp) + fq.register_finalizer(buf) + return buf.wrap(space) def get_richcmp_func(OP_CONST): class wrap_richcmp(W_PyCWrapperObject): From pypy.commits at gmail.com Thu Mar 22 17:54:26 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 14:54:26 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port more wrappers, starting to fix test_arraymodule.py Message-ID: <5ab42612.8d9cdf0a.28894.d9fd@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94098:2e882947bdd3 Date: 2018-03-22 22:33 +0100 http://bitbucket.org/pypy/pypy/changeset/2e882947bdd3/ Log: port more wrappers, starting to fix test_arraymodule.py diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -175,15 +175,17 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.newbool(bool(res)) -def wrap_getattr(space, w_self, w_args, func): - func_target = rffi.cast(getattrfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - name_ptr = rffi.str2charp(space.text_w(args_w[0])) - try: - return generic_cpy_call(space, func_target, w_self, name_ptr) - finally: - rffi.free_charp(name_ptr) +class wrap_getattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrfunc, func) + w_name = __args__.arguments_w[0] + name_ptr = rffi.str2charp(space.text_w(w_name)) + try: + return generic_cpy_call(space, func_target, w_self, name_ptr) + finally: + rffi.free_charp(name_ptr) def wrap_getattro(space, w_self, w_args, func): func_target = rffi.cast(getattrofunc, func) @@ -276,13 +278,15 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) -def wrap_lenfunc(space, w_self, w_args, func): - func_len = rffi.cast(lenfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_len, w_self) - if widen(res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_lenfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_len = rffi.cast(lenfunc, func) + res = generic_cpy_call(space, func_len, w_self) + if widen(res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) def wrap_sq_item(space, w_self, w_args, func): func_target = rffi.cast(ssizeargfunc, func) @@ -335,15 +339,17 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.w_None -def wrap_delitem(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 1) - w_key, = space.fixedview(w_args) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, w_key, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, w_key, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None def wrap_ssizessizeargfunc(space, w_self, w_args, func): func_target = rffi.cast(ssizessizeargfunc, func) @@ -353,14 +359,16 @@ end = space.int_w(args_w[1]) return generic_cpy_call(space, func_target, w_self, start, end) -def wrap_next(space, w_self, w_args, func): - from pypy.module.cpyext.api import generic_cpy_call_expect_null - func_target = rffi.cast(iternextfunc, func) - check_num_args(space, w_args, 0) - w_res = generic_cpy_call_expect_null(space, func_target, w_self) - if not w_res and not PyErr_Occurred(space): - raise OperationError(space.w_StopIteration, space.w_None) - return w_res +class wrap_next(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + from pypy.module.cpyext.api import generic_cpy_call_expect_null + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(iternextfunc, func) + w_res = generic_cpy_call_expect_null(space, func_target, w_self) + if not w_res and not PyErr_Occurred(space): + raise OperationError(space.w_StopIteration, space.w_None) + return w_res class wrap_hashfunc(W_PyCWrapperObject): def call(self, space, w_self, __args__): From pypy.commits at gmail.com Thu Mar 22 17:54:32 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 14:54:32 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port wrap_getattro Message-ID: <5ab42618.32c3df0a.9920d.2607@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94100:5baee26aad6e Date: 2018-03-22 22:53 +0100 http://bitbucket.org/pypy/pypy/changeset/5baee26aad6e/ Log: port wrap_getattro diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -187,11 +187,13 @@ finally: rffi.free_charp(name_ptr) -def wrap_getattro(space, w_self, w_args, func): - func_target = rffi.cast(getattrofunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_target, w_self, args_w[0]) +class wrap_getattro(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrofunc, func) + w_name = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, w_self, w_name) class wrap_setattr(W_PyCWrapperObject): def call(self, space, w_self, __args__): From pypy.commits at gmail.com Thu Mar 22 18:42:04 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 15:42:04 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: test and fix for wrap_sq_delitem Message-ID: <5ab4313c.818bdf0a.2fa30.bbe6@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94103:0db3ba6fe311 Date: 2018-03-22 23:30 +0100 http://bitbucket.org/pypy/pypy/changeset/0db3ba6fe311/ Log: test and fix for wrap_sq_delitem diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -312,15 +312,17 @@ if rffi.cast(lltype.Signed, res) == -1: space.fromcache(State).check_and_raise_exception(always=True) -def wrap_sq_delitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, index, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_sq_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, index, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) # Warning, confusing function name (like CPython). Used only for sq_contains. class wrap_objobjproc(W_PyCWrapperObject): diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -938,7 +938,11 @@ static int sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - int expected = (i == 10 && PyInt_Check(o) && PyInt_AsLong(o) == 42); + int expected; + if (o == NULL) // delitem + expected = (i == 12); + else // setitem + expected = (i == 10 && PyInt_Check(o) && PyInt_AsLong(o) == 42); if (!expected) { PyErr_SetString(PyExc_ValueError, "test failed"); return -1; @@ -960,6 +964,8 @@ obj[10] = 42 raises(ValueError, "obj[10] = 43") raises(ValueError, "obj[11] = 42") + del obj[12] + raises(ValueError, "del obj[13]") def test_tp_iter(self): module = self.import_extension('foo', [ From pypy.commits at gmail.com Thu Mar 22 18:41:59 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 15:41:59 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: add a test for sq_ass_slice and port the corresponding wrapper to the new style Message-ID: <5ab43137.06321c0a.dc3d5.e4ae@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94101:564712927aef Date: 2018-03-22 23:17 +0100 http://bitbucket.org/pypy/pypy/changeset/564712927aef/ Log: add a test for sq_ass_slice and port the corresponding wrapper to the new style diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -269,16 +269,17 @@ w_kwargs = w_kwargs_from_args(space, __args__) return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) -def wrap_ssizessizeobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeobjargproc, func) - check_num_args(space, w_args, 3) - args_w = space.fixedview(w_args) - i = space.int_w(space.index(args_w[0])) - j = space.int_w(space.index(args_w[1])) - w_y = args_w[2] - res = generic_cpy_call(space, func_target, w_self, i, j, w_y) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_ssizessizeobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 3) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeobjargproc, func) + i = space.int_w(space.index(__args__.arguments_w[0])) + j = space.int_w(space.index(__args__.arguments_w[1])) + w_y = __args__.arguments_w[2] + res = generic_cpy_call(space, func_target, w_self, i, j, w_y) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) class wrap_lenfunc(W_PyCWrapperObject): def call(self, space, w_self, __args__): @@ -290,12 +291,14 @@ space.fromcache(State).check_and_raise_exception(always=True) return space.newint(res) -def wrap_sq_item(space, w_self, w_args, func): - func_target = rffi.cast(ssizeargfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - return generic_cpy_call(space, func_target, w_self, index) +class wrap_sq_item(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeargfunc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + return generic_cpy_call(space, func_target, w_self, index) def wrap_sq_setitem(space, w_self, w_args, func): func_target = rffi.cast(ssizeobjargproc, func) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -889,6 +889,43 @@ res = "foo" in obj assert res is True + def test_sq_ass_slice(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) + { + int expected = (a == 10 && b == 20 && + PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_slice = sq_ass_slice; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10:20] = 42 + raises(ValueError, "obj[10:20] = 43") + raises(ValueError, "obj[11:20] = 42") + raises(ValueError, "obj[10:21] = 42") + def test_tp_iter(self): module = self.import_extension('foo', [ ("tp_iter", "METH_VARARGS", From pypy.commits at gmail.com Thu Mar 22 18:42:06 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 15:42:06 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: port the last slot wrapper Message-ID: <5ab4313e.138fdf0a.b2332.5e9d@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94104:f51b4cef71b0 Date: 2018-03-22 23:34 +0100 http://bitbucket.org/pypy/pypy/changeset/f51b4cef71b0/ Log: port the last slot wrapper diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -409,21 +409,23 @@ fq.register_finalizer(view) return space.newbuffer(CBuffer(view)) -def wrap_getwritebuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getwritebuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) class wrap_getbuffer(W_PyCWrapperObject): From pypy.commits at gmail.com Thu Mar 22 18:42:01 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 15:42:01 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: test for sq_ass_item and fix Message-ID: <5ab43139.54d91c0a.86815.2529@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94102:150b6a41b9f2 Date: 2018-03-22 23:24 +0100 http://bitbucket.org/pypy/pypy/changeset/150b6a41b9f2/ Log: test for sq_ass_item and fix diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -300,14 +300,17 @@ index = space.int_w(space.index(w_index)) return generic_cpy_call(space, func_target, w_self, index) -def wrap_sq_setitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_sq_setitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + index = space.int_w(space.index(w_index)) + res = generic_cpy_call(space, func_target, w_self, index, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) def wrap_sq_delitem(space, w_self, w_args, func): func_target = rffi.cast(ssizeobjargproc, func) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -926,6 +926,41 @@ raises(ValueError, "obj[11:20] = 42") raises(ValueError, "obj[10:21] = 42") + def test_sq_ass_item(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) + { + int expected = (i == 10 && PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_item = sq_ass_item; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10] = 42 + raises(ValueError, "obj[10] = 43") + raises(ValueError, "obj[11] = 42") + def test_tp_iter(self): module = self.import_extension('foo', [ ("tp_iter", "METH_VARARGS", From pypy.commits at gmail.com Thu Mar 22 18:42:08 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 15:42:08 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: finally kill all the remainings of the old wrappers :) Message-ID: <5ab43140.78c3df0a.9af44.288d@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94105:b4908ae52515 Date: 2018-03-22 23:41 +0100 http://bitbucket.org/pypy/pypy/changeset/b4908ae52515/ Log: finally kill all the remainings of the old wrappers :) diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -284,39 +284,6 @@ self.w_objclass.name)) - -class W_PyCWrapperObjectGeneric(W_PyCWrapperObject): - """ - slow generic implementation, it should die eventually - """ - - def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func, offset=None): - W_PyCWrapperObject.__init__(self, space, pto, method_name, doc, func, offset) - self.wrapper_func = wrapper_func - self.wrapper_func_kwds = wrapper_func_kwds - - def call(self, space, w_self, __args__): - #xxx - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w) - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - # - func_to_call = self.get_func_to_call() - if self.wrapper_func is None: - assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, - w_kw) - if space.is_true(w_kw): - raise oefmt(space.w_TypeError, - "wrapper %s doesn't take any keyword arguments", - self.method_name) - return self.wrapper_func(space, w_self, w_args, func_to_call) - - - def cmethod_descr_get(space, w_function, w_obj, w_cls=None): asking_for_bound = (space.is_none(w_cls) or not space.is_w(w_obj, space.w_None) or diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -40,29 +40,6 @@ Py_GT = 4 Py_GE = 5 - -def check_num_args(space, w_ob, n): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if n == space.len_w(w_ob): - return - raise oefmt(space.w_TypeError, - "expected %d arguments, got %d", - n, space.len_w(w_ob)) - -def check_num_argsv(space, w_ob, low, high): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if low <=space.len_w(w_ob) <= high: - return - raise oefmt(space.w_TypeError, - "expected %d-%d arguments, got %d", - low, high, space.len_w(w_ob)) - @not_rpython def llslot(space, func): return func.api_func.get_llhelper(space) @@ -884,9 +861,10 @@ missing_wrappers = ['wrap_indexargfunc', 'wrap_delslice', 'wrap_coercefunc'] for name in missing_wrappers: assert name not in globals() - def missing_wrapper(space, w_self, w_args, func): - print "cpyext: missing slot wrapper " + name - raise NotImplementedError("Slot wrapper " + name) + class missing_wrapper(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + print "cpyext: missing slot wrapper " + name + raise NotImplementedError("Slot wrapper " + name) missing_wrapper.__name__ = name globals()[name] = missing_wrapper diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -26,8 +26,7 @@ ) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, - W_PyCMethodObject, W_PyCFunctionObject, - W_PyCWrapperObject, W_PyCWrapperObjectGeneric) + W_PyCMethodObject, W_PyCFunctionObject, W_PyCWrapperObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, get_typedescr, make_typedescr, @@ -340,20 +339,8 @@ if wrapper_class is None: continue - # XXX: this is just a quick hack to distinguish the old wrappers from - # the new ones: eventually, all of them will be subclasses of - # W_PyCWrapperObject - if type(wrapper_class) is type and issubclass(wrapper_class, W_PyCWrapperObject): - # new style - w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, - offset=offset) - else: - # old style - wrapper_func = wrapper_class - wrapper_func_kwds = None - w_obj = W_PyCWrapperObjectGeneric(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, - func_voidp, offset=offset) + assert issubclass(wrapper_class, W_PyCWrapperObject) + w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset) dict_w[method_name] = w_obj if pto.c_tp_doc: dict_w['__doc__'] = space.newtext( From pypy.commits at gmail.com Thu Mar 22 18:49:21 2018 From: pypy.commits at gmail.com (antocuni) Date: Thu, 22 Mar 2018 15:49:21 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: try to see if this improve the performance Message-ID: <5ab432f1.46101c0a.402a7.2975@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94106:cb1e9314536a Date: 2018-03-22 23:48 +0100 http://bitbucket.org/pypy/pypy/changeset/cb1e9314536a/ Log: try to see if this improve the performance diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -221,6 +221,7 @@ """ Abstract class; for concrete subclasses, see slotdefs.py """ + _immutable_fields_ = ['offset[*]'] def __init__(self, space, pto, method_name, doc, func, offset): self.space = space From pypy.commits at gmail.com Fri Mar 23 03:12:07 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Mar 2018 00:12:07 -0700 (PDT) Subject: [pypy-commit] pypy default: Fix for the finalizer test failures after c4654fc5b220 Message-ID: <5ab4a8c7.0ea7df0a.10443.bd4f@mx.google.com> Author: Armin Rigo Branch: Changeset: r94108:8c6acf86401f Date: 2018-03-23 08:11 +0100 http://bitbucket.org/pypy/pypy/changeset/8c6acf86401f/ Log: Fix for the finalizer test failures after c4654fc5b220 diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1592,8 +1592,7 @@ index = self.get_finalizer_queue_index(hop) c_index = rmodel.inputconst(lltype.Signed, index) v_ptr = hop.spaceop.args[1] - v_ptr = hop.genop("cast_opaque_ptr", [v_ptr], - resulttype=llmemory.GCREF) + assert v_ptr.concretetype == llmemory.GCREF hop.genop("direct_call", [self.register_finalizer_ptr, self.c_const_gc, c_index, v_ptr]) diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -235,11 +235,11 @@ obj = deque.popleft() else: obj = llmemory.NULL - return llmemory.cast_adr_to_ptr(obj, rclass.OBJECTPTR) + return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) def gc_fq_register(self, fq_tag, ptr): index = self.get_finalizer_queue_index(fq_tag) - ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) + assert lltype.typeOf(ptr) == llmemory.GCREF self.gc.register_finalizer(index, ptr) # ____________________________________________________________ From pypy.commits at gmail.com Fri Mar 23 06:12:23 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 03:12:23 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: this was done by the cpyext-faster-arg-passing branch Message-ID: <5ab4d307.464a1c0a.239cd.67f6@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5881:572e9c4343cd Date: 2018-03-23 10:50 +0100 http://bitbucket.org/pypy/extradoc/changeset/572e9c4343cd/ Log: this was done by the cpyext-faster-arg-passing branch diff --git a/planning/cpyext.txt b/planning/cpyext.txt --- a/planning/cpyext.txt +++ b/planning/cpyext.txt @@ -9,14 +9,6 @@ - make_typedescr() for the type object (typeobject.py) should pass basestruct=PyHeapTypeObject?? -- right now the conversion from space.w_None to Py_None always does a dict - lookup (inside as_pyobj); investigate whether it is worth to add a special - case for it. A possible more general solution is to add a special field to - some selected W_* classes (for example W_TypeObject, W_NoneObject and an - hypotetical W_CPyObject) which contains a PyObject* field, so that doing the - w_* -> py_* lookup is just a getfield (there is alreadt support for this in - rawrefcount.create_link_pyobj) - - make sure that we run all tests with -A, too - make PyTypeObject be a pointer, like all other PyXxxObject From pypy.commits at gmail.com Fri Mar 23 06:12:25 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 03:12:25 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: this is already on default nowadays Message-ID: <5ab4d309.48aadf0a.3590f.b56b@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5882:aa9bee521afd Date: 2018-03-23 10:51 +0100 http://bitbucket.org/pypy/extradoc/changeset/aa9bee521afd/ Log: this is already on default nowadays diff --git a/planning/cpyext.txt b/planning/cpyext.txt --- a/planning/cpyext.txt +++ b/planning/cpyext.txt @@ -6,9 +6,6 @@ attached to a W_Root object. -- make_typedescr() for the type object (typeobject.py) should pass - basestruct=PyHeapTypeObject?? - - make sure that we run all tests with -A, too - make PyTypeObject be a pointer, like all other PyXxxObject From pypy.commits at gmail.com Fri Mar 23 06:13:00 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Mar 2018 03:13:00 -0700 (PDT) Subject: [pypy-commit] pypy rawrefcount-free-early: (antocuni, arigo, stephan around) Message-ID: <5ab4d32c.8fd0df0a.652b.56c1@mx.google.com> Author: Armin Rigo Branch: rawrefcount-free-early Changeset: r94109:eba667b91fd0 Date: 2018-03-23 11:12 +0100 http://bitbucket.org/pypy/pypy/changeset/eba667b91fd0/ Log: (antocuni, arigo, stephan around) Trying to call rrc_invoke_callback() after every minor collection. This should allow cpyext objects to be freed much earlier (and it was kind of the goal, and was a bug, anyway). diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -871,6 +871,8 @@ break # # + self.rrc_invoke_callback() # XXX think more and write tests + # if self.debug_tiny_nursery >= 0: # for debugging if self.nursery_top - self.nursery_free > self.debug_tiny_nursery: self.nursery_free = self.nursery_top - self.debug_tiny_nursery From pypy.commits at gmail.com Fri Mar 23 06:15:26 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 03:15:26 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: another idea which just came to our minds Message-ID: <5ab4d3be.135e1c0a.2df20.5e11@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5883:d1b1f741d1de Date: 2018-03-23 11:15 +0100 http://bitbucket.org/pypy/extradoc/changeset/d1b1f741d1de/ Log: another idea which just came to our minds diff --git a/planning/cpyext.txt b/planning/cpyext.txt --- a/planning/cpyext.txt +++ b/planning/cpyext.txt @@ -18,3 +18,7 @@ - methodobject.py: be ready for make_ref() to fail, and free the partially-contructed tuple (same logic as tuple_attach) + +- add JIT support to virtualize the pypy side placeholders of PyObject*: this + way, if a PyObject* is converted to W_Root only for the lifetime of the + loop, we can avoid the cost entirely From pypy.commits at gmail.com Fri Mar 23 06:16:48 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 03:16:48 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: fix Message-ID: <5ab4d410.135e1c0a.2df20.5e6e@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94110:0a2ba014f332 Date: 2018-03-23 10:15 +0000 http://bitbucket.org/pypy/pypy/changeset/0a2ba014f332/ Log: fix diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -208,7 +208,7 @@ w_type = __args__.arguments_w[1] else: raise oefmt(space.w_TypeError, - "expected 1 or 2 arguments, got %d", len(args_w)) + "expected 1 or 2 arguments, got %d", len(__args__.arguments_w)) if w_obj is space.w_None: w_obj = None if w_type is space.w_None: From pypy.commits at gmail.com Fri Mar 23 06:16:50 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 03:16:50 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: Merge heads Message-ID: <5ab4d412.82151c0a.4164a.cad2@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94111:d4db2db309a0 Date: 2018-03-23 10:16 +0000 http://bitbucket.org/pypy/pypy/changeset/d4db2db309a0/ Log: Merge heads diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -221,6 +221,7 @@ """ Abstract class; for concrete subclasses, see slotdefs.py """ + _immutable_fields_ = ['offset[*]'] def __init__(self, space, pto, method_name, doc, func, offset): self.space = space From pypy.commits at gmail.com Fri Mar 23 07:14:15 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 04:14:15 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-tls-operror2: Close branch cpyext-tls-operror2 Message-ID: <5ab4e187.54d91c0a.86815.99c6@mx.google.com> Author: Ronan Lamy Branch: cpyext-tls-operror2 Changeset: r94112:6080e4dcb7da Date: 2018-03-23 12:13 +0100 http://bitbucket.org/pypy/pypy/changeset/6080e4dcb7da/ Log: Close branch cpyext-tls-operror2 From pypy.commits at gmail.com Fri Mar 23 07:14:33 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 04:14:33 -0700 (PDT) Subject: [pypy-commit] pypy default: Merged cpyext-tls-operror2 into default Message-ID: <5ab4e199.0a561c0a.64984.c516@mx.google.com> Author: Ronan Lamy Branch: Changeset: r94113:37ea4d5f0500 Date: 2018-03-23 12:13 +0100 http://bitbucket.org/pypy/pypy/changeset/37ea4d5f0500/ Log: Merged cpyext-tls-operror2 into default diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -82,10 +82,10 @@ def PyTraceBack_Here(space, w_frame): from pypy.interpreter.pytraceback import record_application_traceback state = space.fromcache(State) - if state.operror is None: + if state.get_exception() is None: return -1 frame = space.interp_w(PyFrame, w_frame) - record_application_traceback(space, state.operror, frame, 0) + record_application_traceback(space, state.get_exception(), frame, 0) return 0 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -31,9 +31,10 @@ @cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) - if state.operror is None: + operror = state.get_exception() + if operror is None: return None - return state.operror.w_type # borrowed ref + return operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -2,11 +2,18 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter import executioncontext +from pypy.interpreter.executioncontext import ExecutionContext from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rdynload import DLLHANDLE from rpython.rlib import rawrefcount import sys + +# Keep track of exceptions raised in cpyext for a particular execution +# context. +ExecutionContext.cpyext_operror = None + + class State: def __init__(self, space): self.space = space @@ -18,7 +25,8 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.operror = None + ec = self.space.getexecutioncontext() + ec.cpyext_operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track @@ -37,17 +45,24 @@ def set_exception(self, operror): self.clear_exception() - self.operror = operror + ec = self.space.getexecutioncontext() + ec.cpyext_operror = operror def clear_exception(self): """Clear the current exception state, and return the operror.""" - operror = self.operror - self.operror = None + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror + ec.cpyext_operror = None return operror + def get_exception(self): + ec = self.space.getexecutioncontext() + return ec.cpyext_operror + @specialize.arg(1) def check_and_raise_exception(self, always=False): - operror = self.operror + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror if operror: self.clear_exception() raise operror diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -39,7 +39,7 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - operror = state.operror + operror = state.get_exception() if not operror: raise Exception("DID NOT RAISE") if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py --- a/pypy/module/cpyext/test/test_pyerrors.py +++ b/pypy/module/cpyext/test/test_pyerrors.py @@ -52,7 +52,8 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.operror.get_w_value(space), + operror = state.get_exception() + assert space.eq_w(operror.get_w_value(space), space.wrap("a value")) api.PyErr_Clear() @@ -60,12 +61,14 @@ def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.operror.w_type, space.w_KeyError) - assert space.eq_w(state.operror.get_w_value(space), space.w_None) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_KeyError) + assert space.eq_w(operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.operror.w_type, space.w_MemoryError) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_Warning(self, space, api, capfd): @@ -437,3 +440,59 @@ '''), ]) raises(SystemError, module.oops) + + def test_error_thread_race(self): + # Check race condition: thread 0 returns from cpyext with error set, + # after thread 1 has set an error but before it returns. + module = self.import_extension('foo', [ + ("emit_error", "METH_VARARGS", + ''' + PyThreadState *save = NULL; + PyGILState_STATE gilsave; + + /* NB. synchronization due to GIL */ + static volatile int flag = 0; + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + return NULL; + + /* Proceed in thread 1 first */ + save = PyEval_SaveThread(); + while (id == 0 && flag == 0); + gilsave = PyGILState_Ensure(); + + PyErr_Format(PyExc_ValueError, "%d", id); + + /* Proceed in thread 0 first */ + if (id == 1) flag = 1; + PyGILState_Release(gilsave); + while (id == 1 && flag == 1); + PyEval_RestoreThread(save); + + if (id == 0) flag = 0; + return NULL; + ''' + ), + ]) + + import threading + + failures = [] + + def worker(arg): + try: + module.emit_error(arg) + failures.append(True) + except Exception as exc: + if str(exc) != str(arg): + failures.append(exc) + + threads = [threading.Thread(target=worker, args=(j,)) + for j in (0, 1)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not failures From pypy.commits at gmail.com Fri Mar 23 07:32:08 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 04:32:08 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Delete converted test_coroutine.py Message-ID: <5ab4e5b8.12691c0a.704c1.95fc@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94114:03ddbc729dd1 Date: 2018-03-22 17:20 +0100 http://bitbucket.org/pypy/pypy/changeset/03ddbc729dd1/ Log: Delete converted test_coroutine.py diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py --- a/pypy/interpreter/test/apptest_coroutine.py +++ b/pypy/interpreter/test/apptest_coroutine.py @@ -202,6 +202,3 @@ except Done: pass assert result == [42] - -def test_fail(): - assert False diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py deleted file mode 100644 --- a/pypy/interpreter/test/test_coroutine.py +++ /dev/null @@ -1,205 +0,0 @@ - -class AppTestCoroutine: - - def test_cannot_iterate(self): """ - async def f(x): - pass - raises(TypeError, "for i in f(5): pass") - raises(TypeError, iter, f(5)) - raises(TypeError, next, f(5)) - """ - - def test_async_for(self): """ - class X: - def __aiter__(self): - return MyAIter() - class MyAIter: - async def __anext__(self): - return 42 - async def f(x): - sum = 0 - async for a in x: - sum += a - if sum > 100: - break - return sum - cr = f(X()) - try: - cr.send(None) - except StopIteration as e: - assert e.value == 42 * 3 - else: - assert False, "should have raised" - """ - - def test_StopAsyncIteration(self): """ - class X: - def __aiter__(self): - return MyAIter() - class MyAIter: - count = 0 - async def __anext__(self): - if self.count == 3: - raise StopAsyncIteration - self.count += 1 - return 42 - async def f(x): - sum = 0 - async for a in x: - sum += a - return sum - cr = f(X()) - try: - cr.send(None) - except StopIteration as e: - assert e.value == 42 * 3 - else: - assert False, "should have raised" - """ - - def test_async_for_old_style(self): """ - class X: - def __aiter__(self): - return MyAIter() - class MyAIter: - def __await__(self): - return iter([20, 30]) - async def f(x): - sum = 0 - async for a in x: - sum += a - if sum > 100: - break - return sum - cr = f(X()) - assert next(cr.__await__()) == 20 - """ - - def test_set_coroutine_wrapper(self): """ - import sys - async def f(): - pass - seen = [] - def my_wrapper(cr): - seen.append(cr) - return 42 - assert sys.get_coroutine_wrapper() is None - sys.set_coroutine_wrapper(my_wrapper) - assert sys.get_coroutine_wrapper() is my_wrapper - cr = f() - assert cr == 42 - sys.set_coroutine_wrapper(None) - assert sys.get_coroutine_wrapper() is None - """ - - def test_async_with(self): """ - seen = [] - class X: - async def __aenter__(self): - seen.append('aenter') - async def __aexit__(self, *args): - seen.append('aexit') - async def f(x): - async with x: - return 42 - c = f(X()) - try: - c.send(None) - except StopIteration as e: - assert e.value == 42 - else: - assert False, "should have raised" - assert seen == ['aenter', 'aexit'] - """ - - def test_await(self): """ - class X: - def __await__(self): - i1 = yield 40 - assert i1 == 82 - i2 = yield 41 - assert i2 == 93 - async def f(): - await X() - await X() - c = f() - assert c.send(None) == 40 - assert c.send(82) == 41 - assert c.send(93) == 40 - assert c.send(82) == 41 - raises(StopIteration, c.send, 93) - """ - - def test_await_error(self): """ - async def f(): - await [42] - c = f() - try: - c.send(None) - except TypeError as e: - assert str(e) == "object list can't be used in 'await' expression" - else: - assert False, "should have raised" - """ - - def test_async_with_exception_context(self): """ - class CM: - async def __aenter__(self): - pass - async def __aexit__(self, *e): - 1/0 - async def f(): - async with CM(): - raise ValueError - c = f() - try: - c.send(None) - except ZeroDivisionError as e: - assert e.__context__ is not None - assert isinstance(e.__context__, ValueError) - else: - assert False, "should have raised" - """ - - def test_runtime_warning(self): """ - import gc, warnings - async def foobaz(): - pass - with warnings.catch_warnings(record=True) as l: - foobaz() - gc.collect() - gc.collect() - gc.collect() - - assert len(l) == 1, repr(l) - w = l[0].message - assert isinstance(w, RuntimeWarning) - assert str(w).startswith("coroutine ") - assert str(w).endswith("foobaz' was never awaited") - """ - - def test_async_for_with_tuple_subclass(self): """ - class Done(Exception): pass - - class AIter(tuple): - i = 0 - def __aiter__(self): - return self - async def __anext__(self): - if self.i >= len(self): - raise StopAsyncIteration - self.i += 1 - return self[self.i - 1] - - result = [] - async def foo(): - async for i in AIter([42]): - result.append(i) - raise Done - - try: - foo().send(None) - except Done: - pass - assert result == [42] - """ From pypy.commits at gmail.com Fri Mar 23 07:32:10 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 04:32:10 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Add minimal fixture implementation for untranslated apptests Message-ID: <5ab4e5ba.138fdf0a.b2332.7819@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94115:9daca42128ef Date: 2018-03-23 12:16 +0100 http://bitbucket.org/pypy/pypy/changeset/9daca42128ef/ Log: Add minimal fixture implementation for untranslated apptests diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py --- a/pypy/tool/pytest/apptest2.py +++ b/pypy/tool/pytest/apptest2.py @@ -82,8 +82,14 @@ def execute_appex(self, space, w_func): space.getexecutioncontext().set_sys_exc_info(None) + sig = w_func.code._signature + if sig.varargname or sig.kwargname or sig.kwonlyargnames: + raise ValueError( + 'Test functions may not use *args, **kwargs or ' + 'keyword-only args') + args_w = self.get_fixtures(space, sig.argnames) try: - space.call_function(w_func) + space.call_function(w_func, *args_w) except OperationError as e: if self.config.option.raise_operr: raise @@ -99,3 +105,15 @@ """Must return a triple (fspath, lineno, test_name)""" lineno = self.w_obj.code.co_firstlineno return self.parent.fspath, lineno, self.w_obj.name + + def get_fixtures(self, space, fixtures): + if not fixtures: + return [] + import imp + fixtures_mod = imp.load_source( + 'fixtures', str(self.parent.fspath.new(basename='fixtures.py'))) + result = [] + for name in fixtures: + arg = getattr(fixtures_mod, name)(space, self.parent.config) + result.append(arg) + return result diff --git a/pypy/tool/pytest/fake_pytest/__init__.py b/pypy/tool/pytest/fake_pytest/__init__.py --- a/pypy/tool/pytest/fake_pytest/__init__.py +++ b/pypy/tool/pytest/fake_pytest/__init__.py @@ -5,6 +5,7 @@ interpleveldefs = { 'raises': 'interp_pytest.pypyraises', 'skip': 'interp_pytest.pypyskip', + 'fixture': 'interp_pytest.fake_fixture', } appleveldefs = { 'importorskip': 'app_pytest.importorskip', diff --git a/pypy/tool/pytest/fake_pytest/interp_pytest.py b/pypy/tool/pytest/fake_pytest/interp_pytest.py --- a/pypy/tool/pytest/fake_pytest/interp_pytest.py +++ b/pypy/tool/pytest/fake_pytest/interp_pytest.py @@ -1,1 +1,4 @@ from pypy.tool.pytest.appsupport import pypyraises, pypyskip + +def fake_fixture(space, w_arg): + return w_arg From pypy.commits at gmail.com Fri Mar 23 09:01:09 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 06:01:09 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Appdirect tests still need to collect directories Message-ID: <5ab4fa95.04191c0a.7513b.a30e@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94116:6af4a6061177 Date: 2018-03-23 13:58 +0100 http://bitbucket.org/pypy/pypy/changeset/6af4a6061177/ Log: Appdirect tests still need to collect directories diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -203,6 +203,7 @@ def pytest_ignore_collect(path, config): - if config.getoption('runappdirect') and not path.fnmatch(APPLEVEL_FN): + if (config.getoption('runappdirect') and + not path.isdir() and not path.fnmatch(APPLEVEL_FN)): return True return path.check(link=1) From pypy.commits at gmail.com Fri Mar 23 09:01:11 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 06:01:11 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Port test_pyframe.py to new-style apptest_pyframe.py Message-ID: <5ab4fa97.8f121c0a.4d1d.a2da@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94117:9f1ee75179ca Date: 2018-03-23 12:44 +0100 http://bitbucket.org/pypy/pypy/changeset/9f1ee75179ca/ Log: Port test_pyframe.py to new-style apptest_pyframe.py diff --git a/pypy/interpreter/test/apptest_coroutine.py b/pypy/interpreter/test/apptest_coroutine.py --- a/pypy/interpreter/test/apptest_coroutine.py +++ b/pypy/interpreter/test/apptest_coroutine.py @@ -162,7 +162,7 @@ def test_runtime_warning(): - import gc, warnings + import gc, warnings # XXX: importing warnings is expensive untranslated async def foobaz(): pass with warnings.catch_warnings(record=True) as l: diff --git a/pypy/interpreter/test/apptest_pyframe.py b/pypy/interpreter/test/apptest_pyframe.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/apptest_pyframe.py @@ -0,0 +1,767 @@ +import pytest + + at pytest.fixture +def tempfile(tmpdir): + return str(tmpdir / 'tempfile1') + +def test_f_locals(): + import sys + f = sys._getframe() + assert f.f_locals is locals() + +def test_f_globals(): + import sys + f = sys._getframe() + assert f.f_globals is globals() + raises(AttributeError, "f.f_globals = globals()") + +def test_f_builtins(): + import sys, builtins + f = sys._getframe() + assert f.f_builtins is builtins.__dict__ + +def test_f_code(): + def g(): + import sys + f = sys._getframe() + return f.f_code + assert g() is g.__code__ + +def test_f_trace_del(): + import sys + f = sys._getframe() + del f.f_trace + assert f.f_trace is None + +def test_f_lineno(): + def g(): + import sys + f = sys._getframe() + x = f.f_lineno + y = f.f_lineno + z = f.f_lineno + return [x, y, z] + origin = g.__code__.co_firstlineno + assert g() == [origin+3, origin+4, origin+5] + +def test_f_lineno_set(tempfile): + def tracer(f, *args): + def y(f, *args): + return y + def x(f, *args): + f.f_lineno += 1 + return y # "return None" should have the same effect, but see + # test_local_trace_function_returning_None_ignored + return x + + # obscure: call open beforehand, py3k's open invokes some app + # level code that confuses our tracing (likely due to the + # testing env, otherwise it's not a problem) + f = open(tempfile, 'w') + def function(f=f): + xyz + with f as f: + pass + return 3 + + import sys + sys.settrace(tracer) + function() + sys.settrace(None) + # assert did not crash + +def test_f_lineno_set_2(tempfile): + counter = [0] + errors = [] + + def tracer(f, event, *args): + if event == 'line': + counter[0] += 1 + if counter[0] == 2: + try: + f.f_lineno += 2 + except ValueError as e: + errors.append(e) + return tracer + + # obscure: call open beforehand, py3k's open invokes some app + # level code that confuses our tracing (likely due to the + # testing env, otherwise it's not a problem) + f = open(tempfile, 'w') + def function(): + try: + raise ValueError + except ValueError: + x = 42 + return x + + import sys + sys.settrace(tracer) + x = function() + sys.settrace(None) + assert x == 42 + assert len(errors) == 1 + assert str(errors[0]).startswith( + "can't jump into or out of an 'expect' or 'finally' block") + +def test_f_lineno_set_3(): + def jump_in_nested_finally(output): + try: + output.append(2) + finally: + output.append(4) + try: + output.append(6) + finally: + output.append(8) + output.append(9) + output = [] + + def tracer(f, event, *args): + if event == 'line' and len(output) == 1: + f.f_lineno += 5 + return tracer + + import sys + sys.settrace(tracer) + jump_in_nested_finally(output) + sys.settrace(None) + assert output == [2, 9] + +def test_f_lineno_set_firstline(): + seen = [] + def tracer(f, event, *args): + seen.append((event, f.f_lineno)) + if len(seen) == 5: + f.f_lineno = 1 # bug shown only when setting lineno to 1 + return tracer + + def g(): + import sys + source = "x=1\ny=x+1\nz=y+1\nt=z+1\ns=t+1\n" + # compile first to ensure that no spurious events appear in the trace + code = compile(source, '', 'exec') + sys.settrace(tracer) + exec(code, {}) + sys.settrace(None) + + g() + assert seen == [('call', 1), + ('line', 1), + ('line', 2), + ('line', 3), + ('line', 4), + ('line', 2), + ('line', 3), + ('line', 4), + ('line', 5), + ('return', 5)] + +def test_f_back(): + import sys + def f(): + assert sys._getframe().f_code.co_name == g() + def g(): + return sys._getframe().f_back.f_code.co_name + f() + +def test_f_back_virtualref(): + import sys + def f(): + return g() + def g(): + return sys._getframe() + frame = f() + assert frame.f_back.f_code.co_name == 'f' + +def test_virtualref_through_traceback(): + import sys + def g(): + try: + raise ValueError + except: + _, _, tb = sys.exc_info() + return tb + def f(): + return g() + # + tb = f() + assert tb.tb_frame.f_code.co_name == 'g' + assert tb.tb_frame.f_back.f_code.co_name == 'f' + +def test_trace_basic(): + import sys + l = [] + class Tracer: + def __init__(self, i): + self.i = i + def trace(self, frame, event, arg): + l.append((self.i, frame.f_code.co_name, event, arg)) + if frame.f_code.co_name == 'g2': + return None # don't trace g2 + return Tracer(self.i+1).trace + def g3(n): + n -= 5 + return n + def g2(n): + n += g3(2) + n += g3(7) + return n + def g(n): + n += g2(3) + return n + def f(n): + n = g(n) + return n * 7 + sys.settrace(Tracer(0).trace) + x = f(4) + sys.settrace(None) + assert x == 42 + print(l) + assert l == [(0, 'f', 'call', None), + (1, 'f', 'line', None), + (0, 'g', 'call', None), + (1, 'g', 'line', None), + (0, 'g2', 'call', None), + (0, 'g3', 'call', None), + (1, 'g3', 'line', None), + (2, 'g3', 'line', None), + (3, 'g3', 'return', -3), + (0, 'g3', 'call', None), + (1, 'g3', 'line', None), + (2, 'g3', 'line', None), + (3, 'g3', 'return', 2), + (2, 'g', 'line', None), + (3, 'g', 'return', 6), + (2, 'f', 'line', None), + (3, 'f', 'return', 42)] + +def test_trace_exc(): + import sys + l = [] + def ltrace(a,b,c): + if b == 'exception': + l.append(c) + return ltrace + def trace(a,b,c): return ltrace + def f(): + try: + raise Exception + except: + pass + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 1 + assert isinstance(l[0][1], Exception) + +def test_trace_ignore_hidden(): + import sys + import _testing + + l = [] + def trace(a,b,c): + l.append((a,b,c)) + + def f(): + h = _testing.Hidden() + r = h.meth() + return r + + sys.settrace(trace) + res = f() + sys.settrace(None) + assert len(l) == 1 + assert l[0][1] == 'call' + assert res == 'hidden' # sanity + +def test_trace_hidden_applevel_builtins(): + import sys + + l = [] + def trace(a,b,c): + l.append((a,b,c)) + return trace + + def f(): + sum([]) + sum([]) + sum([]) + return "that's the return value" + + sys.settrace(trace) + f() + sys.settrace(None) + # should get 1 "call", 3 "line" and 1 "return" events, and no call + # or return for the internal app-level implementation of sum + assert len(l) == 6 + assert [what for (frame, what, arg) in l] == [ + 'call', 'line', 'line', 'line', 'line', 'return'] + assert l[-1][2] == "that's the return value" + +def test_trace_return_exc(): + import sys + l = [] + def trace(a,b,c): + if b in ('exception', 'return'): + l.append((b, c)) + return trace + + def g(): + raise Exception + def f(): + try: + g() + except: + pass + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 4 + assert l[0][0] == 'exception' + assert isinstance(l[0][1][1], Exception) + assert l[1] == ('return', None) + assert l[2][0] == 'exception' + assert isinstance(l[2][1][1], Exception) + assert l[3] == ('return', None) + +def test_trace_raises_on_return(): + import sys + def trace(frame, event, arg): + if event == 'return': + raise ValueError + else: + return trace + + def f(): return 1 + + for i in range(sys.getrecursionlimit() + 1): + sys.settrace(trace) + try: + f() + except ValueError: + pass + +def test_trace_try_finally(): + import sys + l = [] + def trace(frame, event, arg): + if event == 'exception': + l.append(arg) + return trace + + def g(): + try: + raise Exception + finally: + pass + + def f(): + try: + g() + except: + pass + + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 2 + assert issubclass(l[0][0], Exception) + assert issubclass(l[1][0], Exception) + +def test_trace_generator_finalisation(): + import sys + l = [] + got_exc = [] + def trace(frame, event, arg): + l.append((frame.f_lineno, event)) + if event == 'exception': + got_exc.append(arg) + return trace + + d = {} + exec("""if 1: + def g(): + try: + yield True + finally: + pass + + def f(): + try: + gen = g() + next(gen) + gen.close() + except: + pass + """, d) + f = d['f'] + + sys.settrace(trace) + f() + sys.settrace(None) + assert len(got_exc) == 1 + assert issubclass(got_exc[0][0], GeneratorExit) + assert l == [(8, 'call'), + (9, 'line'), + (10, 'line'), + (11, 'line'), + (2, 'call'), + (3, 'line'), + (4, 'line'), + (4, 'return'), + (12, 'line'), + (4, 'call'), + (4, 'exception'), + (6, 'line'), + (6, 'return'), + (12, 'return')] + +def test_dont_trace_on_reraise(): + import sys + l = [] + def ltrace(a,b,c): + if b == 'exception': + l.append(c) + return ltrace + def trace(a,b,c): return ltrace + def f(): + try: + 1/0 + except: + try: + raise + except: + pass + sys.settrace(trace) + f() + sys.settrace(None) + assert len(l) == 1 + assert issubclass(l[0][0], Exception) + +def test_trace_changes_locals(): + import sys + def trace(frame, what, arg): + frame.f_locals['x'] = 42 + return trace + def f(x): + return x + sys.settrace(trace) + res = f(1) + sys.settrace(None) + assert res == 42 + +def test_trace_onliner_if(): + import sys + l = [] + def trace(frame, event, arg): + l.append((frame.f_lineno, event)) + return trace + def onliners(): + if True: False + else: True + return 0 + sys.settrace(trace) + onliners() + sys.settrace(None) + firstlineno = onliners.__code__.co_firstlineno + assert l == [(firstlineno + 0, 'call'), + (firstlineno + 1, 'line'), + (firstlineno + 3, 'line'), + (firstlineno + 3, 'return')] + +def test_set_unset_f_trace(): + import sys + seen = [] + def trace1(frame, what, arg): + seen.append((1, frame, frame.f_lineno, what, arg)) + return trace1 + def trace2(frame, what, arg): + seen.append((2, frame, frame.f_lineno, what, arg)) + return trace2 + def set_the_trace(f): + f.f_trace = trace1 + sys.settrace(trace2) + len(seen) # take one line: should not be traced + f = sys._getframe() + set_the_trace(f) + len(seen) # take one line: should not be traced + len(seen) # take one line: should not be traced + sys.settrace(None) # and this line should be the last line traced + len(seen) # take one line + del f.f_trace + len(seen) # take one line + firstline = set_the_trace.__code__.co_firstlineno + assert seen == [(1, f, firstline + 6, 'line', None), + (1, f, firstline + 7, 'line', None), + (1, f, firstline + 8, 'line', None)] + +def test_locals2fast_freevar_bug(): + import sys + def f(n): + class A(object): + def g(self): + return n + n = 42 + return A() + res = f(10).g() + assert res == 10 + # + def trace(*args): + return trace + sys.settrace(trace) + res = f(10).g() + sys.settrace(None) + assert res == 10 + +def test_preserve_exc_state_in_generators(): + import sys + def yield_raise(): + try: + raise KeyError("caught") + except KeyError: + yield sys.exc_info()[0] + yield sys.exc_info()[0] + + it = yield_raise() + assert next(it) is KeyError + assert next(it) is KeyError + +def test_frame_clear(): + import sys, gc, weakref + # + raises(RuntimeError, sys._getframe().clear) + def g(): + yield 5 + raises(RuntimeError, sys._getframe().clear) + yield 6 + assert list(g()) == [5, 6] + # + class A: + pass + a1 = A(); a1ref = weakref.ref(a1) + a2 = A(); a2ref = weakref.ref(a2) + seen = [] + def f(): + local_a1 = a1 + for loc in [5, 6, a2]: + try: + yield sys._getframe() + finally: + seen.append(42) + seen.append(43) + gen = f() + frame = next(gen) + a1 = a2 = None + gc.collect(); gc.collect() + assert a1ref() is not None + assert a2ref() is not None + assert seen == [] + frame.clear() + assert seen == [42] + gc.collect(); gc.collect() + assert a1ref() is None, "locals not cleared" + assert a2ref() is None, "stack not cleared" + # + raises(StopIteration, next, gen) + +def test_frame_clear_really(): + import sys + def f(x): + return sys._getframe() + frame = f(42) + assert frame.f_locals['x'] == 42 + frame.clear() + assert frame.f_locals == {} + +def test_throw_trace_bug(): + import sys + def f(): + yield 5 + gen = f() + assert next(gen) == 5 + seen = [] + def trace_func(frame, event, *args): + seen.append(event) + return trace_func + sys.settrace(trace_func) + try: + gen.throw(ValueError) + except ValueError: + pass + sys.settrace(None) + assert seen == ['call', 'exception', 'return'] + +def test_generator_trace_stopiteration(): + import sys + def f(): + yield 5 + gen = f() + assert next(gen) == 5 + seen = [] + frames = [] + def trace_func(frame, event, *args): + print('TRACE:', frame, event, args) + seen.append(event) + frames.append(frame) + return trace_func + def g(): + for x in gen: + never_entered + sys.settrace(trace_func) + g() + sys.settrace(None) + print('seen:', seen) + # on Python 3 we get an extra 'exception' when 'for' catches + # StopIteration (but not always! mess) + assert seen == ['call', 'line', 'call', 'return', 'exception', 'return'] + assert frames[-2].f_code.co_name == 'g' + +def test_nongenerator_trace_stopiteration(): + import sys + gen = iter([5]) + assert next(gen) == 5 + seen = [] + frames = [] + def trace_func(frame, event, *args): + print('TRACE:', frame, event, args) + seen.append(event) + frames.append(frame) + return trace_func + def g(): + for x in gen: + never_entered + sys.settrace(trace_func) + g() + sys.settrace(None) + print('seen:', seen) + # hack: don't report the StopIteration for some "simple" + # iterators. + assert seen == ['call', 'line', 'return'] + assert frames[-2].f_code.co_name == 'g' + +def test_yieldfrom_trace_stopiteration(): + import sys + def f2(): + yield 5 + def f(): + yield from f2() + gen = f() + assert next(gen) == 5 + seen = [] + frames = [] + def trace_func(frame, event, *args): + print('TRACE:', frame, event, args) + seen.append(event) + frames.append(frame) + return trace_func + def g(): + for x in gen: + never_entered + sys.settrace(trace_func) + g() # invokes next_yield_from() from resume_execute_frame() + sys.settrace(None) + print('seen:', seen) + assert seen == ['call', 'line', 'call', 'call', 'return', + 'exception', 'return', 'exception', 'return'] + assert frames[-4].f_code.co_name == 'f' + assert frames[-2].f_code.co_name == 'g' + +def test_yieldfrom_trace_stopiteration_2(): + import sys + def f2(): + if False: + yield 5 + def f(): + yield from f2() + gen = f() + seen = [] + frames = [] + def trace_func(frame, event, *args): + print('TRACE:', frame, event, args) + seen.append(event) + frames.append(frame) + return trace_func + def g(): + for x in gen: + never_entered + sys.settrace(trace_func) + g() # invokes next_yield_from() from YIELD_FROM() + sys.settrace(None) + print('seen:', seen) + assert seen == ['call', 'line', 'call', 'line', 'call', 'line', + 'return', 'exception', 'return', 'exception', 'return'] + assert frames[-4].f_code.co_name == 'f' + assert frames[-2].f_code.co_name == 'g' + +def test_yieldfrom_trace_stopiteration_3(): + import sys + def f(): + yield from [] + gen = f() + seen = [] + frames = [] + def trace_func(frame, event, *args): + print('TRACE:', frame, event, args) + seen.append(event) + frames.append(frame) + return trace_func + def g(): + for x in gen: + never_entered + sys.settrace(trace_func) + g() # invokes next_yield_from() from YIELD_FROM() + sys.settrace(None) + print('seen:', seen) + assert seen == ['call', 'line', 'call', 'line', + 'return', 'exception', 'return'] + assert frames[-4].f_code.co_name == 'f' + +def test_local_trace_function_returning_None_ignored(): + # behave the same as CPython does, and in contradiction with + # the documentation. + def tracer(f, event, arg): + assert event == 'call' + return local_tracer + + seen = [] + def local_tracer(f, event, arg): + seen.append(event) + return None # but 'local_tracer' will be called again + + def function(): + a = 1 + a = 2 + a = 3 + + import sys + sys.settrace(tracer) + function() + sys.settrace(None) + assert seen == ["line", "line", "line", "return"] + +def test_clear_locals(): + def make_frames(): + def outer(): + x = 5 + y = 6 + def inner(): + z = x + 2 + 1/0 + t = 9 + return inner() + try: + outer() + except ZeroDivisionError as e: + tb = e.__traceback__ + frames = [] + while tb: + frames.append(tb.tb_frame) + tb = tb.tb_next + return frames + + f, outer, inner = make_frames() + outer.clear() + inner.clear() + assert not outer.f_locals + assert not inner.f_locals diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -9,8 +9,6 @@ def setup_class(cls): space = cls.space - cls.w_udir = cls.space.wrap(str(udir.udir)) - cls.w_tempfile1 = cls.space.wrap(str(udir.udir.join('tempfile1'))) if not option.runappdirect: w_call_further = cls.space.appexec([], """(): def call_further(f): @@ -25,175 +23,6 @@ # test for the presence of the attributes, not functionality - def test_f_locals(self): - import sys - f = sys._getframe() - assert f.f_locals is locals() - - def test_f_globals(self): - import sys - f = sys._getframe() - assert f.f_globals is globals() - raises(AttributeError, "f.f_globals = globals()") - - def test_f_builtins(self): - import sys, builtins - f = sys._getframe() - assert f.f_builtins is builtins.__dict__ - - def test_f_code(self): - def g(): - import sys - f = sys._getframe() - return f.f_code - assert g() is g.__code__ - - def test_f_trace_del(self): - import sys - f = sys._getframe() - del f.f_trace - assert f.f_trace is None - - def test_f_lineno(self): - def g(): - import sys - f = sys._getframe() - x = f.f_lineno - y = f.f_lineno - z = f.f_lineno - return [x, y, z] - origin = g.__code__.co_firstlineno - assert g() == [origin+3, origin+4, origin+5] - - def test_f_lineno_set(self): - def tracer(f, *args): - def y(f, *args): - return y - def x(f, *args): - f.f_lineno += 1 - return y # "return None" should have the same effect, but see - # test_local_trace_function_returning_None_ignored - return x - - # obscure: call open beforehand, py3k's open invokes some app - # level code that confuses our tracing (likely due to the - # testing env, otherwise it's not a problem) - f = open(self.tempfile1, 'w') - def function(f=f): - xyz - with f as f: - pass - return 3 - - import sys - sys.settrace(tracer) - function() - sys.settrace(None) - # assert did not crash - - def test_f_lineno_set_2(self): - counter = [0] - errors = [] - - def tracer(f, event, *args): - if event == 'line': - counter[0] += 1 - if counter[0] == 2: - try: - f.f_lineno += 2 - except ValueError as e: - errors.append(e) - return tracer - - # obscure: call open beforehand, py3k's open invokes some app - # level code that confuses our tracing (likely due to the - # testing env, otherwise it's not a problem) - f = open(self.tempfile1, 'w') - def function(): - try: - raise ValueError - except ValueError: - x = 42 - return x - - import sys - sys.settrace(tracer) - x = function() - sys.settrace(None) - assert x == 42 - assert len(errors) == 1 - assert str(errors[0]).startswith( - "can't jump into or out of an 'expect' or 'finally' block") - - def test_f_lineno_set_3(self): - def jump_in_nested_finally(output): - try: - output.append(2) - finally: - output.append(4) - try: - output.append(6) - finally: - output.append(8) - output.append(9) - output = [] - - def tracer(f, event, *args): - if event == 'line' and len(output) == 1: - f.f_lineno += 5 - return tracer - - import sys - sys.settrace(tracer) - jump_in_nested_finally(output) - sys.settrace(None) - assert output == [2, 9] - - def test_f_lineno_set_firstline(self): - r""" - seen = [] - def tracer(f, event, *args): - seen.append((event, f.f_lineno)) - if len(seen) == 5: - f.f_lineno = 1 # bug shown only when setting lineno to 1 - return tracer - - def g(): - import sys - sys.settrace(tracer) - exec("x=1\ny=x+1\nz=y+1\nt=z+1\ns=t+1\n", {}) - sys.settrace(None) - - g() - assert seen == [('call', 1), - ('line', 1), - ('line', 2), - ('line', 3), - ('line', 4), - ('line', 2), - ('line', 3), - ('line', 4), - ('line', 5), - ('return', 5)] - """ - - def test_f_back(self): - import sys - def f(): - assert sys._getframe().f_code.co_name == g() - def g(): - return sys._getframe().f_back.f_code.co_name - f() - - def test_f_back_virtualref(self): - import sys - def f(): - return g() - def g(): - return sys._getframe() - frame = f() - assert frame.f_back.f_code.co_name == 'f' - def test_f_back_hidden(self): if not hasattr(self, 'call_further'): skip("not for runappdirect testing") @@ -210,304 +39,6 @@ assert f1bis is f1 assert f0.f_back is f1 - def test_virtualref_through_traceback(self): - import sys - def g(): - try: - raise ValueError - except: - _, _, tb = sys.exc_info() - return tb - def f(): - return g() - # - tb = f() - assert tb.tb_frame.f_code.co_name == 'g' - assert tb.tb_frame.f_back.f_code.co_name == 'f' - - def test_trace_basic(self): - import sys - l = [] - class Tracer: - def __init__(self, i): - self.i = i - def trace(self, frame, event, arg): - l.append((self.i, frame.f_code.co_name, event, arg)) - if frame.f_code.co_name == 'g2': - return None # don't trace g2 - return Tracer(self.i+1).trace - def g3(n): - n -= 5 - return n - def g2(n): - n += g3(2) - n += g3(7) - return n - def g(n): - n += g2(3) - return n - def f(n): - n = g(n) - return n * 7 - sys.settrace(Tracer(0).trace) - x = f(4) - sys.settrace(None) - assert x == 42 - print(l) - assert l == [(0, 'f', 'call', None), - (1, 'f', 'line', None), - (0, 'g', 'call', None), - (1, 'g', 'line', None), - (0, 'g2', 'call', None), - (0, 'g3', 'call', None), - (1, 'g3', 'line', None), - (2, 'g3', 'line', None), - (3, 'g3', 'return', -3), - (0, 'g3', 'call', None), - (1, 'g3', 'line', None), - (2, 'g3', 'line', None), - (3, 'g3', 'return', 2), - (2, 'g', 'line', None), - (3, 'g', 'return', 6), - (2, 'f', 'line', None), - (3, 'f', 'return', 42)] - - def test_trace_exc(self): - import sys - l = [] - def ltrace(a,b,c): - if b == 'exception': - l.append(c) - return ltrace - def trace(a,b,c): return ltrace - def f(): - try: - raise Exception - except: - pass - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 1 - assert isinstance(l[0][1], Exception) - - def test_trace_ignore_hidden(self): - import sys - import _testing - - l = [] - def trace(a,b,c): - l.append((a,b,c)) - - def f(): - h = _testing.Hidden() - r = h.meth() - return r - - sys.settrace(trace) - res = f() - sys.settrace(None) - assert len(l) == 1 - assert l[0][1] == 'call' - assert res == 'hidden' # sanity - - def test_trace_hidden_applevel_builtins(self): - import sys - - l = [] - def trace(a,b,c): - l.append((a,b,c)) - return trace - - def f(): - sum([]) - sum([]) - sum([]) - return "that's the return value" - - sys.settrace(trace) - f() - sys.settrace(None) - # should get 1 "call", 3 "line" and 1 "return" events, and no call - # or return for the internal app-level implementation of sum - assert len(l) == 6 - assert [what for (frame, what, arg) in l] == [ - 'call', 'line', 'line', 'line', 'line', 'return'] - assert l[-1][2] == "that's the return value" - - def test_trace_return_exc(self): - import sys - l = [] - def trace(a,b,c): - if b in ('exception', 'return'): - l.append((b, c)) - return trace - - def g(): - raise Exception - def f(): - try: - g() - except: - pass - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 4 - assert l[0][0] == 'exception' - assert isinstance(l[0][1][1], Exception) - assert l[1] == ('return', None) - assert l[2][0] == 'exception' - assert isinstance(l[2][1][1], Exception) - assert l[3] == ('return', None) - - def test_trace_raises_on_return(self): - import sys - def trace(frame, event, arg): - if event == 'return': - raise ValueError - else: - return trace - - def f(): return 1 - - for i in range(sys.getrecursionlimit() + 1): - sys.settrace(trace) - try: - f() - except ValueError: - pass - - def test_trace_try_finally(self): - import sys - l = [] - def trace(frame, event, arg): - if event == 'exception': - l.append(arg) - return trace - - def g(): - try: - raise Exception - finally: - pass - - def f(): - try: - g() - except: - pass - - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 2 - assert issubclass(l[0][0], Exception) - assert issubclass(l[1][0], Exception) - - def test_trace_generator_finalisation(self): - ''' - import sys - l = [] - got_exc = [] - def trace(frame, event, arg): - l.append((frame.f_lineno, event)) - if event == 'exception': - got_exc.append(arg) - return trace - - d = {} - exec("""if 1: - def g(): - try: - yield True - finally: - pass - - def f(): - try: - gen = g() - next(gen) - gen.close() - except: - pass - """, d) - f = d['f'] - - sys.settrace(trace) - f() - sys.settrace(None) - assert len(got_exc) == 1 - assert issubclass(got_exc[0][0], GeneratorExit) - assert l == [(8, 'call'), - (9, 'line'), - (10, 'line'), - (11, 'line'), - (2, 'call'), - (3, 'line'), - (4, 'line'), - (4, 'return'), - (12, 'line'), - (4, 'call'), - (4, 'exception'), - (6, 'line'), - (6, 'return'), - (12, 'return')] - ''' - - def test_dont_trace_on_reraise(self): - import sys - l = [] - def ltrace(a,b,c): - if b == 'exception': - l.append(c) - return ltrace - def trace(a,b,c): return ltrace - def f(): - try: - 1/0 - except: - try: - raise - except: - pass - sys.settrace(trace) - f() - sys.settrace(None) - assert len(l) == 1 - assert issubclass(l[0][0], Exception) - - def test_trace_changes_locals(self): - import sys - def trace(frame, what, arg): - frame.f_locals['x'] = 42 - return trace - def f(x): - return x - sys.settrace(trace) - res = f(1) - sys.settrace(None) - assert res == 42 - - def test_trace_onliner_if(self): - import sys - l = [] - def trace(frame, event, arg): - l.append((frame.f_lineno, event)) - return trace - def onliners(): - if True: False - else: True - return 0 - sys.settrace(trace) - onliners() - sys.settrace(None) - firstlineno = onliners.__code__.co_firstlineno - assert l == [(firstlineno + 0, 'call'), - (firstlineno + 1, 'line'), - (firstlineno + 3, 'line'), - (firstlineno + 3, 'return')] - def test_fast2locals_called_lazily(self): import sys class FrameHolder: @@ -526,302 +57,3 @@ assert res == 2 if hasattr(self, "check_no_w_locals"): # not appdirect assert self.check_no_w_locals(fh.frame) - - def test_set_unset_f_trace(self): - import sys - seen = [] - def trace1(frame, what, arg): - seen.append((1, frame, frame.f_lineno, what, arg)) - return trace1 - def trace2(frame, what, arg): - seen.append((2, frame, frame.f_lineno, what, arg)) - return trace2 - def set_the_trace(f): - f.f_trace = trace1 - sys.settrace(trace2) - len(seen) # take one line: should not be traced - f = sys._getframe() - set_the_trace(f) - len(seen) # take one line: should not be traced - len(seen) # take one line: should not be traced - sys.settrace(None) # and this line should be the last line traced - len(seen) # take one line - del f.f_trace - len(seen) # take one line - firstline = set_the_trace.__code__.co_firstlineno - assert seen == [(1, f, firstline + 6, 'line', None), - (1, f, firstline + 7, 'line', None), - (1, f, firstline + 8, 'line', None)] - - def test_locals2fast_freevar_bug(self): - import sys - def f(n): - class A(object): - def g(self): - return n - n = 42 - return A() - res = f(10).g() - assert res == 10 - # - def trace(*args): - return trace - sys.settrace(trace) - res = f(10).g() - sys.settrace(None) - assert res == 10 - - def test_preserve_exc_state_in_generators(self): - import sys - def yield_raise(): - try: - raise KeyError("caught") - except KeyError: - yield sys.exc_info()[0] - yield sys.exc_info()[0] - - it = yield_raise() - assert next(it) is KeyError - assert next(it) is KeyError - - def test_frame_clear(self): - import sys, gc, weakref - # - raises(RuntimeError, sys._getframe().clear) - def g(): - yield 5 - raises(RuntimeError, sys._getframe().clear) - yield 6 - assert list(g()) == [5, 6] - # - class A: - pass - a1 = A(); a1ref = weakref.ref(a1) - a2 = A(); a2ref = weakref.ref(a2) - seen = [] - def f(): - local_a1 = a1 - for loc in [5, 6, a2]: - try: - yield sys._getframe() - finally: - seen.append(42) - seen.append(43) - gen = f() - frame = next(gen) - a1 = a2 = None - gc.collect(); gc.collect() - assert a1ref() is not None - assert a2ref() is not None - assert seen == [] - frame.clear() - assert seen == [42] - gc.collect(); gc.collect() - assert a1ref() is None, "locals not cleared" - assert a2ref() is None, "stack not cleared" - # - raises(StopIteration, next, gen) - - def test_frame_clear_really(self): - import sys - def f(x): - return sys._getframe() - frame = f(42) - assert frame.f_locals['x'] == 42 - frame.clear() - assert frame.f_locals == {} - - def test_throw_trace_bug(self): - import sys - def f(): - yield 5 - gen = f() - assert next(gen) == 5 - seen = [] - def trace_func(frame, event, *args): - seen.append(event) - return trace_func - sys.settrace(trace_func) - try: - gen.throw(ValueError) - except ValueError: - pass - sys.settrace(None) - assert seen == ['call', 'exception', 'return'] - - def test_generator_trace_stopiteration(self): - import sys - def f(): - yield 5 - gen = f() - assert next(gen) == 5 - seen = [] - frames = [] - def trace_func(frame, event, *args): - print('TRACE:', frame, event, args) - seen.append(event) - frames.append(frame) - return trace_func - def g(): - for x in gen: - never_entered - sys.settrace(trace_func) - g() - sys.settrace(None) - print('seen:', seen) - # on Python 3 we get an extra 'exception' when 'for' catches - # StopIteration (but not always! mess) - assert seen == ['call', 'line', 'call', 'return', 'exception', 'return'] - assert frames[-2].f_code.co_name == 'g' - - def test_nongenerator_trace_stopiteration(self): - import sys - gen = iter([5]) - assert next(gen) == 5 - seen = [] - frames = [] - def trace_func(frame, event, *args): - print('TRACE:', frame, event, args) - seen.append(event) - frames.append(frame) - return trace_func - def g(): - for x in gen: - never_entered - sys.settrace(trace_func) - g() - sys.settrace(None) - print('seen:', seen) - # hack: don't report the StopIteration for some "simple" - # iterators. - assert seen == ['call', 'line', 'return'] - assert frames[-2].f_code.co_name == 'g' - - def test_yieldfrom_trace_stopiteration(self): """ - import sys - def f2(): - yield 5 - def f(): - yield from f2() - gen = f() - assert next(gen) == 5 - seen = [] - frames = [] - def trace_func(frame, event, *args): - print('TRACE:', frame, event, args) - seen.append(event) - frames.append(frame) - return trace_func - def g(): - for x in gen: - never_entered - sys.settrace(trace_func) - g() # invokes next_yield_from() from resume_execute_frame() - sys.settrace(None) - print('seen:', seen) - assert seen == ['call', 'line', 'call', 'call', 'return', - 'exception', 'return', 'exception', 'return'] - assert frames[-4].f_code.co_name == 'f' - assert frames[-2].f_code.co_name == 'g' - """ - - def test_yieldfrom_trace_stopiteration_2(self): """ - import sys - def f2(): - if False: - yield 5 - def f(): - yield from f2() - gen = f() - seen = [] - frames = [] - def trace_func(frame, event, *args): - print('TRACE:', frame, event, args) - seen.append(event) - frames.append(frame) - return trace_func - def g(): - for x in gen: - never_entered - sys.settrace(trace_func) - g() # invokes next_yield_from() from YIELD_FROM() - sys.settrace(None) - print('seen:', seen) - assert seen == ['call', 'line', 'call', 'line', 'call', 'line', - 'return', 'exception', 'return', 'exception', 'return'] - assert frames[-4].f_code.co_name == 'f' - assert frames[-2].f_code.co_name == 'g' - """ - - def test_yieldfrom_trace_stopiteration_3(self): """ - import sys - def f(): - yield from [] - gen = f() - seen = [] - frames = [] - def trace_func(frame, event, *args): - print('TRACE:', frame, event, args) - seen.append(event) - frames.append(frame) - return trace_func - def g(): - for x in gen: - never_entered - sys.settrace(trace_func) - g() # invokes next_yield_from() from YIELD_FROM() - sys.settrace(None) - print('seen:', seen) - assert seen == ['call', 'line', 'call', 'line', - 'return', 'exception', 'return'] - assert frames[-4].f_code.co_name == 'f' - """ - - def test_local_trace_function_returning_None_ignored(self): - # behave the same as CPython does, and in contradiction with - # the documentation. - def tracer(f, event, arg): - assert event == 'call' - return local_tracer - - seen = [] - def local_tracer(f, event, arg): - seen.append(event) - return None # but 'local_tracer' will be called again - - def function(): - a = 1 - a = 2 - a = 3 - - import sys - sys.settrace(tracer) - function() - sys.settrace(None) - assert seen == ["line", "line", "line", "return"] - - def test_clear_locals(self): - def make_frames(): - def outer(): - x = 5 - y = 6 - def inner(): - z = x + 2 - 1/0 - t = 9 - return inner() - try: - outer() - except ZeroDivisionError as e: - tb = e.__traceback__ - frames = [] - while tb: - frames.append(tb.tb_frame) - tb = tb.tb_next - return frames - - f, outer, inner = make_frames() - outer.clear() - inner.clear() - assert not outer.f_locals - assert not inner.f_locals From pypy.commits at gmail.com Fri Mar 23 09:10:08 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 06:10:08 -0700 (PDT) Subject: [pypy-commit] pypy py3tests: Get old-style -A tests working again Message-ID: <5ab4fcb0.e4a6df0a.1f491.dbd9@mx.google.com> Author: Ronan Lamy Branch: py3tests Changeset: r94118:e8901ea6307b Date: 2018-03-23 14:09 +0100 http://bitbucket.org/pypy/pypy/changeset/e8901ea6307b/ Log: Get old-style -A tests working again diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -9,6 +9,7 @@ PYTHON3 = os.getenv('PYTHON3') or py.path.local.sysfind(LOOK_FOR_PYTHON3) if PYTHON3 is not None: PYTHON3 = str(PYTHON3) +HOST_IS_PY3 = sys.version_info[0] > 2 APPLEVEL_FN = 'apptest_*.py' # pytest settings @@ -39,12 +40,16 @@ pluginmanager.register(LeakFinder()) def pytest_configure(config): + if HOST_IS_PY3 and not config.getoption('runappdirect'): + raise ValueError( + "On top of a Python 3 interpreter, the -A flag is mandatory") global option option = config.option def py3k_skip(message): py.test.skip('[py3k] %s' % message) py.test.py3k_skip = py3k_skip - config.addinivalue_line('python_files', APPLEVEL_FN) + if HOST_IS_PY3: + config.addinivalue_line('python_files', APPLEVEL_FN) def pytest_addoption(parser): group = parser.getgroup("pypy options") @@ -96,13 +101,14 @@ ensure_pytest_builtin_helpers() def pytest_pycollect_makemodule(path, parent): - if not parent.config.getoption('runappdirect'): - if path.fnmatch(APPLEVEL_FN): - from pypy.tool.pytest.apptest2 import AppTestModule - rewrite = parent.config.getoption('applevel_rewrite') - return AppTestModule(path, parent, rewrite_asserts=rewrite) - else: - return PyPyModule(path, parent) + if HOST_IS_PY3: + return + elif path.fnmatch(APPLEVEL_FN): + from pypy.tool.pytest.apptest2 import AppTestModule + rewrite = parent.config.getoption('applevel_rewrite') + return AppTestModule(path, parent, rewrite_asserts=rewrite) + else: + return PyPyModule(path, parent) def is_applevel(item): from pypy.tool.pytest.apptest import AppTestFunction @@ -203,7 +209,6 @@ def pytest_ignore_collect(path, config): - if (config.getoption('runappdirect') and - not path.isdir() and not path.fnmatch(APPLEVEL_FN)): + if (HOST_IS_PY3 and not path.isdir() and not path.fnmatch(APPLEVEL_FN)): return True return path.check(link=1) From pypy.commits at gmail.com Fri Mar 23 10:19:40 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 07:19:40 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: Backport conftest changes from py3tests branch Message-ID: <5ab50cfc.c7591c0a.439f0.83ec@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94120:12655c5f9ce4 Date: 2018-03-23 15:14 +0100 http://bitbucket.org/pypy/pypy/changeset/12655c5f9ce4/ Log: Backport conftest changes from py3tests branch diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -1,6 +1,8 @@ import py, pytest, sys, textwrap from inspect import isclass +APPLEVEL_FN = 'apptest_*.py' + # pytest settings rsyncdirs = ['.', '../lib-python', '../lib_pypy', '../demo'] rsyncignore = ['_cache'] @@ -30,11 +32,9 @@ def pytest_configure(config): global option option = config.option + config.addinivalue_line('python_files', APPLEVEL_FN) def pytest_addoption(parser): - from rpython.conftest import pytest_addoption - pytest_addoption(parser) - group = parser.getgroup("pypy options") group.addoption('-A', '--runappdirect', action="store_true", default=False, dest="runappdirect", @@ -45,6 +45,9 @@ group.addoption('--raise-operr', action="store_true", default=False, dest="raise_operr", help="Show the interp-level OperationError in app-level tests") + group.addoption('--applevel-rewrite', action="store_true", + default=False, dest="applevel_rewrite", + help="Use assert rewriting in app-level test files (slow)") @pytest.fixture(scope='function') def space(request): @@ -75,7 +78,14 @@ ensure_pytest_builtin_helpers() def pytest_pycollect_makemodule(path, parent): - return PyPyModule(path, parent) + if path.fnmatch(APPLEVEL_FN): + if parent.config.getoption('runappdirect'): + return + from pypy.tool.pytest.apptest2 import AppTestModule + rewrite = parent.config.getoption('applevel_rewrite') + return AppTestModule(path, parent, rewrite_asserts=rewrite) + else: + return PyPyModule(path, parent) def is_applevel(item): from pypy.tool.pytest.apptest import AppTestFunction @@ -91,7 +101,8 @@ else: item.add_marker('interplevel') -class PyPyModule(py.test.collect.Module): + +class PyPyModule(pytest.Module): """ we take care of collecting classes both at app level and at interp-level (because we need to stick a space at the class) ourselves. @@ -174,5 +185,5 @@ appclass.obj.runappdirect = option.runappdirect -def pytest_ignore_collect(path): +def pytest_ignore_collect(path, config): return path.check(link=1) From pypy.commits at gmail.com Fri Mar 23 10:19:38 2018 From: pypy.commits at gmail.com (rlamy) Date: Fri, 23 Mar 2018 07:19:38 -0700 (PDT) Subject: [pypy-commit] pypy apptest-file: Backport new files from py3tests branch Message-ID: <5ab50cfa.91bddf0a.bfc84.42c9@mx.google.com> Author: Ronan Lamy Branch: apptest-file Changeset: r94119:ba2f4ba5e8c4 Date: 2018-03-23 14:37 +0100 http://bitbucket.org/pypy/pypy/changeset/ba2f4ba5e8c4/ Log: Backport new files from py3tests branch diff --git a/pypy/tool/pytest/app_rewrite.py b/pypy/tool/pytest/app_rewrite.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/app_rewrite.py @@ -0,0 +1,39 @@ +import re + +ASCII_IS_DEFAULT_ENCODING = False + +cookie_re = re.compile(r"^[ \t\f]*#.*coding[:=][ \t]*[-\w.]+") +BOM_UTF8 = '\xef\xbb\xbf' + +def _prepare_source(fn): + """Read the source code for re-writing.""" + try: + stat = fn.stat() + source = fn.read("rb") + except EnvironmentError: + return None, None + if ASCII_IS_DEFAULT_ENCODING: + # ASCII is the default encoding in Python 2. Without a coding + # declaration, Python 2 will complain about any bytes in the file + # outside the ASCII range. Sadly, this behavior does not extend to + # compile() or ast.parse(), which prefer to interpret the bytes as + # latin-1. (At least they properly handle explicit coding cookies.) To + # preserve this error behavior, we could force ast.parse() to use ASCII + # as the encoding by inserting a coding cookie. Unfortunately, that + # messes up line numbers. Thus, we have to check ourselves if anything + # is outside the ASCII range in the case no encoding is explicitly + # declared. For more context, see issue #269. Yay for Python 3 which + # gets this right. + end1 = source.find("\n") + end2 = source.find("\n", end1 + 1) + if (not source.startswith(BOM_UTF8) and + cookie_re.match(source[0:end1]) is None and + cookie_re.match(source[end1 + 1:end2]) is None): + try: + source.decode("ascii") + except UnicodeDecodeError: + # Let it fail in real import. + return None, None + # On Python versions which are not 2.7 and less than or equal to 3.1, the + # parser expects *nix newlines. + return stat, source diff --git a/pypy/tool/pytest/apptest2.py b/pypy/tool/pytest/apptest2.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/apptest2.py @@ -0,0 +1,119 @@ +import sys +import os + +import pytest +from pypy import pypydir +import pypy.interpreter.function +from pypy.tool.pytest import app_rewrite +from pypy.interpreter.error import OperationError +from pypy.interpreter.module import Module +from pypy.tool.pytest import objspace +from pypy.tool.pytest import appsupport + + +class AppTestModule(pytest.Module): + def __init__(self, path, parent, rewrite_asserts=False): + super(AppTestModule, self).__init__(path, parent) + self.rewrite_asserts = rewrite_asserts + + def collect(self): + _, source = app_rewrite._prepare_source(self.fspath) + space = objspace.gettestobjspace() + w_rootdir = space.newtext( + os.path.join(pypydir, 'tool', 'pytest', 'ast-rewriter')) + w_source = space.newtext(source) + fname = str(self.fspath) + w_fname = space.newtext(fname) + if self.rewrite_asserts: + w_mod = space.appexec([w_rootdir, w_source, w_fname], + """(rootdir, source, fname): + import sys + sys.path.insert(0, rootdir) + from ast_rewrite import rewrite_asserts, create_module + + co = rewrite_asserts(source, fname) + mod = create_module(fname, co) + return mod + """) + else: + w_mod = create_module(space, w_fname, fname, source) + mod_dict = w_mod.getdict(space).unwrap(space) + items = [] + for name, w_obj in mod_dict.items(): + if not name.startswith('test_'): + continue + if not isinstance(w_obj, pypy.interpreter.function.Function): + continue + items.append(AppTestFunction(name, self, w_obj)) + return items + + def setup(self): + pass + +def create_module(space, w_name, filename, source): + w_mod = Module(space, w_name) + w_dict = w_mod.getdict(space) + space.setitem(w_dict, space.newtext('__file__'), space.newtext(filename)) + space.exec_(source, w_dict, w_dict, filename=filename) + return w_mod + + +class AppError(Exception): + + def __init__(self, excinfo): + self.excinfo = excinfo + + +class AppTestFunction(pytest.Item): + + def __init__(self, name, parent, w_obj): + super(AppTestFunction, self).__init__(name, parent) + self.w_obj = w_obj + + def runtest(self): + target = self.w_obj + space = target.space + self.execute_appex(space, target) + + def repr_failure(self, excinfo): + if excinfo.errisinstance(AppError): + excinfo = excinfo.value.excinfo + return super(AppTestFunction, self).repr_failure(excinfo) + + def execute_appex(self, space, w_func): + space.getexecutioncontext().set_sys_exc_info(None) + sig = w_func.code._signature + if sig.varargname or sig.kwargname or sig.kwonlyargnames: + raise ValueError( + 'Test functions may not use *args, **kwargs or ' + 'keyword-only args') + args_w = self.get_fixtures(space, sig.argnames) + try: + space.call_function(w_func, *args_w) + except OperationError as e: + if self.config.option.raise_operr: + raise + tb = sys.exc_info()[2] + if e.match(space, space.w_KeyboardInterrupt): + raise KeyboardInterrupt, KeyboardInterrupt(), tb + appexcinfo = appsupport.AppExceptionInfo(space, e) + if appexcinfo.traceback: + raise AppError, AppError(appexcinfo), tb + raise + + def reportinfo(self): + """Must return a triple (fspath, lineno, test_name)""" + lineno = self.w_obj.code.co_firstlineno + return self.parent.fspath, lineno, self.w_obj.name + + def get_fixtures(self, space, fixtures): + if not fixtures: + return [] + import imp + fixtures_mod = imp.load_source( + 'fixtures', str(self.parent.fspath.new(basename='fixtures.py'))) + result = [] + for name in fixtures: + arg = getattr(fixtures_mod, name)(space, self.parent.config) + result.append(arg) + return result diff --git a/pypy/tool/pytest/ast-rewriter/ast_rewrite.py b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/ast-rewriter/ast_rewrite.py @@ -0,0 +1,648 @@ +"""Rewrite assertion AST to produce nice error messages""" +from __future__ import absolute_import, division, print_function +import ast +import itertools +import marshal +import struct +import sys + +from ast_util import assertrepr_compare, format_explanation as _format_explanation + + +# pytest caches rewritten pycs in __pycache__. +PYTEST_TAG = sys.implementation.cache_tag + "-PYTEST" + +PYC_EXT = ".py" + (__debug__ and "c" or "o") +PYC_TAIL = "." + PYTEST_TAG + PYC_EXT + + +if sys.version_info >= (3, 5): + ast_Call = ast.Call +else: + def ast_Call(a, b, c): + return ast.Call(a, b, c, None, None) + + +def _write_pyc(state, co, source_stat, pyc): + # Technically, we don't have to have the same pyc format as + # (C)Python, since these "pycs" should never be seen by builtin + # import. However, there's little reason deviate, and I hope + # sometime to be able to use imp.load_compiled to load them. (See + # the comment in load_module above.) + try: + fp = open(pyc, "wb") + except IOError: + err = sys.exc_info()[1].errno + state.trace("error writing pyc file at %s: errno=%s" % (pyc, err)) + # we ignore any failure to write the cache file + # there are many reasons, permission-denied, __pycache__ being a + # file etc. + return False + try: + fp.write(imp.get_magic()) + mtime = int(source_stat.mtime) + size = source_stat.size & 0xFFFFFFFF + fp.write(struct.pack(">", + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.FloorDiv: "//", + ast.Mod: "%%", # escaped for string formatting + ast.Eq: "==", + ast.NotEq: "!=", + ast.Lt: "<", + ast.LtE: "<=", + ast.Gt: ">", + ast.GtE: ">=", + ast.Pow: "**", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in" +} +# Python 3.5+ compatibility +try: + binop_map[ast.MatMult] = "@" +except AttributeError: + pass + +# Python 3.4+ compatibility +if hasattr(ast, "NameConstant"): + _NameConstant = ast.NameConstant +else: + def _NameConstant(c): + return ast.Name(str(c), ast.Load()) + + +def set_location(node, lineno, col_offset): + """Set node location information recursively.""" + def _fix(node, lineno, col_offset): + if "lineno" in node._attributes: + node.lineno = lineno + if "col_offset" in node._attributes: + node.col_offset = col_offset + for child in ast.iter_child_nodes(node): + _fix(child, lineno, col_offset) + _fix(node, lineno, col_offset) + return node + + +class AssertionRewriter(ast.NodeVisitor): + """Assertion rewriting implementation. + + The main entrypoint is to call .run() with an ast.Module instance, + this will then find all the assert statements and rewrite them to + provide intermediate values and a detailed assertion error. See + http://pybites.blogspot.be/2011/07/behind-scenes-of-pytests-new-assertion.html + for an overview of how this works. + + The entry point here is .run() which will iterate over all the + statements in an ast.Module and for each ast.Assert statement it + finds call .visit() with it. Then .visit_Assert() takes over and + is responsible for creating new ast statements to replace the + original assert statement: it rewrites the test of an assertion + to provide intermediate values and replace it with an if statement + which raises an assertion error with a detailed explanation in + case the expression is false. + + For this .visit_Assert() uses the visitor pattern to visit all the + AST nodes of the ast.Assert.test field, each visit call returning + an AST node and the corresponding explanation string. During this + state is kept in several instance attributes: + + :statements: All the AST statements which will replace the assert + statement. + + :variables: This is populated by .variable() with each variable + used by the statements so that they can all be set to None at + the end of the statements. + + :variable_counter: Counter to create new unique variables needed + by statements. Variables are created using .variable() and + have the form of "@py_assert0". + + :on_failure: The AST statements which will be executed if the + assertion test fails. This is the code which will construct + the failure message and raises the AssertionError. + + :explanation_specifiers: A dict filled by .explanation_param() + with %-formatting placeholders and their corresponding + expressions to use in the building of an assertion message. + This is used by .pop_format_context() to build a message. + + :stack: A stack of the explanation_specifiers dicts maintained by + .push_format_context() and .pop_format_context() which allows + to build another %-formatted string while already building one. + + This state is reset on every new assert statement visited and used + by the other visitors. + + """ + + def __init__(self, module_path): + super(AssertionRewriter, self).__init__() + self.module_path = module_path + + def run(self, mod): + """Find all assert statements in *mod* and rewrite them.""" + if not mod.body: + # Nothing to do. + return + # Insert some special imports at the top of the module but after any + # docstrings and __future__ imports. + if sys.version_info[0] >= 3: + builtin_name = 'builtins' + else: + builtin_name = '__builtin__' + aliases = [ast.alias(builtin_name, "@py_builtins"), + ast.alias("ast_rewrite", "@pytest_ar")] + doc = getattr(mod, "docstring", None) + expect_docstring = doc is None + if doc is not None and self.is_rewrite_disabled(doc): + return + pos = 0 + lineno = 1 + for item in mod.body: + if (expect_docstring and isinstance(item, ast.Expr) and + isinstance(item.value, ast.Str)): + doc = item.value.s + if self.is_rewrite_disabled(doc): + return + expect_docstring = False + elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or + item.module != "__future__"): + lineno = item.lineno + break + pos += 1 + else: + lineno = item.lineno + imports = [ast.Import([alias], lineno=lineno, col_offset=0) + for alias in aliases] + mod.body[pos:pos] = imports + # Collect asserts. + nodes = [mod] + while nodes: + node = nodes.pop() + for name, field in ast.iter_fields(node): + if isinstance(field, list): + new = [] + for i, child in enumerate(field): + if isinstance(child, ast.Assert): + # Transform assert. + new.extend(self.visit(child)) + else: + new.append(child) + if isinstance(child, ast.AST): + nodes.append(child) + setattr(node, name, new) + elif (isinstance(field, ast.AST) and + # Don't recurse into expressions as they can't contain + # asserts. + not isinstance(field, ast.expr)): + nodes.append(field) + + @staticmethod + def is_rewrite_disabled(docstring): + return "PYTEST_DONT_REWRITE" in docstring + + def variable(self): + """Get a new variable.""" + # Use a character invalid in python identifiers to avoid clashing. + name = "@py_assert" + str(next(self.variable_counter)) + self.variables.append(name) + return name + + def assign(self, expr): + """Give *expr* a name.""" + name = self.variable() + self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr)) + return ast.Name(name, ast.Load()) + + def display(self, expr): + """Call py.io.saferepr on the expression.""" + return self.helper("saferepr", expr) + + def helper(self, name, *args): + """Call a helper in this module.""" + py_name = ast.Name("@pytest_ar", ast.Load()) + attr = ast.Attribute(py_name, "_" + name, ast.Load()) + return ast_Call(attr, list(args), []) + + def builtin(self, name): + """Return the builtin called *name*.""" + builtin_name = ast.Name("@py_builtins", ast.Load()) + return ast.Attribute(builtin_name, name, ast.Load()) + + def explanation_param(self, expr): + """Return a new named %-formatting placeholder for expr. + + This creates a %-formatting placeholder for expr in the + current formatting context, e.g. ``%(py0)s``. The placeholder + and expr are placed in the current format context so that it + can be used on the next call to .pop_format_context(). + + """ + specifier = "py" + str(next(self.variable_counter)) + self.explanation_specifiers[specifier] = expr + return "%(" + specifier + ")s" + + def push_format_context(self): + """Create a new formatting context. + + The format context is used for when an explanation wants to + have a variable value formatted in the assertion message. In + this case the value required can be added using + .explanation_param(). Finally .pop_format_context() is used + to format a string of %-formatted values as added by + .explanation_param(). + + """ + self.explanation_specifiers = {} + self.stack.append(self.explanation_specifiers) + + def pop_format_context(self, expl_expr): + """Format the %-formatted string with current format context. + + The expl_expr should be an ast.Str instance constructed from + the %-placeholders created by .explanation_param(). This will + add the required code to format said string to .on_failure and + return the ast.Name instance of the formatted string. + + """ + current = self.stack.pop() + if self.stack: + self.explanation_specifiers = self.stack[-1] + keys = [ast.Str(key) for key in current.keys()] + format_dict = ast.Dict(keys, list(current.values())) + form = ast.BinOp(expl_expr, ast.Mod(), format_dict) + name = "@py_format" + str(next(self.variable_counter)) + self.on_failure.append(ast.Assign([ast.Name(name, ast.Store())], form)) + return ast.Name(name, ast.Load()) + + def generic_visit(self, node): + """Handle expressions we don't have custom code for.""" + assert isinstance(node, ast.expr) + res = self.assign(node) + return res, self.explanation_param(self.display(res)) + + def visit_Assert(self, assert_): + """Return the AST statements to replace the ast.Assert instance. + + This rewrites the test of an assertion to provide + intermediate values and replace it with an if statement which + raises an assertion error with a detailed explanation in case + the expression is false. + + """ + self.statements = [] + self.variables = [] + self.variable_counter = itertools.count() + self.stack = [] + self.on_failure = [] + self.push_format_context() + # Rewrite assert into a bunch of statements. + top_condition, explanation = self.visit(assert_.test) + # Create failure message. + body = self.on_failure + negation = ast.UnaryOp(ast.Not(), top_condition) + self.statements.append(ast.If(negation, body, [])) + if assert_.msg: + assertmsg = self.helper('format_assertmsg', assert_.msg) + explanation = "\n>assert " + explanation + else: + assertmsg = ast.Str("") + explanation = "assert " + explanation + template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation)) + msg = self.pop_format_context(template) + fmt = self.helper("format_explanation", msg) + err_name = ast.Name("AssertionError", ast.Load()) + exc = ast_Call(err_name, [fmt], []) + if sys.version_info[0] >= 3: + raise_ = ast.Raise(exc, None) + else: + raise_ = ast.Raise(exc, None, None) + body.append(raise_) + # Clear temporary variables by setting them to None. + if self.variables: + variables = [ast.Name(name, ast.Store()) + for name in self.variables] + clear = ast.Assign(variables, _NameConstant(None)) + self.statements.append(clear) + # Fix line numbers. + for stmt in self.statements: + set_location(stmt, assert_.lineno, assert_.col_offset) + return self.statements + + def visit_Name(self, name): + # Display the repr of the name if it's a local variable or + # _should_repr_global_name() thinks it's acceptable. + locs = ast_Call(self.builtin("locals"), [], []) + inlocs = ast.Compare(ast.Str(name.id), [ast.In()], [locs]) + dorepr = self.helper("should_repr_global_name", name) + test = ast.BoolOp(ast.Or(), [inlocs, dorepr]) + expr = ast.IfExp(test, self.display(name), ast.Str(name.id)) + return name, self.explanation_param(expr) + + def visit_BoolOp(self, boolop): + res_var = self.variable() + expl_list = self.assign(ast.List([], ast.Load())) + app = ast.Attribute(expl_list, "append", ast.Load()) + is_or = int(isinstance(boolop.op, ast.Or)) + body = save = self.statements + fail_save = self.on_failure + levels = len(boolop.values) - 1 + self.push_format_context() + # Process each operand, short-circuting if needed. + for i, v in enumerate(boolop.values): + if i: + fail_inner = [] + # cond is set in a prior loop iteration below + self.on_failure.append(ast.If(cond, fail_inner, [])) # noqa + self.on_failure = fail_inner + self.push_format_context() + res, expl = self.visit(v) + body.append(ast.Assign([ast.Name(res_var, ast.Store())], res)) + expl_format = self.pop_format_context(ast.Str(expl)) + call = ast_Call(app, [expl_format], []) + self.on_failure.append(ast.Expr(call)) + if i < levels: + cond = res + if is_or: + cond = ast.UnaryOp(ast.Not(), cond) + inner = [] + self.statements.append(ast.If(cond, inner, [])) + self.statements = body = inner + self.statements = save + self.on_failure = fail_save + expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or)) + expl = self.pop_format_context(expl_template) + return ast.Name(res_var, ast.Load()), self.explanation_param(expl) + + def visit_UnaryOp(self, unary): + pattern = unary_map[unary.op.__class__] + operand_res, operand_expl = self.visit(unary.operand) + res = self.assign(ast.UnaryOp(unary.op, operand_res)) + return res, pattern % (operand_expl,) + + def visit_BinOp(self, binop): + symbol = binop_map[binop.op.__class__] + left_expr, left_expl = self.visit(binop.left) + right_expr, right_expl = self.visit(binop.right) + explanation = "(%s %s %s)" % (left_expl, symbol, right_expl) + res = self.assign(ast.BinOp(left_expr, binop.op, right_expr)) + return res, explanation + + def visit_Call_35(self, call): + """ + visit `ast.Call` nodes on Python3.5 and after + """ + new_func, func_expl = self.visit(call.func) + arg_expls = [] + new_args = [] + new_kwargs = [] + for arg in call.args: + res, expl = self.visit(arg) + arg_expls.append(expl) + new_args.append(res) + for keyword in call.keywords: + res, expl = self.visit(keyword.value) + new_kwargs.append(ast.keyword(keyword.arg, res)) + if keyword.arg: + arg_expls.append(keyword.arg + "=" + expl) + else: # **args have `arg` keywords with an .arg of None + arg_expls.append("**" + expl) + + expl = "%s(%s)" % (func_expl, ', '.join(arg_expls)) + new_call = ast.Call(new_func, new_args, new_kwargs) + res = self.assign(new_call) + res_expl = self.explanation_param(self.display(res)) + outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl) + return res, outer_expl + + def visit_Starred(self, starred): + # From Python 3.5, a Starred node can appear in a function call + res, expl = self.visit(starred.value) + return starred, '*' + expl + + def visit_Call_legacy(self, call): + """ + visit `ast.Call nodes on 3.4 and below` + """ + new_func, func_expl = self.visit(call.func) + arg_expls = [] + new_args = [] + new_kwargs = [] + new_star = new_kwarg = None + for arg in call.args: + res, expl = self.visit(arg) + new_args.append(res) + arg_expls.append(expl) + for keyword in call.keywords: + res, expl = self.visit(keyword.value) + new_kwargs.append(ast.keyword(keyword.arg, res)) + arg_expls.append(keyword.arg + "=" + expl) + if call.starargs: + new_star, expl = self.visit(call.starargs) + arg_expls.append("*" + expl) + if call.kwargs: + new_kwarg, expl = self.visit(call.kwargs) + arg_expls.append("**" + expl) + expl = "%s(%s)" % (func_expl, ', '.join(arg_expls)) + new_call = ast.Call(new_func, new_args, new_kwargs, + new_star, new_kwarg) + res = self.assign(new_call) + res_expl = self.explanation_param(self.display(res)) + outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl) + return res, outer_expl + + # ast.Call signature changed on 3.5, + # conditionally change which methods is named + # visit_Call depending on Python version + if sys.version_info >= (3, 5): + visit_Call = visit_Call_35 + else: + visit_Call = visit_Call_legacy + + def visit_Attribute(self, attr): + if not isinstance(attr.ctx, ast.Load): + return self.generic_visit(attr) + value, value_expl = self.visit(attr.value) + res = self.assign(ast.Attribute(value, attr.attr, ast.Load())) + res_expl = self.explanation_param(self.display(res)) + pat = "%s\n{%s = %s.%s\n}" + expl = pat % (res_expl, res_expl, value_expl, attr.attr) + return res, expl + + def visit_Compare(self, comp): + self.push_format_context() + left_res, left_expl = self.visit(comp.left) + if isinstance(comp.left, (ast.Compare, ast.BoolOp)): + left_expl = "({0})".format(left_expl) + res_variables = [self.variable() for i in range(len(comp.ops))] + load_names = [ast.Name(v, ast.Load()) for v in res_variables] + store_names = [ast.Name(v, ast.Store()) for v in res_variables] + it = zip(range(len(comp.ops)), comp.ops, comp.comparators) + expls = [] + syms = [] + results = [left_res] + for i, op, next_operand in it: + next_res, next_expl = self.visit(next_operand) + if isinstance(next_operand, (ast.Compare, ast.BoolOp)): + next_expl = "({0})".format(next_expl) + results.append(next_res) + sym = binop_map[op.__class__] + syms.append(ast.Str(sym)) + expl = "%s %s %s" % (left_expl, sym, next_expl) + expls.append(ast.Str(expl)) + res_expr = ast.Compare(left_res, [op], [next_res]) + self.statements.append(ast.Assign([store_names[i]], res_expr)) + left_res, left_expl = next_res, next_expl + # Use pytest.assertion.util._reprcompare if that's available. + expl_call = self.helper("call_reprcompare", + ast.Tuple(syms, ast.Load()), + ast.Tuple(load_names, ast.Load()), + ast.Tuple(expls, ast.Load()), + ast.Tuple(results, ast.Load())) + if len(comp.ops) > 1: + res = ast.BoolOp(ast.And(), load_names) + else: + res = load_names[0] + return res, self.explanation_param(self.pop_format_context(expl_call)) diff --git a/pypy/tool/pytest/ast-rewriter/ast_util.py b/pypy/tool/pytest/ast-rewriter/ast_util.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/ast-rewriter/ast_util.py @@ -0,0 +1,289 @@ +"""Utilities for assertion debugging""" +from __future__ import absolute_import, division, print_function + +u = str + +# The _reprcompare attribute on the util module is used by the new assertion +# interpretation code and assertion rewriter to detect this plugin was +# loaded and in turn call the hooks defined here as part of the +# DebugInterpreter. +_reprcompare = None + + +def format_explanation(explanation): + """This formats an explanation + + Normally all embedded newlines are escaped, however there are + three exceptions: \n{, \n} and \n~. The first two are intended + cover nested explanations, see function and attribute explanations + for examples (.visit_Call(), visit_Attribute()). The last one is + for when one explanation needs to span multiple lines, e.g. when + displaying diffs. + """ + lines = _split_explanation(explanation) + result = _format_lines(lines) + return '\n'.join(result) + + +def _split_explanation(explanation): + """Return a list of individual lines in the explanation + + This will return a list of lines split on '\n{', '\n}' and '\n~'. + Any other newlines will be escaped and appear in the line as the + literal '\n' characters. + """ + raw_lines = (explanation or '').split('\n') + lines = [raw_lines[0]] + for values in raw_lines[1:]: + if values and values[0] in ['{', '}', '~', '>']: + lines.append(values) + else: + lines[-1] += '\\n' + values + return lines + + +def _format_lines(lines): + """Format the individual lines + + This will replace the '{', '}' and '~' characters of our mini + formatting language with the proper 'where ...', 'and ...' and ' + + ...' text, taking care of indentation along the way. + + Return a list of formatted lines. + """ + result = lines[:1] + stack = [0] + stackcnt = [0] + for line in lines[1:]: + if line.startswith('{'): + if stackcnt[-1]: + s = 'and ' + else: + s = 'where ' + stack.append(len(result)) + stackcnt[-1] += 1 + stackcnt.append(0) + result.append(' +' + ' ' * (len(stack) - 1) + s + line[1:]) + elif line.startswith('}'): + stack.pop() + stackcnt.pop() + result[stack[-1]] += line[1:] + else: + assert line[0] in ['~', '>'] + stack[-1] += 1 + indent = len(stack) if line.startswith('~') else len(stack) - 1 + result.append(' ' * indent + line[1:]) + assert len(stack) == 1 + return result + + +# Provide basestring in python3 +try: + basestring = basestring +except NameError: + basestring = str + + +def saferepr(obj, maxsize=None): + s = repr(obj) + if maxsize is not None: + s = s[:maxsize] + return s + + +def assertrepr_compare(op, left, right, verbose=False): + """Return specialised explanations for some operators/operands""" + width = 80 - 15 - len(op) - 2 # 15 chars indentation, 1 space around op + left_repr = saferepr(left, maxsize=int(width // 2)) + right_repr = saferepr(right, maxsize=width - len(left_repr)) + + summary = u('%s %s %s') % (left_repr, op, right_repr) + + def issequence(x): + return hasattr(x, '__iter__') and not isinstance(x, basestring) + + def istext(x): + return isinstance(x, basestring) + + def isdict(x): + return isinstance(x, dict) + + def isset(x): + return isinstance(x, (set, frozenset)) + + def isiterable(obj): + try: + iter(obj) + return not istext(obj) + except TypeError: + return False + + explanation = None + try: + if op == '==': + if istext(left) and istext(right): + explanation = _diff_text(left, right, verbose) + else: + if issequence(left) and issequence(right): + explanation = _compare_eq_sequence(left, right, verbose) + elif isset(left) and isset(right): + explanation = _compare_eq_set(left, right, verbose) + elif isdict(left) and isdict(right): + explanation = _compare_eq_dict(left, right, verbose) + if isiterable(left) and isiterable(right): + expl = _compare_eq_iterable(left, right, verbose) + if explanation is not None: + explanation.extend(expl) + else: + explanation = expl + elif op == 'not in': + if istext(left) and istext(right): + explanation = _notin_text(left, right, verbose) + except Exception as err: + explanation = [ + '(pytest assertion: representation of details failed. ' + 'Probably an object has a faulty __repr__.)', str(err)] + + if not explanation: + return None + + return [summary] + explanation + + +def _diff_text(left, right, verbose=False): + """Return the explanation for the diff between text or bytes + + Unless --verbose is used this will skip leading and trailing + characters which are identical to keep the diff minimal. + + If the input are bytes they will be safely converted to text. + """ + from difflib import ndiff + explanation = [] + if not verbose: + i = 0 # just in case left or right has zero length + for i in range(min(len(left), len(right))): + if left[i] != right[i]: + break + if i > 42: + i -= 10 # Provide some context + explanation = ['Skipping %s identical leading ' + 'characters in diff, use -v to show' % i] + left = left[i:] + right = right[i:] + if len(left) == len(right): + for i in range(len(left)): + if left[-i] != right[-i]: + break + if i > 42: + i -= 10 # Provide some context + explanation += ['Skipping %s identical trailing ' + 'characters in diff, use -v to show' % i] + left = left[:-i] + right = right[:-i] + keepends = True + explanation += [line.strip('\n') + for line in ndiff(left.splitlines(keepends), + right.splitlines(keepends))] + return explanation + + +def _compare_eq_iterable(left, right, verbose=False): + if not verbose: + return ['Use -v to get the full diff'] + # dynamic import to speedup pytest + import difflib, pprint + try: + left_formatting = pprint.pformat(left).splitlines() + right_formatting = pprint.pformat(right).splitlines() + explanation = ['Full diff:'] + except Exception: + # hack: PrettyPrinter.pformat() in python 2 fails when formatting items that can't be sorted(), ie, calling + # sorted() on a list would raise. See issue #718. + # As a workaround, the full diff is generated by using the repr() string of each item of each container. + left_formatting = sorted(repr(x) for x in left) + right_formatting = sorted(repr(x) for x in right) + explanation = ['Full diff (fallback to calling repr on each item):'] + explanation.extend(line.strip() for line in difflib.ndiff(left_formatting, right_formatting)) + return explanation + + +def _compare_eq_sequence(left, right, verbose=False): + explanation = [] + for i in range(min(len(left), len(right))): + if left[i] != right[i]: + explanation += [u('At index %s diff: %r != %r') + % (i, left[i], right[i])] + break + if len(left) > len(right): + explanation += [u('Left contains more items, first extra item: %s') + % saferepr(left[len(right)],)] + elif len(left) < len(right): + explanation += [ + u('Right contains more items, first extra item: %s') % + saferepr(right[len(left)],)] + return explanation + + +def _compare_eq_set(left, right, verbose=False): + explanation = [] + diff_left = left - right + diff_right = right - left + if diff_left: + explanation.append(u('Extra items in the left set:')) + for item in diff_left: + explanation.append(saferepr(item)) + if diff_right: + explanation.append(u('Extra items in the right set:')) + for item in diff_right: + explanation.append(saferepr(item)) + return explanation + + +def _compare_eq_dict(left, right, verbose=False): + import pprint + explanation = [] + common = set(left).intersection(set(right)) + same = dict((k, left[k]) for k in common if left[k] == right[k]) + if same and verbose < 2: + explanation += [u('Omitting %s identical items, use -vv to show') % + len(same)] + elif same: + explanation += [u('Common items:')] + explanation += pprint.pformat(same).splitlines() + diff = set(k for k in common if left[k] != right[k]) + if diff: + explanation += [u('Differing items:')] + for k in diff: + explanation += [saferepr({k: left[k]}) + ' != ' + + saferepr({k: right[k]})] + extra_left = set(left) - set(right) + if extra_left: + explanation.append(u('Left contains more items:')) + explanation.extend(pprint.pformat( + dict((k, left[k]) for k in extra_left)).splitlines()) + extra_right = set(right) - set(left) + if extra_right: + explanation.append(u('Right contains more items:')) + explanation.extend(pprint.pformat( + dict((k, right[k]) for k in extra_right)).splitlines()) + return explanation + + +def _notin_text(term, text, verbose=False): + index = text.find(term) + head = text[:index] + tail = text[index + len(term):] + correct_text = head + tail + diff = _diff_text(correct_text, text, verbose) + newdiff = [u('%s is contained here:') % saferepr(term, maxsize=42)] + for line in diff: + if line.startswith(u('Skipping')): + continue + if line.startswith(u('- ')): + continue + if line.startswith(u('+ ')): + newdiff.append(u(' ') + line[2:]) + else: + newdiff.append(line) + return newdiff diff --git a/pypy/tool/pytest/fake_pytest/__init__.py b/pypy/tool/pytest/fake_pytest/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/fake_pytest/__init__.py @@ -0,0 +1,12 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + applevel_name = 'pytest' + interpleveldefs = { + 'raises': 'interp_pytest.pypyraises', + 'skip': 'interp_pytest.pypyskip', + 'fixture': 'interp_pytest.fake_fixture', + } + appleveldefs = { + 'importorskip': 'app_pytest.importorskip', + } diff --git a/pypy/tool/pytest/fake_pytest/app_pytest.py b/pypy/tool/pytest/fake_pytest/app_pytest.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/fake_pytest/app_pytest.py @@ -0,0 +1,8 @@ +import pytest + + +def importorskip(name): + try: + return __import__(name) + except ImportError: + pytest.skip('Module %s not available' % name) diff --git a/pypy/tool/pytest/fake_pytest/interp_pytest.py b/pypy/tool/pytest/fake_pytest/interp_pytest.py new file mode 100644 --- /dev/null +++ b/pypy/tool/pytest/fake_pytest/interp_pytest.py @@ -0,0 +1,4 @@ +from pypy.tool.pytest.appsupport import pypyraises, pypyskip + +def fake_fixture(space, w_arg): + return w_arg From pypy.commits at gmail.com Fri Mar 23 11:04:57 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Mar 2018 08:04:57 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: (remi, arigo) Message-ID: <5ab51799.c3841c0a.aa3be.1b5d@mx.google.com> Author: Armin Rigo Branch: guard-compatible Changeset: r94122:bfc198924823 Date: 2018-03-23 16:04 +0100 http://bitbucket.org/pypy/pypy/changeset/bfc198924823/ Log: (remi, arigo) Found one of the remaining obscuuuuuuuuure bugs: r11 is supposed to have a known value, but can't across a guard_compatible diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1857,7 +1857,9 @@ self.guard_success_cc = rx86.Conditions['E'] self.implement_guard(guard_token) - genop_guard_guard_compatible = genop_guard_guard_value + def genop_guard_guard_compatible(self, guard_op, guard_token, locs, ign): + self.genop_guard_guard_value(guard_op, guard_token, locs, ign) + self.mc.forget_scratch_register() def _cmp_guard_class(self, locs): loc_ptr = locs[0] From pypy.commits at gmail.com Fri Mar 23 11:04:55 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Mar 2018 08:04:55 -0700 (PDT) Subject: [pypy-commit] pypy guard-compatible: Don't use NULL, it doesn't always work here Message-ID: <5ab51797.6291df0a.66f67.148b@mx.google.com> Author: Armin Rigo Branch: guard-compatible Changeset: r94121:ff9206ab5e19 Date: 2018-03-23 16:02 +0100 http://bitbucket.org/pypy/pypy/changeset/ff9206ab5e19/ Log: Don't use NULL, it doesn't always work here diff --git a/rpython/jit/backend/x86/vtune.py b/rpython/jit/backend/x86/vtune.py --- a/rpython/jit/backend/x86/vtune.py +++ b/rpython/jit/backend/x86/vtune.py @@ -19,9 +19,9 @@ RPY_EXTERN void rpy_make_dlopen_strong(char *funcname, Signed addr, Signed size) { // make *really* sure that dlopen&Co are linked so that vtune is happy - dlopen(NULL, 0); - dlsym(NULL, NULL); - dlclose(NULL); + dlopen((void *)0, 0); + dlsym((void *)0, (void *)0); + dlclose((void *)0); } RPY_EXTERN void rpy_vtune_register(char *funcname, Signed addr, Signed size) From pypy.commits at gmail.com Fri Mar 23 11:53:19 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 08:53:19 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: (anto, arigo): write down an idea which we had to improve the performance of PyObject* allocations; this should help a lot in case most PyObject* die young Message-ID: <5ab522ef.abaddf0a.39bcb.c2d2@mx.google.com> Author: Antonio Cuni Branch: extradoc Changeset: r5884:dfbdfb18cb22 Date: 2018-03-23 16:53 +0100 http://bitbucket.org/pypy/extradoc/changeset/dfbdfb18cb22/ Log: (anto, arigo): write down an idea which we had to improve the performance of PyObject* allocations; this should help a lot in case most PyObject* die young diff --git a/planning/cpyext.txt b/planning/cpyext.txt --- a/planning/cpyext.txt +++ b/planning/cpyext.txt @@ -22,3 +22,46 @@ - add JIT support to virtualize the pypy side placeholders of PyObject*: this way, if a PyObject* is converted to W_Root only for the lifetime of the loop, we can avoid the cost entirely + + +Improving the performance of PyObject* -> W_Root conversion +------------------------------------------------------------ + +Currently, if you are in a loop, create lots of PyObject* in C and pass them +to pypy (by calling from_ref), it is very slow. For example, look at +allocate_int and allocate_tuple in antocuni/cpyext-benchmarks. + +This happens because: + +1. we have to keep track of the W_Root->PyObject link inside the minor +collection; this is currently done by putting them in a temporary dict, which +is then "merged" with the big one when a PyObject survives + +2. we have to walk over all the allocated PyObject* which died young, to call tp_dealloc + +The following is a rough proposal to improve both points: + +1. we implement a way to have "extra fields" on objects which are in the + nursery (see later for details) + +2. for each W_Root which has a corresponding PyObject, we add two fields: + + w_obj.pyobj: this maintain the link between W_Root and PyObject while we + are in the nursery; later, when we the object survives, the + link is maintained "as usual" by putting w_obj in a special + dict + + w_obj.w_next: this is used to implement a chained list of "w_obj which have + a pyobj": this way it is very fast to iterate over all of + them during the minor collection + +Alternative: use an AddressStack to maintain the list of PyObject*, instead of +using the chained list using w_next + +How to implement extra fields? + +The simplest way is to have a "parallel nursery": if w_obj is at offset X from +the start of the nursery, its extra fields will start at the same offset in +the parallel nursery. The only requirement is that each W_Root in the nursery +is at least two words, and we need to think about how to guarantee it. + From pypy.commits at gmail.com Fri Mar 23 11:53:54 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 08:53:54 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: translation fix: we need pass a non-resizable list because of immutable_fields[*] Message-ID: <5ab52312.38acdf0a.2f900.495f@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94123:8375d03c4b5a Date: 2018-03-23 15:06 +0000 http://bitbucket.org/pypy/pypy/changeset/8375d03c4b5a/ Log: translation fix: we need pass a non-resizable list because of immutable_fields[*] diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -340,7 +340,7 @@ continue assert issubclass(wrapper_class, W_PyCWrapperObject) - w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset) + w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset[:]) dict_w[method_name] = w_obj if pto.c_tp_doc: dict_w['__doc__'] = space.newtext( From pypy.commits at gmail.com Fri Mar 23 12:08:59 2018 From: pypy.commits at gmail.com (arigo) Date: Fri, 23 Mar 2018 09:08:59 -0700 (PDT) Subject: [pypy-commit] extradoc extradoc: (anto, arigo) thinking Message-ID: <5ab5269b.82151c0a.4164a.1ee3@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r5885:ed01dff131fb Date: 2018-03-23 17:08 +0100 http://bitbucket.org/pypy/extradoc/changeset/ed01dff131fb/ Log: (anto, arigo) thinking diff --git a/planning/cpyext.txt b/planning/cpyext.txt --- a/planning/cpyext.txt +++ b/planning/cpyext.txt @@ -65,3 +65,13 @@ the parallel nursery. The only requirement is that each W_Root in the nursery is at least two words, and we need to think about how to guarantee it. + +To think about: +~~~~~~~~~~~~~~~ + +Maybe we can tweak the nursery collection by adding a GCFLAG_WITH_PYOBJ flag +and when we trace such an object, we know it has got a pyobj and we set a +flag "survived!" into the pyobj. Then the walking of all pyobjs-with-young- +associated-pypy-obj can be faster. It is faster in particular because then +we don't need to bring into the cache anything about the pypy objects +corresponding to pyobjs, if they die. From pypy.commits at gmail.com Fri Mar 23 12:44:52 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 09:44:52 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: now that offset is immutable, we can safely unroll this, I think Message-ID: <5ab52f04.c1241c0a.55b6.4bea@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94124:b28a895c1dca Date: 2018-03-23 17:44 +0100 http://bitbucket.org/pypy/pypy/changeset/b28a895c1dca/ Log: now that offset is immutable, we can safely unroll this, I think diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -240,6 +240,7 @@ def call(self, space, w_self, __args__): raise NotImplementedError + @jit.unroll_safe def get_func_to_call(self): func_to_call = self.func if self.offset: From pypy.commits at gmail.com Fri Mar 23 12:48:37 2018 From: pypy.commits at gmail.com (antocuni) Date: Fri, 23 Mar 2018 09:48:37 -0700 (PDT) Subject: [pypy-commit] pypy rawrefcount-free-early: close this branch; it turns out that we already free the objects early, this just add a second redundant call Message-ID: <5ab52fe5.8d6d1c0a.790ed.a3ac@mx.google.com> Author: Antonio Cuni Branch: rawrefcount-free-early Changeset: r94125:17c6fa64f211 Date: 2018-03-23 17:48 +0100 http://bitbucket.org/pypy/pypy/changeset/17c6fa64f211/ Log: close this branch; it turns out that we already free the objects early, this just add a second redundant call From pypy.commits at gmail.com Sat Mar 24 06:06:13 2018 From: pypy.commits at gmail.com (antocuni) Date: Sat, 24 Mar 2018 03:06:13 -0700 (PDT) Subject: [pypy-commit] pypy default: Merge the first part of the cpyext-fast-typecheck branch. Message-ID: <5ab62315.0481df0a.8a070.6b2f@mx.google.com> Author: Antonio Cuni Branch: Changeset: r94126:66208269e66e Date: 2018-03-24 10:47 +0100 http://bitbucket.org/pypy/pypy/changeset/66208269e66e/ Log: Merge the first part of the cpyext-fast-typecheck branch. This branch introduce a way to implement Py*_Check efficiently as C macros which simply checks a flag on the C typeobject, instead of doing a full roundtrip to RPython land to do a space.isinstance_w(). For now, implement PyFloat_Check, PyBool_Check and PySlice_Check using the new technique. Eventually, we should do the same for all type checkers and possibly kill build_type_checkers(). This commits merges only part of the branch because later commits have diverged from the original scope and implements a different feature (bad me); they will be merged in a separate commit, to keep the diff clear. diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -133,6 +133,11 @@ 'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3 constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name) +# PyPy-specific flags +for name in ('FLOAT',): + constant_names.append('Py_TPPYPYFLAGS_%s_SUBCLASS' % name) + + for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) globals().update(rffi_platform.configure(CConfig_constants)) diff --git a/pypy/module/cpyext/boolobject.py b/pypy/module/cpyext/boolobject.py --- a/pypy/module/cpyext/boolobject.py +++ b/pypy/module/cpyext/boolobject.py @@ -1,9 +1,5 @@ -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, - build_type_checkers) - -# Inheriting from bool isn't actually possible. -PyBool_Check = build_type_checkers("Bool")[1] +from rpython.rtyper.lltypesystem import rffi +from pypy.module.cpyext.api import cpython_api, PyObject @cpython_api([rffi.LONG], PyObject) def PyBool_FromLong(space, value): diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py --- a/pypy/module/cpyext/floatobject.py +++ b/pypy/module/cpyext/floatobject.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (PyObjectFields, bootstrap_function, cpython_struct, - CANNOT_FAIL, cpython_api, PyObject, build_type_checkers, CONST_STRING) + CANNOT_FAIL, cpython_api, PyObject, CONST_STRING) from pypy.module.cpyext.pyobject import ( make_typedescr, track_reference, from_ref) from pypy.interpreter.error import OperationError @@ -38,8 +38,6 @@ track_reference(space, obj, w_obj) return w_obj -PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") - @cpython_api([lltype.Float], PyObject) def PyFloat_FromDouble(space, value): return space.newfloat(value) diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -16,6 +16,8 @@ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h --- a/pypy/module/cpyext/include/floatobject.h +++ b/pypy/module/cpyext/include/floatobject.h @@ -32,6 +32,11 @@ return PyFloat_FromDouble(-Py_HUGE_VAL); \ } while(0) +#define PyFloat_Check(op) \ + _PyPy_Type_FastSubclass((op)->ob_type, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) +#define PyFloat_CheckExact(op) ((op)->ob_type == &PyFloat_Type) + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -228,6 +228,11 @@ #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1L<<30) #define Py_TPFLAGS_TYPE_SUBCLASS (1L<<31) +/* These are conceptually the same as the flags above, but they are + PyPy-specific and are stored inside tp_pypy_flags */ +#define Py_TPPYPYFLAGS_FLOAT_SUBCLASS (1L<<0) + + #define Py_TPFLAGS_DEFAULT_EXTERNAL ( \ Py_TPFLAGS_HAVE_GETCHARBUFFER | \ Py_TPFLAGS_HAVE_SEQUENCE_IN | \ @@ -247,6 +252,8 @@ #define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) #define PyType_FastSubclass(t,f) PyType_HasFeature(t,f) +#define _PyPy_Type_FastSubclass(t,f) (((t)->tp_pypy_flags & (f)) != 0) + #define PyType_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) #define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type) diff --git a/pypy/module/cpyext/include/sliceobject.h b/pypy/module/cpyext/include/sliceobject.h --- a/pypy/module/cpyext/include/sliceobject.h +++ b/pypy/module/cpyext/include/sliceobject.h @@ -17,6 +17,8 @@ PyObject *step; } PySliceObject; +#define PySlice_Check(op) ((op)->ob_type == &PySlice_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -311,6 +311,10 @@ /* Type attribute cache version tag. Added in version 2.6 */ unsigned int tp_version_tag; + /* PyPy specific extra fields: make sure that they are ALWAYS at the end, + for compatibility with CPython */ + long tp_pypy_flags; + } PyTypeObject; typedef struct _heaptypeobject { diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -47,7 +47,6 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) -PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @cpython_api([PyObject, PyObject, PyObject], PyObject) def PySlice_New(space, w_start, w_stop, w_step): @@ -75,9 +74,8 @@ normal slices. Returns 0 on success and -1 on error with exception set.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \ w_slice.indices4(space, length) return 0 @@ -97,9 +95,8 @@ objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, in the source of your extension.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0] = \ w_slice.indices3(space, length) return 0 diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py --- a/pypy/module/cpyext/test/test_boolobject.py +++ b/pypy/module/cpyext/test/test_boolobject.py @@ -1,7 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong -from pypy.module.cpyext.floatobject import PyFloat_FromDouble +from pypy.module.cpyext.boolobject import PyBool_FromLong class TestBoolObject(BaseApiTest): def test_fromlong(self, space): @@ -12,12 +11,6 @@ else: assert obj is space.w_False - def test_check(self, space): - assert PyBool_Check(space, space.w_True) - assert PyBool_Check(space, space.w_False) - assert not PyBool_Check(space, space.w_None) - assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0)) - class AppTestBoolMacros(AppTestCpythonExtensionBase): def test_macros(self): module = self.import_extension('foo', [ @@ -42,4 +35,14 @@ assert module.to_int(False) == 0 assert module.to_int(True) == 1 - + def test_check(self): + module = self.import_extension('foo', [ + ("type_check", "METH_O", + ''' + return PyLong_FromLong(PyBool_Check(args)); + ''')]) + assert module.type_check(True) + assert module.type_check(False) + assert not module.type_check(None) + assert not module.type_check(1.0) + diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -102,9 +102,11 @@ """ PyObject* pyobj = PyFloat_FromDouble(1.0); PyFloatObject* pfo = (PyFloatObject*)pyobj; - int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) && - PyFloat_Check(pfo) && PyFloat_CheckExact(pfo); + int res = (PyFloat_Check(pyobj) + + PyFloat_CheckExact(pyobj) * 10 + + PyFloat_Check(pfo) * 100 + + PyFloat_CheckExact(pfo) * 1000); Py_DecRef(pyobj); return PyLong_FromLong(res);"""), ]) - assert module.test() == 1 + assert module.test() == 1111 diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py --- a/pypy/module/cpyext/test/test_number.py +++ b/pypy/module/cpyext/test/test_number.py @@ -11,7 +11,6 @@ PyNumber_Index, PyNumber_Coerce, PyNumber_CoerceEx, PyNumber_Add, PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute, PyNumber_Power, PyNumber_InPlacePower) -from pypy.module.cpyext.floatobject import PyFloat_Check from pypy.module.cpyext.intobject import PyInt_CheckExact from pypy.module.cpyext.longobject import PyLong_CheckExact from pypy.module.cpyext.object import PyObject_Size @@ -86,7 +85,7 @@ w_res = from_ref(space, ppl[0]) - assert PyFloat_Check(space, w_res) + assert space.isinstance_w(w_res, space.w_float) assert space.unwrap(w_res) == 123. decref(space, pl) decref(space, pf) diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py --- a/pypy/module/cpyext/test/test_sliceobject.py +++ b/pypy/module/cpyext/test/test_sliceobject.py @@ -2,14 +2,8 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP -from pypy.module.cpyext.sliceobject import PySlice_Check class TestSliceObject(BaseApiTest): - def test_slice(self, space): - w_i = space.wrap(10) - w_slice = space.newslice(w_i, w_i, w_i) - assert PySlice_Check(space, w_slice) - assert not PySlice_Check(space, w_i) def test_GetIndicesEx(self, space, api): w = space.wrap @@ -79,3 +73,14 @@ """), ]) assert module.get_ellipsis() is Ellipsis + + def test_typecheck(self): + module = self.import_extension('foo', [ + ("check", "METH_O", + """ + PySliceObject *slice = (PySliceObject *)args; + return PyLong_FromLong(PySlice_Check(slice)); + """), + ]) + s = slice(10, 20, 30) + assert module.check(s) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1537,4 +1537,29 @@ pass assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0 + def test_has_pypy_subclass_flag(self): + module = self.import_extension('foo', [ + ("test_pypy_flags", "METH_VARARGS", + ''' + long long in_flag, my_flag; + PyObject * obj; + if (!PyArg_ParseTuple(args, "OL", &obj, &in_flag)) + return NULL; + if (!PyType_Check(obj)) + { + PyErr_SetString(PyExc_ValueError, "input must be type"); + return NULL; + } + my_flag = ((PyTypeObject*)obj)->tp_pypy_flags; + if ((my_flag & in_flag) != in_flag) + return PyLong_FromLong(-1); + return PyLong_FromLong(0); + '''),]) + # copied from object.h + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + class MyFloat(float): + pass + assert module.test_pypy_flags(float, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + assert module.test_pypy_flags(MyFloat, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -22,6 +22,7 @@ Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS, Py_TPFLAGS_TYPE_SUBCLASS, Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3 + Py_TPPYPYFLAGS_FLOAT_SUBCLASS, ) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, @@ -426,6 +427,9 @@ pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS elif space.issubtype_w(w_obj, space.w_dict): pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS + # the following types are a pypy-specific extensions, using tp_pypy_flags + elif space.issubtype_w(w_obj, space.w_float): + pto.c_tp_pypy_flags |= Py_TPPYPYFLAGS_FLOAT_SUBCLASS def check_descr(space, w_self, w_type): if not space.isinstance_w(w_self, w_type): From pypy.commits at gmail.com Sat Mar 24 06:06:15 2018 From: pypy.commits at gmail.com (antocuni) Date: Sat, 24 Mar 2018 03:06:15 -0700 (PDT) Subject: [pypy-commit] pypy cpyext-fast-typecheck: close branch to be merged Message-ID: <5ab62317.4d051c0a.9623a.9515@mx.google.com> Author: Antonio Cuni Branch: cpyext-fast-typecheck Changeset: r94127:31f70e250480 Date: 2018-03-24 10:47 +0100 http://bitbucket.org/pypy/pypy/changeset/31f70e250480/ Log: close branch to be merged From pypy.commits at gmail.com Sat Mar 24 06:06:17 2018 From: pypy.commits at gmail.com (antocuni) Date: Sat, 24 Mar 2018 03:06:17 -0700 (PDT) Subject: [pypy-commit] pypy default: Merge the second part of the cpyext-fast-typecheck branch. Despite the name, Message-ID: <5ab62319.85b81c0a.adf35.bb64@mx.google.com> Author: Antonio Cuni Branch: Changeset: r94128:f902cda5d7ea Date: 2018-03-24 11:05 +0100 http://bitbucket.org/pypy/pypy/changeset/f902cda5d7ea/ Log: Merge the second part of the cpyext-fast-typecheck branch. Despite the name, this implements a very different feature :) This heavily refactors and simplify W_PyCWrapperObject, which is used to call all C slots from Python. Instead of taking a generic callback to call, we create a specialized subclass for each kind of slot. In particular, this lets us to avoid creating a full tuple (and possibly a dict) to contain the wrapped arguments. The end result is a hugh speedup in some of the antocuni/cpyext- benchmarks microbenchmarks; in particular, compared to default: - len(Foo()) is 9x faster - Foo()[0] is 5.7x faster - np.__getitem__ is ~50% faster - np.mean is ~20% faster diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -45,6 +45,18 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) +def w_kwargs_from_args(space, __args__): + w_kwargs = None + if __args__.keywords: + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) + return w_kwargs + class W_PyCFunctionObject(W_Root): _immutable_fields_ = ["flags"] @@ -103,15 +115,7 @@ def call_keywords(self, space, w_self, __args__): func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) py_args = tuple_from_args_w(space, __args__.arguments_w) - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + w_kwargs = w_kwargs_from_args(space, __args__) try: return generic_cpy_call(space, func, w_self, py_args, w_kwargs) finally: @@ -213,14 +217,15 @@ (self.name, self.w_objclass.getname(self.space))) +class W_PyCWrapperObject(W_Root): + """ + Abstract class; for concrete subclasses, see slotdefs.py + """ + _immutable_fields_ = ['offset[*]'] -class W_PyCWrapperObject(W_Root): - def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func, offset=None): + def __init__(self, space, pto, method_name, doc, func, offset): self.space = space self.method_name = method_name - self.wrapper_func = wrapper_func - self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func self.offset = offset @@ -229,10 +234,17 @@ assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type - def call(self, space, w_self, w_args, w_kw): + def descr_call(self, space, w_self, __args__): + return self.call(space, w_self, __args__) + + def call(self, space, w_self, __args__): + raise NotImplementedError + + @jit.unroll_safe + def get_func_to_call(self): func_to_call = self.func if self.offset: - pto = as_pyobj(space, self.w_objclass) + pto = as_pyobj(self.space, self.w_objclass) # make ptr the equivalent of this, using the offsets #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) if pto: @@ -246,31 +258,33 @@ assert False, "failed to convert w_type %s to PyObject" % str( self.w_objclass) assert func_to_call - if self.wrapper_func is None: - assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, - w_kw) - if space.is_true(w_kw): - raise oefmt(space.w_TypeError, + return func_to_call + + def check_args(self, __args__, arity): + length = len(__args__.arguments_w) + if length != arity: + raise oefmt(self.space.w_TypeError, "expected %d arguments, got %d", + arity, length) + if __args__.keywords: + raise oefmt(self.space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - return self.wrapper_func(space, w_self, w_args, func_to_call) + + def check_argsv(self, __args__, min, max): + length = len(__args__.arguments_w) + if not min <= length <= max: + raise oefmt(self.space.w_TypeError, "expected %d-%d arguments, got %d", + min, max, length) + if __args__.keywords: + raise oefmt(self.space.w_TypeError, + "wrapper %s doesn't take any keyword arguments", + self.method_name) def descr_method_repr(self): return self.space.newtext("" % (self.method_name, self.w_objclass.name)) - at jit.dont_look_inside -def cwrapper_descr_call(space, w_self, __args__): - self = space.interp_w(W_PyCWrapperObject, w_self) - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w[1:]) - w_self = args_w[0] - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - return self.call(space, w_self, w_args, w_kw) def cmethod_descr_get(space, w_function, w_obj, w_cls=None): asking_for_bound = (space.is_none(w_cls) or @@ -323,7 +337,7 @@ W_PyCWrapperObject.typedef = TypeDef( 'wrapper_descriptor', - __call__ = interp2app(cwrapper_descr_call), + __call__ = interp2app(W_PyCWrapperObject.descr_call), __get__ = interp2app(cmethod_descr_get), __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject, wrapfn="newtext_or_none"), diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -19,6 +19,8 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext import userslot from pypy.module.cpyext.buffer import CBuffer, CPyBuffer, fq +from pypy.module.cpyext.methodobject import (W_PyCWrapperObject, tuple_from_args_w, + w_kwargs_from_args) from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable @@ -38,29 +40,6 @@ Py_GT = 4 Py_GE = 5 - -def check_num_args(space, w_ob, n): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if n == space.len_w(w_ob): - return - raise oefmt(space.w_TypeError, - "expected %d arguments, got %d", - n, space.len_w(w_ob)) - -def check_num_argsv(space, w_ob, low, high): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if low <=space.len_w(w_ob) <= high: - return - raise oefmt(space.w_TypeError, - "expected %d-%d arguments, got %d", - low, high, space.len_w(w_ob)) - @not_rpython def llslot(space, func): return func.api_func.get_llhelper(space) @@ -71,337 +50,413 @@ get_llhelper = v_func.value.api_func.get_llhelper return ctx.appcall(get_llhelper, v_space) +# NOTE: the following wrap_* are subclasses of W_PyCWrapperObject, even if +# they don't follow the usual W_* naming convention for subclasses of W_Root: +# we do this because we automatically generate most of the slots from the +# CPython code copy&pasted inside slotdefs_str, and thus we need to keep the +# same names as they are used in C. -def wrap_init(space, w_self, w_args, func, w_kwargs): - func_init = rffi.cast(initproc, func) - res = generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return None +class wrap_init(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_init = rffi.cast(initproc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return None -def wrap_unaryfunc(space, w_self, w_args, func): - func_unary = rffi.cast(unaryfunc, func) - check_num_args(space, w_args, 0) - return generic_cpy_call(space, func_unary, w_self) +class wrap_unaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_unary = rffi.cast(unaryfunc, func) + return generic_cpy_call(space, func_unary, w_self) -def wrap_binaryfunc(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_binary, w_self, args_w[0]) +class wrap_binaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_x = __args__.arguments_w[0] + return generic_cpy_call(space, func_binary, w_self, w_x) def _get_ob_type(space, w_obj): # please ensure that w_obj stays alive ob_type = as_pyobj(space, space.type(w_obj)) return rffi.cast(PyTypeObjectPtr, ob_type) -def wrap_binaryfunc_l(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - ob_type = _get_ob_type(space, w_self) - if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and - not space.issubtype_w(space.type(args_w[0]), space.type(w_self))): - return space.w_NotImplemented - return generic_cpy_call(space, func_binary, w_self, args_w[0]) +class wrap_binaryfunc_l(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_value = __args__.arguments_w[0] + ob_type = _get_ob_type(space, w_self) + if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.issubtype_w(space.type(w_value), space.type(w_self))): + return space.w_NotImplemented + return generic_cpy_call(space, func_binary, w_self, w_value) -def wrap_binaryfunc_r(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - ob_type = _get_ob_type(space, w_self) - if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and - not space.issubtype_w(space.type(args_w[0]), space.type(w_self))): - return space.w_NotImplemented - return generic_cpy_call(space, func_binary, args_w[0], w_self) +class wrap_binaryfunc_r(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_value = __args__.arguments_w[0] + ob_type = _get_ob_type(space, w_self) + if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.issubtype_w(space.type(w_value), space.type(w_self))): + return space.w_NotImplemented + return generic_cpy_call(space, func_binary, w_value, w_self) -def wrap_ternaryfunc(space, w_self, w_args, func): - # The third argument is optional - func_ternary = rffi.cast(ternaryfunc, func) - check_num_argsv(space, w_args, 1, 2) - args_w = space.fixedview(w_args) - arg3 = space.w_None - if len(args_w) > 1: - arg3 = args_w[1] - return generic_cpy_call(space, func_ternary, w_self, args_w[0], arg3) +class wrap_ternaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + # The third argument is optional + self.check_argsv(__args__, 1, 2) + func = self.get_func_to_call() + func_ternary = rffi.cast(ternaryfunc, func) + w_arg0 = __args__.arguments_w[0] + if len(__args__.arguments_w) == 2: + w_arg1 = __args__.arguments_w[1] + else: + w_arg1 = space.w_None + return generic_cpy_call(space, func_ternary, w_self, w_arg0, w_arg1) -def wrap_ternaryfunc_r(space, w_self, w_args, func): - # The third argument is optional - func_ternary = rffi.cast(ternaryfunc, func) - check_num_argsv(space, w_args, 1, 2) - args_w = space.fixedview(w_args) - ob_type = _get_ob_type(space, w_self) - if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and - not space.issubtype_w(space.type(args_w[0]), space.type(w_self))): - return space.w_NotImplemented - arg3 = space.w_None - if len(args_w) > 1: - arg3 = args_w[1] - return generic_cpy_call(space, func_ternary, args_w[0], w_self, arg3) +class wrap_ternaryfunc_r(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + # The third argument is optional + self.check_argsv(__args__, 1, 2) + func = self.get_func_to_call() + func_ternary = rffi.cast(ternaryfunc, func) + w_arg0 = __args__.arguments_w[0] + if len(__args__.arguments_w) == 2: + w_arg1 = __args__.arguments_w[1] + else: + w_arg1 = space.w_None + ob_type = _get_ob_type(space, w_self) + if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.issubtype_w(space.type(w_arg0), space.type(w_self))): + return space.w_NotImplemented + return generic_cpy_call(space, func_ternary, w_arg0, w_self, w_arg1) +class wrap_inquirypred(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_inquiry = rffi.cast(inquiry, func) + res = generic_cpy_call(space, func_inquiry, w_self) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) -def wrap_inquirypred(space, w_self, w_args, func): - func_inquiry = rffi.cast(inquiry, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_inquiry, w_self) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class wrap_getattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrfunc, func) + w_name = __args__.arguments_w[0] + name_ptr = rffi.str2charp(space.text_w(w_name)) + try: + return generic_cpy_call(space, func_target, w_self, name_ptr) + finally: + rffi.free_charp(name_ptr) -def wrap_getattr(space, w_self, w_args, func): - func_target = rffi.cast(getattrfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - name_ptr = rffi.str2charp(space.text_w(args_w[0])) - try: - return generic_cpy_call(space, func_target, w_self, name_ptr) - finally: - rffi.free_charp(name_ptr) +class wrap_getattro(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrofunc, func) + w_name = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, w_self, w_name) -def wrap_getattro(space, w_self, w_args, func): - func_target = rffi.cast(getattrofunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_target, w_self, args_w[0]) +class wrap_setattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_setattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 2) - w_name, w_value = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_delattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_delattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 1) - w_name, = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_descr_get(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(descrgetfunc, func) + length = len(__args__.arguments_w) + if length == 1: + w_obj = __args__.arguments_w[0] + w_type = None + elif length == 2: + w_obj = __args__.arguments_w[0] + w_type = __args__.arguments_w[1] + else: + raise oefmt(space.w_TypeError, + "expected 1 or 2 arguments, got %d", len(__args__.arguments_w)) + if w_obj is space.w_None: + w_obj = None + if w_type is space.w_None: + w_type = None + if w_obj is None and w_type is None: + raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") + return generic_cpy_call(space, func_target, w_self, w_obj, w_type) -def wrap_descr_get(space, w_self, w_args, func): - func_target = rffi.cast(descrgetfunc, func) - args_w = space.fixedview(w_args) - if len(args_w) == 1: - w_obj, = args_w - w_type = None - elif len(args_w) == 2: - w_obj, w_type = args_w - else: - raise oefmt(space.w_TypeError, - "expected 1 or 2 arguments, got %d", len(args_w)) - if w_obj is space.w_None: - w_obj = None - if w_type is space.w_None: - w_type = None - if w_obj is None and w_type is None: - raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") - return generic_cpy_call(space, func_target, w_self, w_obj, w_type) +class wrap_descr_set(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(descrsetfunc, func) + w_obj = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + res = generic_cpy_call(space, func_target, w_self, w_obj, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_descr_set(space, w_self, w_args, func): - func_target = rffi.cast(descrsetfunc, func) - check_num_args(space, w_args, 2) - w_obj, w_value = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_obj, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_descr_delete(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(descrsetfunc, func) + w_obj = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_obj, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_descr_delete(space, w_self, w_args, func): - func_target = rffi.cast(descrsetfunc, func) - check_num_args(space, w_args, 1) - w_obj, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_obj, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_call(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(ternaryfunc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) -def wrap_call(space, w_self, w_args, func, w_kwds): - func_target = rffi.cast(ternaryfunc, func) - return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) +class wrap_ssizessizeobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 3) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeobjargproc, func) + i = space.int_w(space.index(__args__.arguments_w[0])) + j = space.int_w(space.index(__args__.arguments_w[1])) + w_y = __args__.arguments_w[2] + res = generic_cpy_call(space, func_target, w_self, i, j, w_y) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_ssizessizeobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeobjargproc, func) - check_num_args(space, w_args, 3) - args_w = space.fixedview(w_args) - i = space.int_w(space.index(args_w[0])) - j = space.int_w(space.index(args_w[1])) - w_y = args_w[2] - res = generic_cpy_call(space, func_target, w_self, i, j, w_y) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_lenfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_len = rffi.cast(lenfunc, func) + res = generic_cpy_call(space, func_len, w_self) + if widen(res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) -def wrap_lenfunc(space, w_self, w_args, func): - func_len = rffi.cast(lenfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_len, w_self) - if widen(res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_sq_item(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeargfunc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + return generic_cpy_call(space, func_target, w_self, index) -def wrap_sq_item(space, w_self, w_args, func): - func_target = rffi.cast(ssizeargfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - return generic_cpy_call(space, func_target, w_self, index) +class wrap_sq_setitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + index = space.int_w(space.index(w_index)) + res = generic_cpy_call(space, func_target, w_self, index, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_sq_setitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - -def wrap_sq_delitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, index, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_sq_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, index, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) # Warning, confusing function name (like CPython). Used only for sq_contains. -def wrap_objobjproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjproc, func) - check_num_args(space, w_args, 1) - w_value, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_value) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class wrap_objobjproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjproc, func) + w_value = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_value) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) -def wrap_objobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 2) - w_key, w_value = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_key, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_objobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + res = generic_cpy_call(space, func_target, w_self, w_key, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None -def wrap_delitem(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 1) - w_key, = space.fixedview(w_args) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, w_key, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, w_key, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None -def wrap_ssizessizeargfunc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeargfunc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - start = space.int_w(args_w[0]) - end = space.int_w(args_w[1]) - return generic_cpy_call(space, func_target, w_self, start, end) +class wrap_ssizessizeargfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeargfunc, func) + start = space.int_w(__args__.arguments_w[0]) + end = space.int_w(__args__.arguments_w[1]) + return generic_cpy_call(space, func_target, w_self, start, end) -def wrap_next(space, w_self, w_args, func): - from pypy.module.cpyext.api import generic_cpy_call_expect_null - func_target = rffi.cast(iternextfunc, func) - check_num_args(space, w_args, 0) - w_res = generic_cpy_call_expect_null(space, func_target, w_self) - if not w_res and not PyErr_Occurred(space): - raise OperationError(space.w_StopIteration, space.w_None) - return w_res +class wrap_next(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + from pypy.module.cpyext.api import generic_cpy_call_expect_null + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(iternextfunc, func) + w_res = generic_cpy_call_expect_null(space, func_target, w_self) + if not w_res and not PyErr_Occurred(space): + raise OperationError(space.w_StopIteration, space.w_None) + return w_res -def wrap_hashfunc(space, w_self, w_args, func): - func_target = rffi.cast(hashfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_target, w_self) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_hashfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(hashfunc, func) + res = generic_cpy_call(space, func_target, w_self) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) -def wrap_getreadbuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getreadbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) -def wrap_getwritebuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getwritebuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) -def wrap_getbuffer(space, w_self, w_args, func): - func_target = rffi.cast(getbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(Py_buffer) as pybuf: - _flags = 0 - if space.len_w(w_args) > 0: - _flags = space.int_w(space.listview(w_args)[0]) - flags = rffi.cast(rffi.INT_real,_flags) - size = generic_cpy_call(space, func_target, w_self, pybuf, flags) - if widen(size) < 0: - space.fromcache(State).check_and_raise_exception(always=True) - ptr = pybuf.c_buf - size = pybuf.c_len - ndim = widen(pybuf.c_ndim) - shape = None - if pybuf.c_shape: - shape = [pybuf.c_shape[i] for i in range(ndim)] - strides = None - if pybuf.c_strides: - strides = [pybuf.c_strides[i] for i in range(ndim)] - if pybuf.c_format: - format = rffi.charp2str(pybuf.c_format) - else: - format = 'B' - # the CPython docs mandates that you do an incref whenever you call - # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't - # leak we release the buffer: - # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer - buf = CPyBuffer(space, ptr, size, w_self, format=format, - ndim=ndim, shape=shape, strides=strides, - itemsize=pybuf.c_itemsize, - readonly=widen(pybuf.c_readonly), - needs_decref=True, - releasebufferproc = rbp) - fq.register_finalizer(buf) - return buf.wrap(space) + +class wrap_getbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(getbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(Py_buffer) as pybuf: + _flags = 0 + if len(__args__.arguments_w) > 0: + _flags = space.int_w(__args__.arguments_w[0]) + flags = rffi.cast(rffi.INT_real,_flags) + size = generic_cpy_call(space, func_target, w_self, pybuf, flags) + if widen(size) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + ptr = pybuf.c_buf + size = pybuf.c_len + ndim = widen(pybuf.c_ndim) + shape = None + if pybuf.c_shape: + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None + if pybuf.c_strides: + strides = [pybuf.c_strides[i] for i in range(ndim)] + if pybuf.c_format: + format = rffi.charp2str(pybuf.c_format) + else: + format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer + buf = CPyBuffer(space, ptr, size, w_self, format=format, + ndim=ndim, shape=shape, strides=strides, + itemsize=pybuf.c_itemsize, + readonly=widen(pybuf.c_readonly), + needs_decref=True, + releasebufferproc = rbp) + fq.register_finalizer(buf) + return buf.wrap(space) def get_richcmp_func(OP_CONST): - def inner(space, w_self, w_args, func): - func_target = rffi.cast(richcmpfunc, func) - check_num_args(space, w_args, 1) - w_other, = space.fixedview(w_args) - return generic_cpy_call(space, func_target, - w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST)) - return inner + class wrap_richcmp(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(richcmpfunc, func) + w_other = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, + w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST)) + return wrap_richcmp richcmp_eq = get_richcmp_func(Py_EQ) richcmp_ne = get_richcmp_func(Py_NE) @@ -410,17 +465,19 @@ richcmp_gt = get_richcmp_func(Py_GT) richcmp_ge = get_richcmp_func(Py_GE) -def wrap_cmpfunc(space, w_self, w_args, func): - func_target = rffi.cast(cmpfunc, func) - check_num_args(space, w_args, 1) - w_other, = space.fixedview(w_args) +class wrap_cmpfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(cmpfunc, func) + w_other = __args__.arguments_w[0] - if not space.issubtype_w(space.type(w_self), space.type(w_other)): - raise oefmt(space.w_TypeError, - "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'", - w_self, w_self, w_other) + if not space.issubtype_w(space.type(w_self), space.type(w_other)): + raise oefmt(space.w_TypeError, + "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'", + w_self, w_self, w_other) - return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) + return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) SLOT_FACTORIES = {} def slot_factory(tp_name): @@ -804,9 +861,10 @@ missing_wrappers = ['wrap_indexargfunc', 'wrap_delslice', 'wrap_coercefunc'] for name in missing_wrappers: assert name not in globals() - def missing_wrapper(space, w_self, w_args, func): - print "cpyext: missing slot wrapper " + name - raise NotImplementedError("Slot wrapper " + name) + class missing_wrapper(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + print "cpyext: missing slot wrapper " + name + raise NotImplementedError("Slot wrapper " + name) missing_wrapper.__name__ = name globals()[name] = missing_wrapper @@ -836,13 +894,12 @@ PyWrapperFlag_KEYWORDS = 1 class TypeSlot: - def __init__(self, method_name, slot_name, function, wrapper1, wrapper2, doc): + def __init__(self, method_name, slot_name, function, wrapper, doc): self.method_name = method_name self.slot_name = slot_name self.slot_names = tuple(("c_" + slot_name).split(".")) self.slot_func = function - self.wrapper_func = wrapper1 - self.wrapper_func_kwds = wrapper2 + self.wrapper_class = wrapper self.doc = doc # adapted from typeobject.c @@ -863,13 +920,7 @@ function = getattr(userslot, FUNCTION or '!missing', None) assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS - if FLAGS: - wrapper1 = None - wrapper2 = wrapper - else: - wrapper1 = wrapper - wrapper2 = None - return TypeSlot(NAME, SLOT, function, wrapper1, wrapper2, DOC) + return TypeSlot(NAME, SLOT, function, wrapper, DOC) def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC): return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0) @@ -1158,7 +1209,7 @@ x.slot_func.api_func if x.slot_func else None) for x in slotdefs]) slotdefs_for_wrappers = unrolling_iterable( - [(x.method_name, x.slot_names, x.wrapper_func, x.wrapper_func_kwds, x.doc) + [(x.method_name, x.slot_names, x.wrapper_class, x.doc) for x in slotdefs]) if __name__ == "__main__": diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -636,7 +636,8 @@ Py_ssize_t refcnt_after; Py_INCREF(true_obj); Py_INCREF(true_obj); - PyBool_Check(true_obj); + if (!PyBool_Check(true_obj)) + Py_RETURN_NONE; refcnt_after = true_obj->ob_refcnt; Py_DECREF(true_obj); Py_DECREF(true_obj); diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -796,6 +796,36 @@ assert module.tp_init(list, x, ("hi",)) is None assert x == ["h", "i"] + def test_mp_subscript(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static PyObject* + mp_subscript(PyObject *self, PyObject *key) + { + return Py_BuildValue("i", 42); + } + PyMappingMethods tp_as_mapping; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init = ''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_mapping = &tp_as_mapping; + tp_as_mapping.mp_subscript = (binaryfunc)mp_subscript; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + assert obj[100] == 42 + raises(TypeError, "obj.__getitem__(100, 101)") + raises(TypeError, "obj.__getitem__(100, a=42)") + def test_mp_ass_subscript(self): module = self.import_extension('foo', [ ("new_obj", "METH_NOARGS", @@ -859,6 +889,84 @@ res = "foo" in obj assert res is True + def test_sq_ass_slice(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) + { + int expected = (a == 10 && b == 20 && + PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_slice = sq_ass_slice; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10:20] = 42 + raises(ValueError, "obj[10:20] = 43") + raises(ValueError, "obj[11:20] = 42") + raises(ValueError, "obj[10:21] = 42") + + def test_sq_ass_item(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) + { + int expected; + if (o == NULL) // delitem + expected = (i == 12); + else // setitem + expected = (i == 10 && PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_item = sq_ass_item; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10] = 42 + raises(ValueError, "obj[10] = 43") + raises(ValueError, "obj[11] = 42") + del obj[12] + raises(ValueError, "del obj[13]") + def test_tp_iter(self): module = self.import_extension('foo', [ ("tp_iter", "METH_VARARGS", diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -25,8 +25,8 @@ Py_TPPYPYFLAGS_FLOAT_SUBCLASS, ) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, - W_PyCMethodObject, W_PyCFunctionObject) + PyCFunction_NewEx, PyCFunction, PyMethodDef, + W_PyCMethodObject, W_PyCFunctionObject, W_PyCWrapperObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, get_typedescr, make_typedescr, @@ -311,7 +311,7 @@ def add_operators(space, dict_w, pto): from pypy.module.cpyext.object import PyObject_HashNotImplemented hash_not_impl = llslot(space, PyObject_HashNotImplemented) - for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: + for method_name, slot_names, wrapper_class, doc in slotdefs_for_wrappers: if method_name in dict_w: continue offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] @@ -336,10 +336,11 @@ func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue - if wrapper_func is None and wrapper_func_kwds is None: + if wrapper_class is None: continue - w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func_voidp, offset=offset) + + assert issubclass(wrapper_class, W_PyCWrapperObject) + w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset[:]) dict_w[method_name] = w_obj if pto.c_tp_doc: dict_w['__doc__'] = space.newtext( From pypy.commits at gmail.com Sat Mar 24 11:53:51 2018 From: pypy.commits at gmail.com (Raemi) Date: Sat, 24 Mar 2018 08:53:51 -0700 (PDT) Subject: [pypy-commit] pypy guard-value-limit: (arigo, remi) a branch to experiment with limiting the length of the chain of guard_values Message-ID: <5ab6748f.43b8df0a.8e8b0.82cb@mx.google.com> Author: Remi Meier Branch: guard-value-limit Changeset: r94129:c72f96e9c08b Date: 2018-03-24 15:22 +0100 http://bitbucket.org/pypy/pypy/changeset/c72f96e9c08b/ Log: (arigo,remi) a branch to experiment with limiting the length of the chain of guard_values quick hack to tell pyjitpl.opimpl_guard_value that it is the x-th in a chain of guard_values. Then simply do not promote the result and do not emit a guard_value. diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -862,12 +862,14 @@ class ResumeGuardDescr(AbstractResumeGuardDescr): _attrs_ = ('rd_numb', 'rd_consts', 'rd_virtuals', - 'rd_pendingfields', 'status') + 'rd_pendingfields', 'status', 'guard_value_counter') rd_numb = lltype.nullptr(NUMBERING) rd_consts = None rd_virtuals = None rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) + guard_value_counter = 0 + def copy_all_attributes_from(self, other): other = other.get_resumestorage() assert isinstance(other, ResumeGuardDescr) @@ -881,6 +883,17 @@ else: other.rd_vector_info = None + def compile_and_attach(self, metainterp, new_loop, orig_inputargs): + for op in new_loop.operations: + if op.is_guard(): + if op.getopnum() == rop.GUARD_VALUE: + descr = op.getdescr() + if isinstance(descr, ResumeGuardDescr): + descr.guard_value_counter = self.guard_value_counter + 1 + break + + AbstractResumeGuardDescr.compile_and_attach(self, metainterp, new_loop, orig_inputargs) + def store_final_boxes(self, guard_op, boxes, metainterp_sd): guard_op.setfailargs(boxes) self.store_hash(metainterp_sd) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1215,7 +1215,9 @@ @arguments("box", "orgpc") def _opimpl_guard_value(self, box, orgpc): - self.implement_guard_value(box, orgpc) + if self.metainterp.guard_value_counter <= 4: + return self.implement_guard_value(box, orgpc) + return box @arguments("box", "box", "descr", "orgpc") def opimpl_str_guard_value(self, box, funcbox, descr, orgpc): @@ -1551,6 +1553,7 @@ pc = self.pc op = ord(self.bytecode[pc]) staticdata.opcode_implementations[op](self, pc) + self.metainterp.guard_value_counter = 0 except ChangeFrame: pass @@ -2428,6 +2431,7 @@ # is also available as 'self.jitdriver_sd', because we need to # specialize this function and a few other ones for the '*args'. debug_start('jit-tracing') + self.guard_value_counter = 0 self.staticdata._setup_once() self.staticdata.profiler.start_tracing() assert jitdriver_sd is self.jitdriver_sd @@ -2460,6 +2464,7 @@ self.staticdata.profiler.start_tracing() key = resumedescr.get_resumestorage() assert isinstance(key, compile.ResumeGuardDescr) + self.guard_value_counter = key.guard_value_counter + 1 # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp self.resumekey_original_loop_token = resumedescr.rd_loop_token.loop_token_wref() From pypy.commits at gmail.com Sat Mar 24 11:53:53 2018 From: pypy.commits at gmail.com (Raemi) Date: Sat, 24 Mar 2018 08:53:53 -0700 (PDT) Subject: [pypy-commit] pypy guard-value-limit: (arigo, remi) make cut-off limit configurable Message-ID: <5ab67491.464a1c0a.239cd.95bf@mx.google.com> Author: Remi Meier Branch: guard-value-limit Changeset: r94130:3cb72c044ec2 Date: 2018-03-24 16:52 +0100 http://bitbucket.org/pypy/pypy/changeset/3cb72c044ec2/ Log: (arigo, remi) make cut-off limit configurable diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1215,9 +1215,13 @@ @arguments("box", "orgpc") def _opimpl_guard_value(self, box, orgpc): - if self.metainterp.guard_value_counter <= 4: - return self.implement_guard_value(box, orgpc) - return box + limit = self.metainterp.jitdriver_sd.warmstate.guard_value_limit + if box is self.metainterp.not_guarded_value_box or self.metainterp.guard_value_counter > limit: + # limit reached or guard on the same box as the first elided guard_value + self.metainterp.not_guarded_value_box = box + return box + + return self.implement_guard_value(box, orgpc) @arguments("box", "box", "descr", "orgpc") def opimpl_str_guard_value(self, box, funcbox, descr, orgpc): @@ -2432,6 +2436,7 @@ # specialize this function and a few other ones for the '*args'. debug_start('jit-tracing') self.guard_value_counter = 0 + self.not_guarded_value_box = None self.staticdata._setup_once() self.staticdata.profiler.start_tracing() assert jitdriver_sd is self.jitdriver_sd @@ -2465,6 +2470,7 @@ key = resumedescr.get_resumestorage() assert isinstance(key, compile.ResumeGuardDescr) self.guard_value_counter = key.guard_value_counter + 1 + self.not_guarded_value_box = None # store the resumekey.wref_original_loop_token() on 'self' to make # sure that it stays alive as long as this MetaInterp self.resumekey_original_loop_token = resumedescr.rd_loop_token.loop_token_wref() diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -92,6 +92,7 @@ disable_unrolling=sys.maxint, enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, max_unroll_recursion=7, vec=0, vec_all=0, vec_cost=0, + guard_value_limit=4, **kwds): from rpython.config.config import ConfigError translator = interp.typer.annotator.translator @@ -119,6 +120,7 @@ jd.warmstate.set_param_vec(vec) jd.warmstate.set_param_vec_all(vec_all) jd.warmstate.set_param_vec_cost(vec_cost) + jd.warmstate.set_param_guard_value_limit(guard_value_limit) warmrunnerdesc.finish() if graph_and_interp_only: return interp, graph diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -318,6 +318,9 @@ def set_param_vec_cost(self, ivalue): self.vec_cost = ivalue + def set_param_guard_value_limit(self, ivalue): + self.guard_value_limit = ivalue + def disable_noninlinable_function(self, greenkey): cell = self.JitCell.ensure_jit_cell_at_key(greenkey) cell.flags |= JC_DONT_TRACE_HERE diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -561,6 +561,7 @@ 'vec_cost': 'threshold for which traces to bail. Unpacking increases the counter,'\ ' vector operation decrease the cost', 'vec_all': 'try to vectorize trace loops that occur outside of the numpypy library', + 'guard_value_limit': 'limits the chain of guard_values coming from a single guard_value', } PARAMETERS = {'threshold': 1039, # just above 1024, prime @@ -579,6 +580,7 @@ 'vec': 0, 'vec_all': 0, 'vec_cost': 0, + 'guard_value_limit': 4, } unroll_parameters = unrolling_iterable(PARAMETERS.items()) From pypy.commits at gmail.com Sun Mar 25 03:04:06 2018 From: pypy.commits at gmail.com (mattip) Date: Sun, 25 Mar 2018 00:04:06 -0700 (PDT) Subject: [pypy-commit] pypy default: improve (?) docs Message-ID: <5ab749e6.0ea7df0a.10443.30f8@mx.google.com> Author: Matti Picus Branch: Changeset: r94131:ba9c5b2e6e60 Date: 2018-03-25 10:00 +0300 http://bitbucket.org/pypy/pypy/changeset/ba9c5b2e6e60/ Log: improve (?) docs diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,42 +4,40 @@ Welcome to PyPy! -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. +PyPy is an interperter that implements the Python programming language, based +on the RPython compiler framework for dynamic language implementations. -The home page is: +The home page for the interpreter is: http://pypy.org/ -If you want to help developing PyPy, this document might help you: +If you want to help developing PyPy, this documentation might help you: http://doc.pypy.org/ -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! +More documentation about the RPython framework can be found here - the pypy-dev team + http://rpython.readthedocs.io +The source for the documentation is in the pypy/doc directory + +Using PyPy instead of CPython +============================= + +Please read the information at http://pypy.org to find the correct way to +download and use PyPy as an alternative to CPython. Building ======== -First switch to or download the correct branch. The basic choices are -``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X -branch (e.g. ``py3.5``). +Building PyPy is not the recommended way to obtain the PyPy alternative python +interpreter. It is time-consuming and requires significant computing resources. +More information can be found here -Build with: + http://doc.pypy.org/en/latest/build.html -.. code-block:: console +Enjoy and send us feedback! - $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + the pypy-dev team -This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy -directory. We suggest to use virtualenv with the resulting -pypy-c/pypy3-c as the interpreter; you can find more details about -various installation schemes here: - http://doc.pypy.org/en/latest/install.html diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -17,13 +17,18 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~ The quickest way to start using PyPy is to download a prebuilt binary for your -OS and architecture. You can either use the `most recent release`_ or one of -our `development nightly build`_. Please note that the nightly builds are not +OS and architecture. You may be able to use either use the +`most recent release`_ or one of our `development nightly build`_. These +builds depend on dynamically linked libraries that may not be available on your +OS. See the section about `Linux binaries` for more info and alternatives that +may work on your system. + +Please note that the nightly builds are not guaranteed to be as stable as official releases, use them at your own risk. .. _most recent release: http://pypy.org/download.html .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/ - +.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions Installing PyPy ~~~~~~~~~~~~~~~ @@ -69,9 +74,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is often convenient to run pypy inside a virtualenv. To do this -you need a recent version of virtualenv -- 1.6.1 or greater. You can +you need a version of virtualenv -- 1.6.1 or greater. You can then install PyPy both from a precompiled tarball or from a mercurial -checkout:: +checkout after translation:: # from a tarball $ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env From pypy.commits at gmail.com Mon Mar 26 03:06:09 2018 From: pypy.commits at gmail.com (Raemi) Date: Mon, 26 Mar 2018 00:06:09 -0700 (PDT) Subject: [pypy-commit] stmgc evaluation: branch to evaluate configurations Message-ID: <5ab89be1.845d1c0a.8ebad.05b7@mx.google.com> Author: Remi Meier Branch: evaluation Changeset: r2157:3a80cd885fe0 Date: 2018-03-26 08:39 +0200 http://bitbucket.org/pypy/stmgc/changeset/3a80cd885fe0/ Log: branch to evaluate configurations From pypy.commits at gmail.com Mon Mar 26 03:06:11 2018 From: pypy.commits at gmail.com (Raemi) Date: Mon, 26 Mar 2018 00:06:11 -0700 (PDT) Subject: [pypy-commit] stmgc evaluation: merge c8-efficient-serial-execution-master Message-ID: <5ab89be3.480f1c0a.77363.8244@mx.google.com> Author: Remi Meier Branch: evaluation Changeset: r2158:ce8f8880e0e2 Date: 2018-03-26 09:05 +0200 http://bitbucket.org/pypy/stmgc/changeset/ce8f8880e0e2/ Log: merge c8-efficient-serial-execution-master diff --git a/c8/demo/Makefile b/c8/demo/Makefile --- a/c8/demo/Makefile +++ b/c8/demo/Makefile @@ -17,7 +17,7 @@ H_FILES = ../stmgc.h ../stm/*.h C_FILES = ../stmgc.c ../stm/*.c -COMMON = -I.. -pthread -lrt -g -Wall -Werror -DSTM_LARGEMALLOC_TEST +COMMON = -I.. -pthread -lrt -lm -g -Wall -Werror -DSTM_LARGEMALLOC_TEST CC = gcc-seg-gs diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -381,6 +381,14 @@ static void readd_wb_executed_flags(void); static void check_all_write_barrier_flags(char *segbase, struct list_s *list); +static void signal_commit_to_inevitable_transaction(void) { + struct stm_priv_segment_info_s* inevitable_segement = get_inevitable_thread_segment(); + if (inevitable_segement != 0) { + // the inevitable thread is still running: set its "please commit" flag (is ignored by the inevitable thread if it is atomic) + inevitable_segement->commit_if_not_atomic = true; + } +} + static void wait_for_inevitable(void) { intptr_t detached = 0; @@ -397,6 +405,8 @@ try to detach an inevitable transaction regularly */ detached = fetch_detached_transaction(); if (detached == 0) { + // the inevitable trx was not detached or it was detached but is atomic + signal_commit_to_inevitable_transaction(); EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE); if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001)) goto wait_some_more; @@ -1138,11 +1148,10 @@ } _do_start_transaction(tl); - if (repeat_count == 0) { /* else, 'nursery_mark' was already set - in abort_data_structures_from_segment_num() */ - STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start + - stm_fill_mark_nursery_bytes); - } + STM_PSEGMENT->commit_if_not_atomic = false; + STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start + + stm_get_transaction_length(tl)); + return repeat_count; } @@ -1271,6 +1280,8 @@ bool was_inev = STM_PSEGMENT->transaction_state == TS_INEVITABLE; _validate_and_add_to_commit_log(); + + stm_thread_local_t *tl_for_trx_len = STM_SEGMENT->running_thread; if (external) { /* from this point on, unlink the original 'stm_thread_local_t *' from its segment. Better do it as soon as possible, because @@ -1318,6 +1329,8 @@ s_mutex_unlock(); + stm_transaction_length_handle_validation(tl_for_trx_len, false); + /* between transactions, call finalizers. this will execute a transaction itself */ if (tl != NULL) @@ -1484,22 +1497,6 @@ if (pseg->active_queues) queues_deactivate_all(pseg, /*at_commit=*/false); - - /* Set the next nursery_mark: first compute the value that - nursery_mark must have had at the start of the aborted transaction */ - stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery; - - /* This means that the limit, in term of bytes, was: */ - uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start; - - /* If 'total_throw_away_nursery' is smaller than old_limit, use that */ - if (pseg->total_throw_away_nursery < old_limit) - old_limit = pseg->total_throw_away_nursery; - - /* Now set the new limit to 90% of the old limit */ - pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start + - (uintptr_t)(old_limit * 0.9)); - #ifdef STM_NO_AUTOMATIC_SETJMP did_abort = 1; #endif @@ -1534,6 +1531,8 @@ tl->self_or_0_if_atomic = (intptr_t)tl; /* clear the 'atomic' flag */ STM_PSEGMENT->atomic_nesting_levels = 0; + stm_transaction_length_handle_validation(tl, true); + if (tl->mem_clear_on_abort) memset(tl->mem_clear_on_abort, 0, tl->mem_bytes_to_clear_on_abort); if (tl->mem_reset_on_abort) { @@ -1588,7 +1587,7 @@ void _stm_become_inevitable(const char *msg) { - int num_waits = 0; + int num_waits = 1; timing_become_inevitable(); @@ -1599,50 +1598,48 @@ _stm_collectable_safe_point(); dprintf(("become_inevitable: %s\n", msg)); - if (any_soon_finished_or_inevitable_thread_segment() && - num_waits <= NB_SEGMENTS) { + if (any_soon_finished_or_inevitable_thread_segment()) { #if STM_TESTS /* for tests: another transaction */ stm_abort_transaction(); /* is already inevitable, abort */ #endif - bool timed_out = false; + signal_commit_to_inevitable_transaction(); s_mutex_lock(); if (any_soon_finished_or_inevitable_thread_segment() && - !safe_point_requested()) { + !safe_point_requested() && + num_waits <= NB_SEGMENTS) { /* wait until C_SEGMENT_FREE_OR_SAFE_POINT_REQ is signalled */ EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE); - if (!cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, - 0.000054321)) - timed_out = true; + if (cond_wait_timeout(C_SEGMENT_FREE_OR_SAFE_POINT_REQ, 0.00001)) { + num_waits++; + } } s_mutex_unlock(); - - if (timed_out) { - /* try to detach another inevitable transaction, but - only after waiting a bit. This is necessary to avoid - deadlocks in some situations, which are hopefully - not too common. We don't want two threads constantly - detaching each other. */ - intptr_t detached = fetch_detached_transaction(); - if (detached != 0) { - EMIT_WAIT_DONE(); - commit_fetched_detached_transaction(detached); - } - } - else { - num_waits++; + /* XXX try to detach another inevitable transaction, but + only after waiting a bit. This is necessary to avoid + deadlocks in some situations, which are hopefully + not too common. We don't want two threads constantly + detaching each other. */ + intptr_t detached = fetch_detached_transaction(); + if (detached != 0) { + EMIT_WAIT_DONE(); + commit_fetched_detached_transaction(detached); + EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE); } goto retry_from_start; } - EMIT_WAIT_DONE(); - if (!_validate_and_turn_inevitable()) - goto retry_from_start; + else { + EMIT_WAIT_DONE(); + if (!_validate_and_turn_inevitable()) { + EMIT_WAIT(STM_WAIT_OTHER_INEVITABLE); + goto retry_from_start; + } + } } - else { - if (!_validate_and_turn_inevitable()) - return; + else if (!_validate_and_turn_inevitable()) { + return; } /* There may be a concurrent commit of a detached Tx going on. @@ -1654,6 +1651,7 @@ stm_spin_loop(); assert(_stm_detached_inevitable_from_thread == 0); + STM_PSEGMENT->commit_if_not_atomic = false; soon_finished_or_inevitable_thread_segment(); STM_PSEGMENT->transaction_state = TS_INEVITABLE; diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -168,6 +168,9 @@ /* For stm_enable_atomic() */ uintptr_t atomic_nesting_levels; + + // TODO signal flag that is checked in throw_away_nursery() for making immediate commit + bool commit_if_not_atomic; }; enum /* safe_point */ { diff --git a/c8/stm/detach.c b/c8/stm/detach.c --- a/c8/stm/detach.c +++ b/c8/stm/detach.c @@ -215,6 +215,7 @@ } } +// TODO write tests, verify is working, verify no overflows with adaptive mode uintptr_t stm_is_atomic(stm_thread_local_t *tl) { assert(STM_SEGMENT->running_thread == tl); @@ -228,14 +229,18 @@ return STM_PSEGMENT->atomic_nesting_levels; } +// max intptr_t value is 7FFFFFFFFFFFFFFF on 64-bit => larger than 2 * huge value #define HUGE_INTPTR_VALUE 0x3000000000000000L void stm_enable_atomic(stm_thread_local_t *tl) { if (!stm_is_atomic(tl)) { + // do for outermost atomic block only tl->self_or_0_if_atomic = 0; /* increment 'nursery_mark' by HUGE_INTPTR_VALUE, so that - stm_should_break_transaction() returns always false */ + stm_should_break_transaction() returns always false. + preserves the previous nursery_mark, unless it is < 0 + or >= huge value */ intptr_t mark = (intptr_t)STM_SEGMENT->nursery_mark; if (mark < 0) mark = 0; @@ -255,6 +260,7 @@ STM_PSEGMENT->atomic_nesting_levels--; if (STM_PSEGMENT->atomic_nesting_levels == 0) { + // revert changes by stm_enable_atomic only if we left the outermost atomic block tl->self_or_0_if_atomic = (intptr_t)tl; /* decrement 'nursery_mark' by HUGE_INTPTR_VALUE, to cancel what was done in stm_enable_atomic() */ diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -4,6 +4,8 @@ #endif #include "finalizer.h" +#include +#include /************************************************************/ @@ -13,14 +15,77 @@ static uintptr_t _stm_nursery_start; +#define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4) -#define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4) +// corresponds to ~4 GB +#define LARGE_FILL_MARK_NURSERY_BYTES 0x100000000L -uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES; +// corresponds to ~4 MB nursery fill +#define STM_DEFAULT_RELATIVE_TRANSACTION_LENGTH (0.001) +// corresponds to ~400 KB nursery fill +#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.0001) + +#define BACKOFF_COUNT (20) +#define BACKOFF_MULTIPLIER (BACKOFF_COUNT / -log10(STM_MIN_RELATIVE_TRANSACTION_LENGTH)) + +static inline void set_backoff(stm_thread_local_t *tl, double rel_trx_len) { + /* the shorter the trx, the more backoff: + think a*x + b = backoff, x := -log(rel-trx-len), + backoff is + b at default trx length, + linear decrease to b at max trx length */ + const int b = 5; + int new_backoff = (int)((BACKOFF_MULTIPLIER * -log10(rel_trx_len)) + b); + tl->transaction_length_backoff = new_backoff; + // printf("thread %d, backoff %d\n", tl->thread_local_counter, tl->transaction_length_backoff); + tl->linear_transaction_length_increment = rel_trx_len / new_backoff; +} + +static inline double get_new_transaction_length(stm_thread_local_t *tl, bool aborts) { + const int multiplier = 2; + double previous = tl->relative_transaction_length; + double new; + if (aborts) { + new = previous / multiplier; + if (new < STM_MIN_RELATIVE_TRANSACTION_LENGTH) { + new = STM_MIN_RELATIVE_TRANSACTION_LENGTH; + } + set_backoff(tl, new); + } else if (tl->transaction_length_backoff == 0) { + // backoff counter is zero, exponential increase up to 1 + new = previous * multiplier; + if (new > 1) { + new = 1; + } + if (tl->linear_transaction_length_increment != 0) { + // thread had to abort before: slow start + set_backoff(tl, new); + } + } else { // not abort and backoff != 0 + // in backoff, linear increase up to 1 + new = previous + tl->linear_transaction_length_increment; + if (new > 1) { + new = 1; + } + tl->transaction_length_backoff -= 1; + } + return new; +} + +static inline void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts) { + tl->relative_transaction_length = get_new_transaction_length(tl, aborts); +} + +static inline uintptr_t stm_get_transaction_length(stm_thread_local_t *tl) { + double relative_additional_length = tl->relative_transaction_length; + uintptr_t result = + (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length); + // printf("%020" PRIxPTR "\n", result); + return result; +} + /************************************************************/ - static void setup_nursery(void) { assert(_STM_FAST_ALLOC <= NURSERY_SIZE); @@ -500,6 +565,14 @@ pseg->pub.nursery_current = (stm_char *)_stm_nursery_start; pseg->pub.nursery_mark -= nursery_used; + assert((pseg->transaction_state == TS_INEVITABLE) || !pseg->commit_if_not_atomic); + if (pseg->commit_if_not_atomic + && pseg->transaction_state == TS_INEVITABLE + && pseg->pub.running_thread->self_or_0_if_atomic != 0) { + // transaction is inevitable, not atomic, and commit has been signalled by waiting thread: commit immediately + pseg->pub.nursery_mark = 0; + } + /* free any object left from 'young_outside_nursery' */ if (!tree_is_cleared(pseg->young_outside_nursery)) { wlog_t *item; diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h --- a/c8/stm/nursery.h +++ b/c8/stm/nursery.h @@ -56,4 +56,7 @@ static inline struct object_s *mark_loc(object_t *obj); static inline bool _is_from_same_transaction(object_t *obj); +static inline void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts); +static inline uintptr_t stm_get_transaction_length(stm_thread_local_t *tl); + #endif diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -277,6 +277,12 @@ numbers automatically. */ tl->last_associated_segment_num = num + 1; tl->thread_local_counter = ++thread_local_counters; + + /* init adaptive transaction length mode */ + tl->relative_transaction_length = STM_DEFAULT_RELATIVE_TRANSACTION_LENGTH; + tl->transaction_length_backoff = 0; + tl->linear_transaction_length_increment = 0; + *_get_cpth(tl) = pthread_self(); _init_shadow_stack(tl); set_gs_register(get_segment_base(num + 1)); diff --git a/c8/stm/sync.c b/c8/stm/sync.c --- a/c8/stm/sync.c +++ b/c8/stm/sync.c @@ -176,6 +176,18 @@ /************************************************************/ +#if 0 +static uint8_t number_of_segments_in_use(void) { + uint8_t result = 0; + int num; + for (num = 1; num < NB_SEGMENTS; num++) { + if (sync_ctl.in_use1[num] > 0) { + result++; + } + } + return result; +} +#endif #if 0 void stm_wait_for_current_inevitable_transaction(void) @@ -202,7 +214,6 @@ } #endif - static void acquire_thread_segment(stm_thread_local_t *tl) { /* This function acquires a segment for the currently running thread, @@ -293,6 +304,19 @@ return false; } +static struct stm_priv_segment_info_s* get_inevitable_thread_segment(void) +{ + struct stm_priv_segment_info_s* segment; + int num; + for (num = 1; num < NB_SEGMENTS; num++) { + segment = get_priv_segment(num); + if (segment->transaction_state == TS_INEVITABLE) { + return segment; + } + } + return 0; +} + __attribute__((unused)) static bool _seems_to_be_running_transaction(void) { diff --git a/c8/stm/sync.h b/c8/stm/sync.h --- a/c8/stm/sync.h +++ b/c8/stm/sync.h @@ -29,6 +29,7 @@ static void release_thread_segment(stm_thread_local_t *tl); static void soon_finished_or_inevitable_thread_segment(void); static bool any_soon_finished_or_inevitable_thread_segment(void); +static struct stm_priv_segment_info_s* get_inevitable_thread_segment(void); enum sync_type_e { STOP_OTHERS_UNTIL_MUTEX_UNLOCK, diff --git a/c8/stmgc.h b/c8/stmgc.h --- a/c8/stmgc.h +++ b/c8/stmgc.h @@ -88,6 +88,13 @@ struct stm_thread_local_s *prev, *next; intptr_t self_or_0_if_atomic; void *creating_pthread[2]; + /* == adaptive single thread mode == */ + /* factor that is multiplied with max transaction length before the start of the next transaction on this thread */ + double relative_transaction_length; + /* when zero, transaction length may increase exponentially, otherwise transaction length may only increase linearly. is (re-)set to some value upon abort and counted down until zero upon successful validation. */ + int transaction_length_backoff; + /* during the backoff, transaction length may increase linearly by this increment on every successful validation */ + double linear_transaction_length_increment; } stm_thread_local_t; @@ -202,7 +209,7 @@ /* ==================== PUBLIC API ==================== */ /* Number of segments (i.e. how many transactions can be executed in - parallel, in maximum). If you try to start transactions in more + parallel, at maximum). If you try to start transactions in more threads than the number of segments, it will block, waiting for the next segment to become free. */ @@ -464,14 +471,6 @@ return ((intptr_t)STM_SEGMENT->nursery_current >= (intptr_t)STM_SEGMENT->nursery_mark); } -extern uintptr_t stm_fill_mark_nursery_bytes; -/* ^^^ at the start of a transaction, 'nursery_mark' is initialized to - 'stm_fill_mark_nursery_bytes' inside the nursery. This value can - be larger than the nursery; every minor collection shifts the - current 'nursery_mark' down by one nursery-size. After an abort - and restart, 'nursery_mark' is set to ~90% of the value it reached - in the last attempt. -*/ /* "atomic" transaction: a transaction where stm_should_break_transaction() always returns false, and where stm_leave_transactional_zone() never @@ -575,21 +574,49 @@ STM_GC_MAJOR_START, STM_GC_MAJOR_DONE, + /* execution duration profiling events */ + STM_WARMUP_COMPLETE, + + STM_DURATION_START_TRX, + STM_DURATION_WRITE_GC_ONLY, + STM_DURATION_WRITE_SLOWPATH, + STM_DURATION_VALIDATION, + STM_DURATION_CREATE_CLE, + STM_DURATION_COMMIT_EXCEPT_GC, + STM_DURATION_MINOR_GC, + STM_DURATION_MAJOR_GC_LOG_ONLY, + STM_DURATION_MAJOR_GC_FULL, + + STM_SINGLE_THREAD_MODE_ON, + STM_SINGLE_THREAD_MODE_OFF, + STM_SINGLE_THREAD_MODE_ADAPTIVE, + _STM_EVENT_N }; -#define STM_EVENT_NAMES \ - "transaction start", \ - "transaction commit", \ - "transaction abort", \ - "contention write read", \ - "wait free segment", \ - "wait other inevitable", \ - "wait done", \ - "gc minor start", \ - "gc minor done", \ - "gc major start", \ - "gc major done" +#define STM_EVENT_NAMES \ + "transaction start", \ + "transaction commit", \ + "transaction abort", \ + "contention write read", \ + "wait free segment", \ + "wait other inevitable", \ + "wait done", \ + "gc minor start", \ + "gc minor done", \ + "gc major start", \ + "gc major done", \ + /* names of duration events */ \ + "marks completion of benchmark warm up phase" \ + "duration of transaction start", \ + "duration of gc due to write", \ + "duration of write slowpath", \ + "duration of validation", \ + "duration of commit log entry creation", \ + "duration of commit except gc", \ + "duration of minor gc", \ + "duration of major gc doing log clean up only", \ + "duration of full major gc" /* The markers pushed in the shadowstack are an odd number followed by a regular object pointer. */ diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -45,7 +45,6 @@ } stm_thread_local_t; char *stm_object_pages; -uintptr_t stm_fill_mark_nursery_bytes; void stm_read(object_t *obj); /*void stm_write(object_t *obj); use _checked_stm_write() instead */ @@ -671,7 +670,7 @@ undef_macros=['NDEBUG'], include_dirs=[parent_dir], extra_compile_args=['-g', '-O0', '-Werror', '-Wall'], #, '-ferror-limit=5'], - extra_link_args=['-g', '-lrt'], + extra_link_args=['-g', '-lrt', '-lm'], force_generic_engine=True) diff --git a/c8/test/test_basic.py b/c8/test/test_basic.py --- a/c8/test/test_basic.py +++ b/c8/test/test_basic.py @@ -736,6 +736,7 @@ self.check_char_everywhere(lp1, 'X') def test_stm_should_break_transaction_1(self): + py.test.skip("replaced by tcp logic") lib.stm_fill_mark_nursery_bytes = 100 # self.start_transaction() @@ -772,6 +773,7 @@ self.commit_transaction() def test_stm_should_break_transaction_2(self): + py.test.skip("replaced by tcp logic") lib.stm_fill_mark_nursery_bytes = 10000000 # n = 10000000 diff --git a/gcc-seg-gs/README.txt b/gcc-seg-gs/README.txt --- a/gcc-seg-gs/README.txt +++ b/gcc-seg-gs/README.txt @@ -8,9 +8,8 @@ compile the standard gcc. Of course, it is likely that gcc 6.1 will soon be available from your Linux distribution directly. -Note that with gcc 6.1, you no longer need gcc-5.1.0-patch.diff, and you -should not need the "-fno-*" options either (but we didn't check that -yet). +Note that with gcc 6.1, you no longer need gcc-5.1.0-patch.diff, but you +still need the "-fno-*" options. From pypy.commits at gmail.com Mon Mar 26 05:46:49 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Mar 2018 02:46:49 -0700 (PDT) Subject: [pypy-commit] pypy default: workaround for issue 2225 Message-ID: <5ab8c189.15741c0a.39fec.d427@mx.google.com> Author: Matti Picus Branch: Changeset: r94132:d721da4573ad Date: 2018-03-26 12:45 +0300 http://bitbucket.org/pypy/pypy/changeset/d721da4573ad/ Log: workaround for issue 2225 diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -275,12 +275,13 @@ sublist_w = [] n = last_pos = 0 while not count or n < count: + pattern = ctx.pattern sub_jitdriver.jit_merge_point( self=self, use_builder=use_builder, filter_is_callable=filter_is_callable, filter_type=type(w_filter), - ctx=ctx, + ctx=ctx, pattern=pattern, w_filter=w_filter, strbuilder=strbuilder, unicodebuilder=unicodebuilder, @@ -355,7 +356,7 @@ filter_as_unicode w_string sublist_w self""".split(), - greens=["filter_is_callable", "use_builder", "filter_type", "ctx.pattern"]) + greens=["filter_is_callable", "use_builder", "filter_type", "pattern"]) def _sub_append_slice(ctx, space, use_builder, sublist_w, From pypy.commits at gmail.com Mon Mar 26 09:51:30 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Mar 2018 06:51:30 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: stop using greenfields in sre, instead pass the pattern around Message-ID: <5ab8fae2.8e6f1c0a.de05.5010@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94133:c93d31a2fabd Date: 2018-03-26 14:41 +0200 http://bitbucket.org/pypy/pypy/changeset/c93d31a2fabd/ Log: stop using greenfields in sre, instead pass the pattern around diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -77,15 +77,15 @@ w_import = space.getattr(w_builtin, space.newtext("__import__")) return space.call_function(w_import, space.newtext("re")) -def matchcontext(space, ctx): +def matchcontext(space, ctx, pattern): try: - return rsre_core.match_context(ctx) + return rsre_core.match_context(ctx, pattern) except rsre_core.Error as e: raise OperationError(space.w_RuntimeError, space.newtext(e.msg)) -def searchcontext(space, ctx): +def searchcontext(space, ctx, pattern): try: - return rsre_core.search_context(ctx) + return rsre_core.search_context(ctx, pattern) except rsre_core.Error as e: raise OperationError(space.w_RuntimeError, space.newtext(e.msg)) @@ -144,12 +144,12 @@ @unwrap_spec(pos=int, endpos=int) def match_w(self, w_string, pos=0, endpos=sys.maxint): ctx = self.make_ctx(w_string, pos, endpos) - return self.getmatch(ctx, matchcontext(self.space, ctx)) + return self.getmatch(ctx, matchcontext(self.space, ctx, self.code)) @unwrap_spec(pos=int, endpos=int) def search_w(self, w_string, pos=0, endpos=sys.maxint): ctx = self.make_ctx(w_string, pos, endpos) - return self.getmatch(ctx, searchcontext(self.space, ctx)) + return self.getmatch(ctx, searchcontext(self.space, ctx, self.code)) @unwrap_spec(pos=int, endpos=int) def findall_w(self, w_string, pos=0, endpos=sys.maxint): @@ -157,7 +157,7 @@ matchlist_w = [] ctx = self.make_ctx(w_string, pos, endpos) while ctx.match_start <= ctx.end: - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, self.code): break num_groups = self.num_groups w_emptystr = space.newtext("") @@ -182,7 +182,7 @@ # this also works as the implementation of the undocumented # scanner() method. ctx = self.make_ctx(w_string, pos, endpos) - scanner = W_SRE_Scanner(self, ctx) + scanner = W_SRE_Scanner(self, ctx, self.code) return scanner @unwrap_spec(maxsplit=int) @@ -193,7 +193,7 @@ last = 0 ctx = self.make_ctx(w_string) while not maxsplit or n < maxsplit: - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, self.code): break if ctx.match_start == ctx.match_end: # zero-width match if ctx.match_start == ctx.end: # or end of string @@ -274,8 +274,8 @@ else: sublist_w = [] n = last_pos = 0 + pattern = self.code while not count or n < count: - pattern = ctx.pattern sub_jitdriver.jit_merge_point( self=self, use_builder=use_builder, @@ -292,7 +292,7 @@ n=n, last_pos=last_pos, sublist_w=sublist_w ) space = self.space - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, pattern): break if last_pos < ctx.match_start: _sub_append_slice( @@ -388,7 +388,7 @@ srepat.space = space srepat.w_pattern = w_pattern # the original uncompiled pattern srepat.flags = flags - srepat.code = code + srepat.code = rsre_core.CompiledPattern(code) srepat.num_groups = groups srepat.w_groupindex = w_groupindex srepat.w_indexgroup = w_indexgroup @@ -611,10 +611,11 @@ # Our version is also directly iterable, to make finditer() easier. class W_SRE_Scanner(W_Root): - def __init__(self, pattern, ctx): + def __init__(self, pattern, ctx, code): self.space = pattern.space self.srepat = pattern self.ctx = ctx + self.code = code # 'self.ctx' is always a fresh context in which no searching # or matching succeeded so far. @@ -624,19 +625,19 @@ def next_w(self): if self.ctx.match_start > self.ctx.end: raise OperationError(self.space.w_StopIteration, self.space.w_None) - if not searchcontext(self.space, self.ctx): + if not searchcontext(self.space, self.ctx, self.code): raise OperationError(self.space.w_StopIteration, self.space.w_None) return self.getmatch(True) def match_w(self): if self.ctx.match_start > self.ctx.end: return self.space.w_None - return self.getmatch(matchcontext(self.space, self.ctx)) + return self.getmatch(matchcontext(self.space, self.ctx, self.code)) def search_w(self): if self.ctx.match_start > self.ctx.end: return self.space.w_None - return self.getmatch(searchcontext(self.space, self.ctx)) + return self.getmatch(searchcontext(self.space, self.ctx, self.code)) def getmatch(self, found): if found: diff --git a/rpython/rlib/rsre/rpy/_sre.py b/rpython/rlib/rsre/rpy/_sre.py --- a/rpython/rlib/rsre/rpy/_sre.py +++ b/rpython/rlib/rsre/rpy/_sre.py @@ -1,4 +1,4 @@ -from rpython.rlib.rsre import rsre_char +from rpython.rlib.rsre import rsre_char, rsre_core from rpython.rlib.rarithmetic import intmask VERSION = "2.7.6" @@ -12,7 +12,7 @@ pass def compile(pattern, flags, code, *args): - raise GotIt([intmask(i) for i in code], flags, args) + raise GotIt(rsre_core.CompiledPattern([intmask(i) for i in code]), flags, args) def get_code(regexp, flags=0, allargs=False): diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py --- a/rpython/rlib/rsre/rsre_char.py +++ b/rpython/rlib/rsre/rsre_char.py @@ -152,17 +152,16 @@ ##### Charset evaluation @jit.unroll_safe -def check_charset(ctx, ppos, char_code): +def check_charset(ctx, pattern, ppos, char_code): """Checks whether a character matches set of arbitrary length. The set starts at pattern[ppos].""" negated = False result = False - pattern = ctx.pattern while True: - opcode = pattern[ppos] + opcode = pattern.pattern[ppos] for i, function in set_dispatch_unroll: if opcode == i: - newresult, ppos = function(ctx, ppos, char_code) + newresult, ppos = function(ctx, pattern, ppos, char_code) result |= newresult break else: @@ -177,50 +176,44 @@ return not result return result -def set_literal(ctx, index, char_code): +def set_literal(ctx, pattern, index, char_code): # - pat = ctx.pattern - match = pat[index+1] == char_code + match = pattern.pattern[index+1] == char_code return match, index + 2 -def set_category(ctx, index, char_code): +def set_category(ctx, pattern, index, char_code): # - pat = ctx.pattern - match = category_dispatch(pat[index+1], char_code) + match = category_dispatch(pattern.pattern[index+1], char_code) return match, index + 2 -def set_charset(ctx, index, char_code): +def set_charset(ctx, pattern, index, char_code): # (16 bits per code word) - pat = ctx.pattern if CODESIZE == 2: match = char_code < 256 and \ - (pat[index+1+(char_code >> 4)] & (1 << (char_code & 15))) + (pattern.pattern[index+1+(char_code >> 4)] & (1 << (char_code & 15))) return match, index + 17 # skip bitmap else: match = char_code < 256 and \ - (pat[index+1+(char_code >> 5)] & (1 << (char_code & 31))) + (pattern.pattern[index+1+(char_code >> 5)] & (1 << (char_code & 31))) return match, index + 9 # skip bitmap -def set_range(ctx, index, char_code): +def set_range(ctx, pattern, index, char_code): # - pat = ctx.pattern - match = int_between(pat[index+1], char_code, pat[index+2] + 1) + match = int_between(pattern.pattern[index+1], char_code, pattern.pattern[index+2] + 1) return match, index + 3 -def set_range_ignore(ctx, index, char_code): +def set_range_ignore(ctx, pattern, index, char_code): # # the char_code is already lower cased - pat = ctx.pattern - lower = pat[index + 1] - upper = pat[index + 2] + lower = pattern.pattern[index + 1] + upper = pattern.pattern[index + 2] match1 = int_between(lower, char_code, upper + 1) match2 = int_between(lower, getupper(char_code, ctx.flags), upper + 1) return match1 | match2, index + 3 -def set_bigcharset(ctx, index, char_code): +def set_bigcharset(ctx, pattern, index, char_code): # <256 blockindices> - pat = ctx.pattern - count = pat[index+1] + count = pattern.pattern[index+1] index += 2 if CODESIZE == 2: @@ -238,7 +231,7 @@ return False, index shift = 5 - block = pat[index + (char_code >> (shift + 5))] + block = pattern.pattern[index + (char_code >> (shift + 5))] block_shift = char_code >> 5 if BIG_ENDIAN: @@ -247,23 +240,22 @@ block = (block >> block_shift) & 0xFF index += 256 / CODESIZE - block_value = pat[index+(block * (32 / CODESIZE) + block_value = pattern.pattern[index+(block * (32 / CODESIZE) + ((char_code & 255) >> shift))] match = (block_value & (1 << (char_code & ((8 * CODESIZE) - 1)))) index += count * (32 / CODESIZE) # skip blocks return match, index -def set_unicode_general_category(ctx, index, char_code): +def set_unicode_general_category(ctx, pattern, index, char_code): # Unicode "General category property code" (not used by Python). - # A general category is two letters. 'pat[index+1]' contains both + # A general category is two letters. 'pattern.pattern[index+1]' contains both # the first character, and the second character shifted by 8. # http://en.wikipedia.org/wiki/Unicode_character_property#General_Category # Also supports single-character categories, if the second character is 0. # Negative matches are triggered by bit number 7. assert unicodedb is not None cat = unicodedb.category(char_code) - pat = ctx.pattern - category_code = pat[index + 1] + category_code = pattern.pattern[index + 1] first_character = category_code & 0x7F second_character = (category_code >> 8) & 0x7F negative_match = category_code & 0x80 diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -83,9 +83,31 @@ def __init__(self, msg): self.msg = msg + +class CompiledPattern(object): + _immutable_fields_ = ['pattern[*]'] + + def __init__(self, pattern): + self.pattern = pattern + # check we don't get the old value of MAXREPEAT + # during the untranslated tests + if not we_are_translated(): + assert 65535 not in pattern + + def pat(self, index): + jit.promote(self) + check_nonneg(index) + result = self.pattern[index] + # Check that we only return non-negative integers from this helper. + # It is possible that self.pattern contains negative integers + # (see set_charset() and set_bigcharset() in rsre_char.py) + # but they should not be fetched via this helper here. + assert result >= 0 + return result + class AbstractMatchContext(object): """Abstract base class""" - _immutable_fields_ = ['pattern[*]', 'flags', 'end'] + _immutable_fields_ = ['pattern', 'flags', 'end'] match_start = 0 match_end = 0 match_marks = None @@ -97,30 +119,17 @@ # and they must not be more than len(string). check_nonneg(match_start) check_nonneg(end) + assert isinstance(pattern, CompiledPattern) self.pattern = pattern self.match_start = match_start self.end = end self.flags = flags - # check we don't get the old value of MAXREPEAT - # during the untranslated tests - if not we_are_translated(): - assert 65535 not in pattern def reset(self, start): self.match_start = start self.match_marks = None self.match_marks_flat = None - def pat(self, index): - check_nonneg(index) - result = self.pattern[index] - # Check that we only return non-negative integers from this helper. - # It is possible that self.pattern contains negative integers - # (see set_charset() and set_bigcharset() in rsre_char.py) - # but they should not be fetched via this helper here. - assert result >= 0 - return result - @not_rpython def str(self, index): """Must be overridden in a concrete subclass. @@ -265,16 +274,16 @@ class MatchResult(object): subresult = None - def move_to_next_result(self, ctx): + def move_to_next_result(self, ctx, pattern): # returns either 'self' or None result = self.subresult if result is None: return - if result.move_to_next_result(ctx): + if result.move_to_next_result(ctx, pattern): return self - return self.find_next_result(ctx) + return self.find_next_result(ctx, pattern) - def find_next_result(self, ctx): + def find_next_result(self, ctx, pattern): raise NotImplementedError MATCHED_OK = MatchResult() @@ -287,11 +296,11 @@ self.start_marks = marks @jit.unroll_safe - def find_first_result(self, ctx): + def find_first_result(self, ctx, pattern): ppos = jit.hint(self.ppos, promote=True) - while ctx.pat(ppos): - result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks) - ppos += ctx.pat(ppos) + while pattern.pat(ppos): + result = sre_match(ctx, pattern, ppos + 1, self.start_ptr, self.start_marks) + ppos += pattern.pat(ppos) if result is not None: self.subresult = result self.ppos = ppos @@ -300,7 +309,7 @@ class RepeatOneMatchResult(MatchResult): install_jitdriver('RepeatOne', - greens=['nextppos', 'ctx.pattern'], + greens=['nextppos', 'pattern'], reds=['ptr', 'self', 'ctx'], debugprint=(1, 0)) # indices in 'greens' @@ -310,13 +319,14 @@ self.start_ptr = ptr self.start_marks = marks - def find_first_result(self, ctx): + def find_first_result(self, ctx, pattern): ptr = self.start_ptr nextppos = self.nextppos while ptr >= self.minptr: ctx.jitdriver_RepeatOne.jit_merge_point( - self=self, ptr=ptr, ctx=ctx, nextppos=nextppos) - result = sre_match(ctx, nextppos, ptr, self.start_marks) + self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, + pattern=pattern) + result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks) ptr -= 1 if result is not None: self.subresult = result @@ -327,7 +337,7 @@ class MinRepeatOneMatchResult(MatchResult): install_jitdriver('MinRepeatOne', - greens=['nextppos', 'ppos3', 'ctx.pattern'], + greens=['nextppos', 'ppos3', 'pattern'], reds=['ptr', 'self', 'ctx'], debugprint=(2, 0)) # indices in 'greens' @@ -338,39 +348,40 @@ self.start_ptr = ptr self.start_marks = marks - def find_first_result(self, ctx): + def find_first_result(self, ctx, pattern): ptr = self.start_ptr nextppos = self.nextppos ppos3 = self.ppos3 while ptr <= self.maxptr: ctx.jitdriver_MinRepeatOne.jit_merge_point( - self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3) - result = sre_match(ctx, nextppos, ptr, self.start_marks) + self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3, + pattern=pattern) + result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks) if result is not None: self.subresult = result self.start_ptr = ptr return self - if not self.next_char_ok(ctx, ptr, ppos3): + if not self.next_char_ok(ctx, pattern, ptr, ppos3): break ptr += 1 - def find_next_result(self, ctx): + def find_next_result(self, ctx, pattern): ptr = self.start_ptr - if not self.next_char_ok(ctx, ptr, self.ppos3): + if not self.next_char_ok(ctx, pattern, ptr, self.ppos3): return self.start_ptr = ptr + 1 - return self.find_first_result(ctx) + return self.find_first_result(ctx, pattern) - def next_char_ok(self, ctx, ptr, ppos): + def next_char_ok(self, ctx, pattern, ptr, ppos): if ptr == ctx.end: return False - op = ctx.pat(ppos) + op = pattern.pat(ppos) for op1, checkerfn in unroll_char_checker: if op1 == op: - return checkerfn(ctx, ptr, ppos) + return checkerfn(ctx, pattern, ptr, ppos) # obscure case: it should be a single char pattern, but isn't # one of the opcodes in unroll_char_checker (see test_ext_opcode) - return sre_match(ctx, ppos, ptr, self.start_marks) is not None + return sre_match(ctx, pattern, ppos, ptr, self.start_marks) is not None class AbstractUntilMatchResult(MatchResult): @@ -391,17 +402,17 @@ class MaxUntilMatchResult(AbstractUntilMatchResult): install_jitdriver('MaxUntil', - greens=['ppos', 'tailppos', 'match_more', 'ctx.pattern'], + greens=['ppos', 'tailppos', 'match_more', 'pattern'], reds=['ptr', 'marks', 'self', 'ctx'], debugprint=(3, 0, 2)) - def find_first_result(self, ctx): - return self.search_next(ctx, match_more=True) + def find_first_result(self, ctx, pattern): + return self.search_next(ctx, pattern, match_more=True) - def find_next_result(self, ctx): - return self.search_next(ctx, match_more=False) + def find_next_result(self, ctx, pattern): + return self.search_next(ctx, pattern, match_more=False) - def search_next(self, ctx, match_more): + def search_next(self, ctx, pattern, match_more): ppos = self.ppos tailppos = self.tailppos ptr = self.cur_ptr @@ -409,12 +420,13 @@ while True: ctx.jitdriver_MaxUntil.jit_merge_point( ppos=ppos, tailppos=tailppos, match_more=match_more, - ptr=ptr, marks=marks, self=self, ctx=ctx) + ptr=ptr, marks=marks, self=self, ctx=ctx, + pattern=pattern) if match_more: - max = ctx.pat(ppos+2) + max = pattern.pat(ppos+2) if max == rsre_char.MAXREPEAT or self.num_pending < max: # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + enum = sre_match(ctx, pattern, ppos + 3, ptr, marks) else: enum = None # 'max' reached, no more matches else: @@ -425,9 +437,9 @@ self.num_pending -= 1 ptr = p.ptr marks = p.marks - enum = p.enum.move_to_next_result(ctx) + enum = p.enum.move_to_next_result(ctx, pattern) # - min = ctx.pat(ppos+1) + min = pattern.pat(ppos+1) if enum is not None: # matched one more 'item'. record it and continue. last_match_length = ctx.match_end - ptr @@ -447,7 +459,7 @@ # 'item' no longer matches. if self.num_pending >= min: # try to match 'tail' if we have enough 'item' - result = sre_match(ctx, tailppos, ptr, marks) + result = sre_match(ctx, pattern, tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -457,23 +469,23 @@ class MinUntilMatchResult(AbstractUntilMatchResult): - def find_first_result(self, ctx): - return self.search_next(ctx, resume=False) + def find_first_result(self, ctx, pattern): + return self.search_next(ctx, pattern, resume=False) - def find_next_result(self, ctx): - return self.search_next(ctx, resume=True) + def find_next_result(self, ctx, pattern): + return self.search_next(ctx, pattern, resume=True) - def search_next(self, ctx, resume): + def search_next(self, ctx, pattern, resume): # XXX missing jit support here ppos = self.ppos - min = ctx.pat(ppos+1) - max = ctx.pat(ppos+2) + min = pattern.pat(ppos+1) + max = pattern.pat(ppos+2) ptr = self.cur_ptr marks = self.cur_marks while True: # try to match 'tail' if we have enough 'item' if not resume and self.num_pending >= min: - result = sre_match(ctx, self.tailppos, ptr, marks) + result = sre_match(ctx, pattern, self.tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -483,12 +495,12 @@ if max == rsre_char.MAXREPEAT or self.num_pending < max: # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + enum = sre_match(ctx, pattern, ppos + 3, ptr, marks) # # zero-width match protection if self.num_pending >= min: while enum is not None and ptr == ctx.match_end: - enum = enum.move_to_next_result(ctx) + enum = enum.move_to_next_result(ctx, pattern) else: enum = None # 'max' reached, no more matches @@ -502,7 +514,7 @@ self.num_pending -= 1 ptr = p.ptr marks = p.marks - enum = p.enum.move_to_next_result(ctx) + enum = p.enum.move_to_next_result(ctx, pattern) # matched one more 'item'. record it and continue self.pending = Pending(ptr, marks, enum, self.pending) @@ -514,13 +526,13 @@ @specializectx @jit.unroll_safe -def sre_match(ctx, ppos, ptr, marks): +def sre_match(ctx, pattern, ppos, ptr, marks): """Returns either None or a MatchResult object. Usually we only need the first result, but there is the case of REPEAT...UNTIL where we need all results; in that case we use the method move_to_next_result() of the MatchResult.""" while True: - op = ctx.pat(ppos) + op = pattern.pat(ppos) ppos += 1 #jit.jit_debug("sre_match", op, ppos, ptr) @@ -563,33 +575,33 @@ elif op == OPCODE_ASSERT: # assert subpattern # <0=skip> <1=back> - ptr1 = ptr - ctx.pat(ppos+1) + ptr1 = ptr - pattern.pat(ppos+1) saved = ctx.fullmatch_only ctx.fullmatch_only = False - stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None + stop = ptr1 < 0 or sre_match(ctx, pattern, ppos + 2, ptr1, marks) is None ctx.fullmatch_only = saved if stop: return marks = ctx.match_marks - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_ASSERT_NOT: # assert not subpattern # <0=skip> <1=back> - ptr1 = ptr - ctx.pat(ppos+1) + ptr1 = ptr - pattern.pat(ppos+1) saved = ctx.fullmatch_only ctx.fullmatch_only = False - stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) + stop = (ptr1 >= 0 and sre_match(ctx, pattern, ppos + 2, ptr1, marks) is not None) ctx.fullmatch_only = saved if stop: return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_AT: # match at given position (e.g. at beginning, at boundary, etc.) # - if not sre_at(ctx, ctx.pat(ppos), ptr): + if not sre_at(ctx, pattern.pat(ppos), ptr): return ppos += 1 @@ -597,14 +609,14 @@ # alternation # <0=skip> code ... result = BranchMatchResult(ppos, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) elif op == OPCODE_CATEGORY: # seems to be never produced, but used by some tests from # pypy/module/_sre/test # if (ptr == ctx.end or - not rsre_char.category_dispatch(ctx.pat(ppos), ctx.str(ptr))): + not rsre_char.category_dispatch(pattern.pat(ppos), ctx.str(ptr))): return ptr += 1 ppos += 1 @@ -612,7 +624,7 @@ elif op == OPCODE_GROUPREF: # match backreference # - startptr, length = get_group_ref(marks, ctx.pat(ppos)) + startptr, length = get_group_ref(marks, pattern.pat(ppos)) if length < 0: return # group was not previously defined if not match_repeated(ctx, ptr, startptr, length): @@ -623,7 +635,7 @@ elif op == OPCODE_GROUPREF_IGNORE: # match backreference # - startptr, length = get_group_ref(marks, ctx.pat(ppos)) + startptr, length = get_group_ref(marks, pattern.pat(ppos)) if length < 0: return # group was not previously defined if not match_repeated_ignore(ctx, ptr, startptr, length): @@ -634,44 +646,44 @@ elif op == OPCODE_GROUPREF_EXISTS: # conditional match depending on the existence of a group # codeyes codeno ... - _, length = get_group_ref(marks, ctx.pat(ppos)) + _, length = get_group_ref(marks, pattern.pat(ppos)) if length >= 0: ppos += 2 # jump to 'codeyes' else: - ppos += ctx.pat(ppos+1) # jump to 'codeno' + ppos += pattern.pat(ppos+1) # jump to 'codeno' elif op == OPCODE_IN: # match set member (or non_member) # - if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1, + if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1, ctx.str(ptr)): return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) ptr += 1 elif op == OPCODE_IN_IGNORE: # match set member (or non_member), ignoring case # - if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1, + if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1, ctx.lowstr(ptr)): return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) ptr += 1 elif op == OPCODE_INFO: # optimization info block # <0=skip> <1=flags> <2=min> ... - if (ctx.end - ptr) < ctx.pat(ppos+2): + if (ctx.end - ptr) < pattern.pat(ppos+2): return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_JUMP: - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_LITERAL: # match literal string # - if ptr >= ctx.end or ctx.str(ptr) != ctx.pat(ppos): + if ptr >= ctx.end or ctx.str(ptr) != pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -679,7 +691,7 @@ elif op == OPCODE_LITERAL_IGNORE: # match literal string, ignoring case # - if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.pat(ppos): + if ptr >= ctx.end or ctx.lowstr(ptr) != pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -687,14 +699,14 @@ elif op == OPCODE_MARK: # set mark # - gid = ctx.pat(ppos) + gid = pattern.pat(ppos) marks = Mark(gid, ptr, marks) ppos += 1 elif op == OPCODE_NOT_LITERAL: # match if it's not a literal string # - if ptr >= ctx.end or ctx.str(ptr) == ctx.pat(ppos): + if ptr >= ctx.end or ctx.str(ptr) == pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -702,7 +714,7 @@ elif op == OPCODE_NOT_LITERAL_IGNORE: # match if it's not a literal string, ignoring case # - if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos): + if ptr >= ctx.end or ctx.lowstr(ptr) == pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -715,22 +727,22 @@ # decode the later UNTIL operator to see if it is actually # a MAX_UNTIL or MIN_UNTIL - untilppos = ppos + ctx.pat(ppos) + untilppos = ppos + pattern.pat(ppos) tailppos = untilppos + 1 - op = ctx.pat(untilppos) + op = pattern.pat(untilppos) if op == OPCODE_MAX_UNTIL: # the hard case: we have to match as many repetitions as # possible, followed by the 'tail'. we do this by # remembering each state for each possible number of # 'item' matching. result = MaxUntilMatchResult(ppos, tailppos, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) elif op == OPCODE_MIN_UNTIL: # first try to match the 'tail', and if it fails, try # to match one more 'item' and try again result = MinUntilMatchResult(ppos, tailppos, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) else: raise Error("missing UNTIL after REPEAT") @@ -743,17 +755,18 @@ # use the MAX_REPEAT operator. # <1=min> <2=max> item tail start = ptr - minptr = start + ctx.pat(ppos+1) + minptr = start + pattern.pat(ppos+1) if minptr > ctx.end: return # cannot match - ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2), + ptr = find_repetition_end(ctx, pattern, ppos+3, start, + pattern.pat(ppos+2), marks) # when we arrive here, ptr points to the tail of the target # string. check if the rest of the pattern matches, # and backtrack if not. - nextppos = ppos + ctx.pat(ppos) + nextppos = ppos + pattern.pat(ppos) result = RepeatOneMatchResult(nextppos, minptr, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) elif op == OPCODE_MIN_REPEAT_ONE: # match repeated sequence (minimizing regexp). @@ -763,26 +776,26 @@ # use the MIN_REPEAT operator. # <1=min> <2=max> item tail start = ptr - min = ctx.pat(ppos+1) + min = pattern.pat(ppos+1) if min > 0: minptr = ptr + min if minptr > ctx.end: return # cannot match # count using pattern min as the maximum - ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks) + ptr = find_repetition_end(ctx, pattern, ppos+3, ptr, min, marks) if ptr < minptr: return # did not match minimum number of times maxptr = ctx.end - max = ctx.pat(ppos+2) + max = pattern.pat(ppos+2) if max != rsre_char.MAXREPEAT: maxptr1 = start + max if maxptr1 <= maxptr: maxptr = maxptr1 - nextppos = ppos + ctx.pat(ppos) + nextppos = ppos + pattern.pat(ppos) result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) else: raise Error("bad pattern code %d" % op) @@ -816,7 +829,7 @@ return True @specializectx -def find_repetition_end(ctx, ppos, ptr, maxcount, marks): +def find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks): end = ctx.end ptrp1 = ptr + 1 # First get rid of the cases where we don't have room for any match. @@ -826,16 +839,16 @@ # The idea is to be fast for cases like re.search("b+"), where we expect # the common case to be a non-match. It's much faster with the JIT to # have the non-match inlined here rather than detect it in the fre() call. - op = ctx.pat(ppos) + op = pattern.pat(ppos) for op1, checkerfn in unroll_char_checker: if op1 == op: - if checkerfn(ctx, ptr, ppos): + if checkerfn(ctx, pattern, ptr, ppos): break return ptr else: # obscure case: it should be a single char pattern, but isn't # one of the opcodes in unroll_char_checker (see test_ext_opcode) - return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks) + return general_find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks) # It matches at least once. If maxcount == 1 (relatively common), # then we are done. if maxcount == 1: @@ -846,14 +859,14 @@ end1 = ptr + maxcount if end1 <= end: end = end1 - op = ctx.pat(ppos) + op = pattern.pat(ppos) for op1, fre in unroll_fre_checker: if op1 == op: - return fre(ctx, ptrp1, end, ppos) + return fre(ctx, pattern, ptrp1, end, ppos) raise Error("rsre.find_repetition_end[%d]" % op) @specializectx -def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks): +def general_find_repetition_end(ctx, patern, ppos, ptr, maxcount, marks): # moved into its own JIT-opaque function end = ctx.end if maxcount != rsre_char.MAXREPEAT: @@ -861,63 +874,65 @@ end1 = ptr + maxcount if end1 <= end: end = end1 - while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None: + while ptr < end and sre_match(ctx, patern, ppos, ptr, marks) is not None: ptr += 1 return ptr @specializectx -def match_ANY(ctx, ptr, ppos): # dot wildcard. +def match_ANY(ctx, pattern, ptr, ppos): # dot wildcard. return not rsre_char.is_linebreak(ctx.str(ptr)) -def match_ANY_ALL(ctx, ptr, ppos): +def match_ANY_ALL(ctx, pattern, ptr, ppos): return True # match anything (including a newline) @specializectx -def match_IN(ctx, ptr, ppos): - return rsre_char.check_charset(ctx, ppos+2, ctx.str(ptr)) +def match_IN(ctx, pattern, ptr, ppos): + return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.str(ptr)) @specializectx -def match_IN_IGNORE(ctx, ptr, ppos): - return rsre_char.check_charset(ctx, ppos+2, ctx.lowstr(ptr)) +def match_IN_IGNORE(ctx, pattern, ptr, ppos): + return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.lowstr(ptr)) @specializectx -def match_LITERAL(ctx, ptr, ppos): - return ctx.str(ptr) == ctx.pat(ppos+1) +def match_LITERAL(ctx, pattern, ptr, ppos): + return ctx.str(ptr) == pattern.pat(ppos+1) @specializectx -def match_LITERAL_IGNORE(ctx, ptr, ppos): - return ctx.lowstr(ptr) == ctx.pat(ppos+1) +def match_LITERAL_IGNORE(ctx, pattern, ptr, ppos): + return ctx.lowstr(ptr) == pattern.pat(ppos+1) @specializectx -def match_NOT_LITERAL(ctx, ptr, ppos): - return ctx.str(ptr) != ctx.pat(ppos+1) +def match_NOT_LITERAL(ctx, pattern, ptr, ppos): + return ctx.str(ptr) != pattern.pat(ppos+1) @specializectx -def match_NOT_LITERAL_IGNORE(ctx, ptr, ppos): - return ctx.lowstr(ptr) != ctx.pat(ppos+1) +def match_NOT_LITERAL_IGNORE(ctx, pattern, ptr, ppos): + return ctx.lowstr(ptr) != pattern.pat(ppos+1) def _make_fre(checkerfn): if checkerfn == match_ANY_ALL: - def fre(ctx, ptr, end, ppos): + def fre(ctx, pattern, ptr, end, ppos): return end elif checkerfn == match_IN: install_jitdriver_spec('MatchIn', - greens=['ppos', 'ctx.pattern'], + greens=['ppos', 'pattern'], reds=['ptr', 'end', 'ctx'], debugprint=(1, 0)) @specializectx - def fre(ctx, ptr, end, ppos): + def fre(ctx, pattern, ptr, end, ppos): while True: ctx.jitdriver_MatchIn.jit_merge_point(ctx=ctx, ptr=ptr, - end=end, ppos=ppos) - if ptr < end and checkerfn(ctx, ptr, ppos): + end=end, ppos=ppos, + pattern=pattern) + if ptr < end and checkerfn(ctx, pattern, ptr, ppos): ptr += 1 else: return ptr elif checkerfn == match_IN_IGNORE: install_jitdriver_spec('MatchInIgnore', - greens=['ppos', 'ctx.pattern'], + greens=['ppos', 'pattern'], reds=['ptr', 'end', 'ctx'], debugprint=(1, 0)) @specializectx - def fre(ctx, ptr, end, ppos): + def fre(ctx, pattern, ptr, end, ppos): while True: ctx.jitdriver_MatchInIgnore.jit_merge_point(ctx=ctx, ptr=ptr, - end=end, ppos=ppos) - if ptr < end and checkerfn(ctx, ptr, ppos): + end=end, ppos=ppos, + pattern=pattern) + if ptr < end and checkerfn(ctx, pattern, ptr, ppos): ptr += 1 else: return ptr @@ -925,8 +940,8 @@ # in the other cases, the fre() function is not JITted at all # and is present as a residual call. @specializectx - def fre(ctx, ptr, end, ppos): - while ptr < end and checkerfn(ctx, ptr, ppos): + def fre(ctx, pattern, ptr, end, ppos): + while ptr < end and checkerfn(ctx, pattern, ptr, ppos): ptr += 1 return ptr fre = func_with_new_name(fre, 'fre_' + checkerfn.__name__) @@ -1037,10 +1052,11 @@ return start, end def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False): + assert isinstance(pattern, CompiledPattern) start, end = _adjust(start, end, len(string)) ctx = StrMatchContext(pattern, string, start, end, flags) ctx.fullmatch_only = fullmatch - if match_context(ctx): + if match_context(ctx, pattern): return ctx else: return None @@ -1049,105 +1065,106 @@ return match(pattern, string, start, end, flags, fullmatch=True) def search(pattern, string, start=0, end=sys.maxint, flags=0): + assert isinstance(pattern, CompiledPattern) start, end = _adjust(start, end, len(string)) ctx = StrMatchContext(pattern, string, start, end, flags) - if search_context(ctx): + if search_context(ctx, pattern): return ctx else: return None install_jitdriver('Match', - greens=['ctx.pattern'], reds=['ctx'], + greens=['pattern'], reds=['ctx'], debugprint=(0,)) -def match_context(ctx): +def match_context(ctx, pattern): ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: return False - ctx.jitdriver_Match.jit_merge_point(ctx=ctx) - return sre_match(ctx, 0, ctx.match_start, None) is not None + ctx.jitdriver_Match.jit_merge_point(ctx=ctx, pattern=pattern) + return sre_match(ctx, pattern, 0, ctx.match_start, None) is not None -def search_context(ctx): +def search_context(ctx, pattern): ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: return False base = 0 charset = False - if ctx.pat(base) == OPCODE_INFO: - flags = ctx.pat(2) + if pattern.pat(base) == OPCODE_INFO: + flags = pattern.pat(2) if flags & rsre_char.SRE_INFO_PREFIX: - if ctx.pat(5) > 1: - return fast_search(ctx) + if pattern.pat(5) > 1: + return fast_search(ctx, pattern) else: charset = (flags & rsre_char.SRE_INFO_CHARSET) - base += 1 + ctx.pat(1) - if ctx.pat(base) == OPCODE_LITERAL: - return literal_search(ctx, base) + base += 1 + pattern.pat(1) + if pattern.pat(base) == OPCODE_LITERAL: + return literal_search(ctx, pattern, base) if charset: - return charset_search(ctx, base) - return regular_search(ctx, base) + return charset_search(ctx, pattern, base) + return regular_search(ctx, pattern, base) install_jitdriver('RegularSearch', - greens=['base', 'ctx.pattern'], + greens=['base', 'pattern'], reds=['start', 'ctx'], debugprint=(1, 0)) -def regular_search(ctx, base): +def regular_search(ctx, pattern, base): start = ctx.match_start while start <= ctx.end: ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, start=start, - base=base) - if sre_match(ctx, base, start, None) is not None: + base=base, pattern=pattern) + if sre_match(ctx, pattern, base, start, None) is not None: ctx.match_start = start return True start += 1 return False install_jitdriver_spec("LiteralSearch", - greens=['base', 'character', 'ctx.pattern'], + greens=['base', 'character', 'pattern'], reds=['start', 'ctx'], debugprint=(2, 0, 1)) @specializectx -def literal_search(ctx, base): +def literal_search(ctx, pattern, base): # pattern starts with a literal character. this is used # for short prefixes, and if fast search is disabled - character = ctx.pat(base + 1) + character = pattern.pat(base + 1) base += 2 start = ctx.match_start while start < ctx.end: ctx.jitdriver_LiteralSearch.jit_merge_point(ctx=ctx, start=start, - base=base, character=character) + base=base, character=character, pattern=pattern) if ctx.str(start) == character: - if sre_match(ctx, base, start + 1, None) is not None: + if sre_match(ctx, pattern, base, start + 1, None) is not None: ctx.match_start = start return True start += 1 return False install_jitdriver_spec("CharsetSearch", - greens=['base', 'ctx.pattern'], + greens=['base', 'pattern'], reds=['start', 'ctx'], debugprint=(1, 0)) @specializectx -def charset_search(ctx, base): +def charset_search(ctx, pattern, base): # pattern starts with a character from a known set start = ctx.match_start while start < ctx.end: ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start, - base=base) - if rsre_char.check_charset(ctx, 5, ctx.str(start)): - if sre_match(ctx, base, start, None) is not None: + base=base, pattern=pattern) + if rsre_char.check_charset(ctx, pattern, 5, ctx.str(start)): + if sre_match(ctx, pattern, base, start, None) is not None: ctx.match_start = start return True start += 1 return False install_jitdriver_spec('FastSearch', - greens=['i', 'prefix_len', 'ctx.pattern'], + greens=['i', 'prefix_len', 'pattern'], reds=['string_position', 'ctx'], debugprint=(2, 0)) @specializectx -def fast_search(ctx): +def fast_search(ctx, pattern): # skips forward in a string as fast as possible using information from # an optimization info block # <1=skip> <2=flags> <3=min> <4=...> @@ -1155,17 +1172,18 @@ string_position = ctx.match_start if string_position >= ctx.end: return False - prefix_len = ctx.pat(5) + prefix_len = pattern.pat(5) assert prefix_len >= 0 i = 0 while True: ctx.jitdriver_FastSearch.jit_merge_point(ctx=ctx, - string_position=string_position, i=i, prefix_len=prefix_len) + string_position=string_position, i=i, prefix_len=prefix_len, + pattern=pattern) char_ord = ctx.str(string_position) - if char_ord != ctx.pat(7 + i): + if char_ord != pattern.pat(7 + i): if i > 0: overlap_offset = prefix_len + (7 - 1) - i = ctx.pat(overlap_offset + i) + i = pattern.pat(overlap_offset + i) continue else: i += 1 @@ -1173,22 +1191,22 @@ # found a potential match start = string_position + 1 - prefix_len assert start >= 0 - prefix_skip = ctx.pat(6) + prefix_skip = pattern.pat(6) ptr = start + prefix_skip - #flags = ctx.pat(2) + #flags = pattern.pat(2) #if flags & rsre_char.SRE_INFO_LITERAL: # # matched all of pure literal pattern # ctx.match_start = start # ctx.match_end = ptr # ctx.match_marks = None # return True - pattern_offset = ctx.pat(1) + 1 + pattern_offset = pattern.pat(1) + 1 ppos_start = pattern_offset + 2 * prefix_skip - if sre_match(ctx, ppos_start, ptr, None) is not None: + if sre_match(ctx, pattern, ppos_start, ptr, None) is not None: ctx.match_start = start return True overlap_offset = prefix_len + (7 - 1) - i = ctx.pat(overlap_offset + i) + i = pattern.pat(overlap_offset + i) string_position += 1 if string_position >= ctx.end: return False diff --git a/rpython/rlib/rsre/test/test_char.py b/rpython/rlib/rsre/test/test_char.py --- a/rpython/rlib/rsre/test/test_char.py +++ b/rpython/rlib/rsre/test/test_char.py @@ -1,10 +1,16 @@ -from rpython.rlib.rsre import rsre_char +from rpython.rlib.rsre import rsre_char, rsre_core from rpython.rlib.rsre.rsre_char import SRE_FLAG_LOCALE, SRE_FLAG_UNICODE def setup_module(mod): from rpython.rlib.unicodedata import unicodedb rsre_char.set_unicode_db(unicodedb) + +def check_charset(pattern, idx, char): + p = rsre_core.CompiledPattern(pattern) + return rsre_char.check_charset(Ctx(p), p, idx, char) + + UPPER_PI = 0x3a0 LOWER_PI = 0x3c0 INDIAN_DIGIT = 0x966 @@ -157,12 +163,12 @@ pat_neg = [70, ord(cat) | 0x80, 0] for c in positive: assert unicodedb.category(ord(c)).startswith(cat) - assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert check_charset(pat_pos, 0, ord(c)) + assert not check_charset(pat_neg, 0, ord(c)) for c in negative: assert not unicodedb.category(ord(c)).startswith(cat) - assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert not check_charset(pat_pos, 0, ord(c)) + assert check_charset(pat_neg, 0, ord(c)) def cat2num(cat): return ord(cat[0]) | (ord(cat[1]) << 8) @@ -173,17 +179,16 @@ pat_neg = [70, cat2num(cat) | 0x80, 0] for c in positive: assert unicodedb.category(ord(c)) == cat - assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert check_charset(pat_pos, 0, ord(c)) + assert not check_charset(pat_neg, 0, ord(c)) for c in negative: assert unicodedb.category(ord(c)) != cat - assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert not check_charset(pat_pos, 0, ord(c)) + assert check_charset(pat_neg, 0, ord(c)) # test for how the common 'L&' pattern might be compiled pat = [70, cat2num('Lu'), 70, cat2num('Ll'), 70, cat2num('Lt'), 0] - assert rsre_char.check_charset(Ctx(pat), 0, 65) # Lu - assert rsre_char.check_charset(Ctx(pat), 0, 99) # Ll - assert rsre_char.check_charset(Ctx(pat), 0, 453) # Lt - assert not rsre_char.check_charset(Ctx(pat), 0, 688) # Lm - assert not rsre_char.check_charset(Ctx(pat), 0, 5870) # Nl + assert check_charset(pat, 0, 65) # Lu + assert check_charset(pat, 0, 99) # Lcheck_charset(pat, 0, 453) # Lt + assert not check_charset(pat, 0, 688) # Lm + assert not check_charset(pat, 0, 5870) # Nl diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py --- a/rpython/rlib/rsre/test/test_ext_opcode.py +++ b/rpython/rlib/rsre/test/test_ext_opcode.py @@ -17,10 +17,10 @@ # it's a valid optimization because \1 is always one character long r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, GROUPREF, 0, SUCCESS, SUCCESS] - assert rsre_core.match(r, "aaa").match_end == 3 + assert rsre_core.match(rsre_core.CompiledPattern(r), "aaa").match_end == 3 def test_min_repeat_one_with_backref(): # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT, GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS] - assert rsre_core.match(r, "aaab").match_end == 4 + assert rsre_core.match(rsre_core.CompiledPattern(r), "aaab").match_end == 4 diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py --- a/rpython/rlib/rsre/test/test_match.py +++ b/rpython/rlib/rsre/test/test_match.py @@ -9,7 +9,7 @@ def test_get_code_repetition(): c1 = get_code(r"a+") c2 = get_code(r"a+") - assert c1 == c2 + assert c1.pattern == c2.pattern class TestMatch: @@ -305,6 +305,6 @@ rsre_char.set_unicode_db(unicodedb) # r = get_code(u"[\U00010428-\U0001044f]", re.I) - assert r.count(27) == 1 # OPCODE_RANGE - r[r.index(27)] = 32 # => OPCODE_RANGE_IGNORE + assert r.pattern.count(27) == 1 # OPCODE_RANGE + r.pattern[r.pattern.index(27)] = 32 # => OPCODE_RANGE_IGNORE assert rsre_core.match(r, u"\U00010428") diff --git a/rpython/rlib/rsre/test/test_re.py b/rpython/rlib/rsre/test/test_re.py --- a/rpython/rlib/rsre/test/test_re.py +++ b/rpython/rlib/rsre/test/test_re.py @@ -426,31 +426,6 @@ assert pat.match(p) is not None assert pat.match(p).span() == (0,256) - def test_pickling(self): - import pickle - self.pickle_test(pickle) - import cPickle - self.pickle_test(cPickle) - # old pickles expect the _compile() reconstructor in sre module - import warnings - original_filters = warnings.filters[:] - try: - warnings.filterwarnings("ignore", "The sre module is deprecated", - DeprecationWarning) - from sre import _compile - finally: - warnings.filters = original_filters - - def pickle_test(self, pickle): - oldpat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)') - s = pickle.dumps(oldpat) - newpat = pickle.loads(s) - # Not using object identity for _sre.py, since some Python builds do - # not seem to preserve that in all cases (observed on an UCS-4 build - # of 2.4.1). - #self.assertEqual(oldpat, newpat) - assert oldpat.__dict__ == newpat.__dict__ - def test_constants(self): assert re.I == re.IGNORECASE assert re.L == re.LOCALE diff --git a/rpython/rlib/rsre/test/test_zinterp.py b/rpython/rlib/rsre/test/test_zinterp.py --- a/rpython/rlib/rsre/test/test_zinterp.py +++ b/rpython/rlib/rsre/test/test_zinterp.py @@ -11,6 +11,7 @@ rsre_core.search(pattern, string) # unicodestr = unichr(n) * n + pattern = rsre_core.CompiledPattern(pattern) ctx = rsre_core.UnicodeMatchContext(pattern, unicodestr, 0, len(unicodestr), 0) rsre_core.search_context(ctx) diff --git a/rpython/rlib/rsre/test/test_zjit.py b/rpython/rlib/rsre/test/test_zjit.py --- a/rpython/rlib/rsre/test/test_zjit.py +++ b/rpython/rlib/rsre/test/test_zjit.py @@ -6,7 +6,7 @@ from rpython.rtyper.annlowlevel import llstr, hlstr def entrypoint1(r, string, repeat): - r = array2list(r) + r = rsre_core.CompiledPattern(array2list(r)) string = hlstr(string) match = None for i in range(repeat): @@ -17,7 +17,7 @@ return match.match_end def entrypoint2(r, string, repeat): - r = array2list(r) + r = rsre_core.CompiledPattern(array2list(r)) string = hlstr(string) match = None for i in range(repeat): @@ -48,13 +48,13 @@ def meta_interp_match(self, pattern, string, repeat=1): r = get_code(pattern) - return self.meta_interp(entrypoint1, [list2array(r), llstr(string), + return self.meta_interp(entrypoint1, [list2array(r.pattern), llstr(string), repeat], listcomp=True, backendopt=True) def meta_interp_search(self, pattern, string, repeat=1): r = get_code(pattern) - return self.meta_interp(entrypoint2, [list2array(r), llstr(string), + return self.meta_interp(entrypoint2, [list2array(r.pattern), llstr(string), repeat], listcomp=True, backendopt=True) From pypy.commits at gmail.com Mon Mar 26 09:51:32 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Mar 2018 06:51:32 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: the context doesn't need the pattern any more Message-ID: <5ab8fae4.42da1c0a.ae2ea.4bd5@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94134:c553f2e29955 Date: 2018-03-26 14:44 +0200 http://bitbucket.org/pypy/pypy/changeset/c553f2e29955/ Log: the context doesn't need the pattern any more diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -114,7 +114,7 @@ pos = len(unicodestr) if endpos > len(unicodestr): endpos = len(unicodestr) - return rsre_core.UnicodeMatchContext(self.code, unicodestr, + return rsre_core.UnicodeMatchContext(unicodestr, pos, endpos, self.flags) elif space.isinstance_w(w_string, space.w_bytes): str = space.bytes_w(w_string) @@ -122,7 +122,7 @@ pos = len(str) if endpos > len(str): endpos = len(str) - return rsre_core.StrMatchContext(self.code, str, + return rsre_core.StrMatchContext(str, pos, endpos, self.flags) else: buf = space.readbuf_w(w_string) @@ -132,7 +132,7 @@ pos = size if endpos > size: endpos = size - return rsre_core.BufMatchContext(self.code, buf, + return rsre_core.BufMatchContext(buf, pos, endpos, self.flags) def getmatch(self, ctx, found): diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -107,20 +107,18 @@ class AbstractMatchContext(object): """Abstract base class""" - _immutable_fields_ = ['pattern', 'flags', 'end'] + _immutable_fields_ = ['flags', 'end'] match_start = 0 match_end = 0 match_marks = None match_marks_flat = None fullmatch_only = False - def __init__(self, pattern, match_start, end, flags): + def __init__(self, match_start, end, flags): # 'match_start' and 'end' must be known to be non-negative # and they must not be more than len(string). check_nonneg(match_start) check_nonneg(end) - assert isinstance(pattern, CompiledPattern) - self.pattern = pattern self.match_start = match_start self.end = end self.flags = flags @@ -192,8 +190,8 @@ _immutable_fields_ = ["_buffer"] - def __init__(self, pattern, buf, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, buf, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._buffer = buf def str(self, index): @@ -205,7 +203,7 @@ return rsre_char.getlower(c, self.flags) def fresh_copy(self, start): - return BufMatchContext(self.pattern, self._buffer, start, + return BufMatchContext(self._buffer, start, self.end, self.flags) class StrMatchContext(AbstractMatchContext): @@ -213,8 +211,8 @@ _immutable_fields_ = ["_string"] - def __init__(self, pattern, string, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, string, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._string = string if not we_are_translated() and isinstance(string, unicode): self.flags |= rsre_char.SRE_FLAG_UNICODE # for rsre_re.py @@ -228,7 +226,7 @@ return rsre_char.getlower(c, self.flags) def fresh_copy(self, start): - return StrMatchContext(self.pattern, self._string, start, + return StrMatchContext(self._string, start, self.end, self.flags) class UnicodeMatchContext(AbstractMatchContext): @@ -236,8 +234,8 @@ _immutable_fields_ = ["_unicodestr"] - def __init__(self, pattern, unicodestr, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, unicodestr, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._unicodestr = unicodestr def str(self, index): @@ -249,7 +247,7 @@ return rsre_char.getlower(c, self.flags) def fresh_copy(self, start): - return UnicodeMatchContext(self.pattern, self._unicodestr, start, + return UnicodeMatchContext(self._unicodestr, start, self.end, self.flags) # ____________________________________________________________ @@ -1054,7 +1052,7 @@ def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False): assert isinstance(pattern, CompiledPattern) start, end = _adjust(start, end, len(string)) - ctx = StrMatchContext(pattern, string, start, end, flags) + ctx = StrMatchContext(string, start, end, flags) ctx.fullmatch_only = fullmatch if match_context(ctx, pattern): return ctx @@ -1067,7 +1065,7 @@ def search(pattern, string, start=0, end=sys.maxint, flags=0): assert isinstance(pattern, CompiledPattern) start, end = _adjust(start, end, len(string)) - ctx = StrMatchContext(pattern, string, start, end, flags) + ctx = StrMatchContext(string, start, end, flags) if search_context(ctx, pattern): return ctx else: From pypy.commits at gmail.com Mon Mar 26 12:01:48 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Mar 2018 09:01:48 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: fix issue 2777: Message-ID: <5ab9196c.8e6f1c0a.de05.7014@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94135:b3264f1f3592 Date: 2018-03-26 18:01 +0200 http://bitbucket.org/pypy/pypy/changeset/b3264f1f3592/ Log: fix issue 2777: before this commit, the following problem could occur: when the tags of the opencoder overflow, it raises SwitchToBlackHole in the middle of a jitcode opcode. Potentially, in the middle of a guard, *before* the guard changed the pc on the MIFrame! That way, the blackhole interpreter would continue in the wrong branch :-(. Fix this by changing the interface slightly: opencoder.Trace.record will now just not record anything, if the tags overflow. Instead, it will set a flag on itself, which the metatracer needs to check after every opcode, to find out whether to stop tracing. diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -701,6 +701,9 @@ def length(self): return self.trace._count - len(self.trace.inputargs) + def trace_tag_overflow(self): + return self.trace.tag_overflow + def get_trace_position(self): return self.trace.cut_point() diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -293,6 +293,7 @@ self._start = len(inputargs) self._pos = self._start self.inputargs = inputargs + self.tag_overflow = False def append(self, v): model = get_model(self) @@ -300,7 +301,8 @@ # grow by 2X self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops) if not model.MIN_VALUE <= v <= model.MAX_VALUE: - raise frontend_tag_overflow() + v = 0 # broken value, but that's fine, tracing will stop soon + self.tag_overflow = True self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v) self._pos += 1 @@ -379,6 +381,7 @@ def record_op(self, opnum, argboxes, descr=None): pos = self._index + old_pos = self._pos self.append(opnum) expected_arity = oparity[opnum] if expected_arity == -1: @@ -397,6 +400,10 @@ self._count += 1 if opclasses[opnum].type != 'v': self._index += 1 + if self.tag_overflow: + # potentially a broken op is left behind + # clean it up + self._pos = old_pos return pos def _encode_descr(self, descr): @@ -424,10 +431,11 @@ vref_array = self._list_of_boxes(vref_boxes) s = TopSnapshot(combine_uint(jitcode.index, pc), array, vable_array, vref_array) - assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 # guards have no descr self._snapshots.append(s) - self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) + if not self.tag_overflow: # otherwise we're broken anyway + assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 + self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) return s def create_empty_top_snapshot(self, vable_boxes, vref_boxes): @@ -436,10 +444,11 @@ vref_array = self._list_of_boxes(vref_boxes) s = TopSnapshot(combine_uint(2**16 - 1, 0), [], vable_array, vref_array) - assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 # guards have no descr self._snapshots.append(s) - self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) + if not self.tag_overflow: # otherwise we're broken anyway + assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 + self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) return s def create_snapshot(self, jitcode, pc, frame, flag): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2384,7 +2384,8 @@ def blackhole_if_trace_too_long(self): warmrunnerstate = self.jitdriver_sd.warmstate - if self.history.length() > warmrunnerstate.trace_limit: + if (self.history.length() > warmrunnerstate.trace_limit or + self.history.trace_tag_overflow()): jd_sd, greenkey_of_huge_function = self.find_biggest_function() self.history.trace.done() self.staticdata.stats.record_aborted(greenkey_of_huge_function) diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4661,3 +4661,36 @@ f() # finishes self.meta_interp(f, []) + + def test_trace_too_long_bug(self): + driver = JitDriver(greens=[], reds=['i']) + @unroll_safe + def match(s): + l = len(s) + p = 0 + for i in range(2500): # produces too long trace + c = s[p] + if c != 'a': + return False + p += 1 + if p >= l: + return True + c = s[p] + if c != '\n': + p += 1 + if p >= l: + return True + else: + return False + return True + + def f(i): + while i > 0: + driver.jit_merge_point(i=i) + match('a' * (500 * i)) + i -= 1 + return i + + res = self.meta_interp(f, [10]) + assert res == f(10) + diff --git a/rpython/jit/metainterp/test/test_opencoder.py b/rpython/jit/metainterp/test/test_opencoder.py --- a/rpython/jit/metainterp/test/test_opencoder.py +++ b/rpython/jit/metainterp/test/test_opencoder.py @@ -209,5 +209,8 @@ def test_tag_overflow(self): t = Trace([], metainterp_sd) i0 = FakeOp(100000) - py.test.raises(SwitchToBlackhole, t.record_op, rop.FINISH, [i0]) - assert t.unpack() == ([], []) + # if we overflow, we can keep recording + for i in range(10): + t.record_op(rop.FINISH, [i0]) + assert t.unpack() == ([], []) + assert t.tag_overflow diff --git a/rpython/rlib/rsre/test/test_zjit.py b/rpython/rlib/rsre/test/test_zjit.py --- a/rpython/rlib/rsre/test/test_zjit.py +++ b/rpython/rlib/rsre/test/test_zjit.py @@ -11,6 +11,8 @@ match = None for i in range(repeat): match = rsre_core.match(r, string) + if match is None: + return -1 if match is None: return -1 else: @@ -166,3 +168,9 @@ res = self.meta_interp_search(r"b+", "a"*30 + "b") assert res == 30 self.check_resops(call=0) + + def test_match_jit_bug(self): + pattern = ".a" * 2500 + text = "a" * 6000 + res = self.meta_interp_match(pattern, text, repeat=10) + assert res != -1 From pypy.commits at gmail.com Mon Mar 26 12:18:56 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Mar 2018 09:18:56 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: disable support for greenfields Message-ID: <5ab91d70.95921c0a.f4ca.2040@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94136:fc18cb1d88b2 Date: 2018-03-26 18:18 +0200 http://bitbucket.org/pypy/pypy/changeset/fc18cb1d88b2/ Log: disable support for greenfields diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py --- a/rpython/jit/metainterp/test/test_greenfield.py +++ b/rpython/jit/metainterp/test/test_greenfield.py @@ -1,6 +1,17 @@ +import pytest from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rlib.jit import JitDriver, assert_green +pytest.skip("this feature is disabled at the moment!") + +# note why it is disabled: before d721da4573ad +# there was a failing assert when inlining python -> sre -> python: +# https://bitbucket.org/pypy/pypy/issues/2775/ +# this shows, that the interaction of greenfields and virtualizables is broken, +# because greenfields use MetaInterp.virtualizable_boxes, which confuses +# MetaInterp._nonstandard_virtualizable somehow (and makes no sense +# conceptually anyway). to fix greenfields, the two mechanisms would have to be +# disentangled. class GreenFieldsTests: diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -653,6 +653,9 @@ self._make_extregistryentries() assert get_jitcell_at is None, "get_jitcell_at no longer used" assert set_jitcell_at is None, "set_jitcell_at no longer used" + for green in self.greens: + if "." in green: + raise ValueError("green fields are buggy! if you need them fixed, please talk to us") self.get_printable_location = get_printable_location self.get_location = get_location self.has_unique_id = (get_unique_id is not None) From pypy.commits at gmail.com Mon Mar 26 13:21:01 2018 From: pypy.commits at gmail.com (cfbolz) Date: Mon, 26 Mar 2018 10:21:01 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: remove another green field Message-ID: <5ab92bfd.a9b6df0a.43965.9d5d@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94137:2e575fee8b75 Date: 2018-03-26 19:20 +0200 http://bitbucket.org/pypy/pypy/changeset/2e575fee8b75/ Log: remove another green field diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -232,7 +232,9 @@ "different from the 'ffi.h' file seen at compile-time)") def py_invoke(self, ll_res, ll_args): + key_pycode = self.key_pycode jitdriver1.jit_merge_point(callback=self, + key_pycode=key_pycode, ll_res=ll_res, ll_args=ll_args) self.do_invoke(ll_res, ll_args) @@ -294,7 +296,7 @@ return 'cffi_callback ' + key_pycode.get_repr() jitdriver1 = jit.JitDriver(name='cffi_callback', - greens=['callback.key_pycode'], + greens=['key_pycode'], reds=['ll_res', 'll_args', 'callback'], get_printable_location=get_printable_location1) From pypy.commits at gmail.com Mon Mar 26 18:12:49 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Mar 2018 15:12:49 -0700 (PDT) Subject: [pypy-commit] pypy default: minimize diff from py3 Message-ID: <5ab97061.94101c0a.24829.00d8@mx.google.com> Author: Matti Picus Branch: Changeset: r94139:bb10e77aa623 Date: 2018-03-27 01:08 +0300 http://bitbucket.org/pypy/pypy/changeset/bb10e77aa623/ Log: minimize diff from py3 diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1,3 +1,4 @@ +import pytest from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import rffi from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -6,6 +7,7 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import PyTypeObjectPtr + class AppTestTypeObject(AppTestCpythonExtensionBase): def setup_class(cls): @@ -136,8 +138,10 @@ module = self.import_module(name='foo') descr = module.fooType.copy assert type(descr).__name__ == 'method_descriptor' - assert str(descr) == "" - assert repr(descr) == "" + assert str(descr) in ("", + "") + assert repr(descr) in ("", + "") raises(TypeError, descr, None) def test_cython_fake_classmethod(self): @@ -250,7 +254,7 @@ import re assert re.sre_compile._sre is module s = u"Foo " * 1000 + u"Bar" - prog = re.compile(ur"Foo.*Bar") + prog = re.compile(u"Foo.*Bar") assert prog.match(s) m = re.search(u"xyz", u"xyzxyz") assert m @@ -319,7 +323,7 @@ def test_tp_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("read_tp_dict", "METH_O", + ("read_tp_dict", "METH_O", ''' PyObject *method; if (!args->ob_type->tp_dict) @@ -420,7 +424,7 @@ return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject*)type, "a"); + value = PyObject_GetAttrString((PyObject *)type, "a"); Py_DECREF(value); if (PyDict_SetItemString(type->tp_dict, "a", @@ -428,7 +432,7 @@ return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject*)type, "a"); + value = PyObject_GetAttrString((PyObject *)type, "a"); return value; ''' ) @@ -529,7 +533,7 @@ py_type = rffi.cast(PyTypeObjectPtr, ref) w_dict = from_ref(space, py_type.c_tp_dict) - w_name = space.wrap('a') + w_name = space.newtext('a') space.setitem(w_dict, w_name, space.wrap(1)) assert space.int_w(space.getattr(w_class, w_name)) == 1 space.delitem(w_dict, w_name) @@ -611,16 +615,21 @@ module = self.import_extension('foo', [ ("test_tp_getattro", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #define PyIntObject PyLongObject + #define PyInt_AsLong PyLong_AsLong + #endif PyObject *name, *obj = PyTuple_GET_ITEM(args, 0); - PyIntObject *attr, *value = (PyIntObject*) PyTuple_GET_ITEM(args, 1); + PyObject *attr, *value = PyTuple_GET_ITEM(args, 1); if (!obj->ob_type->tp_getattro) { PyErr_SetString(PyExc_ValueError, "missing tp_getattro"); return NULL; } name = PyString_FromString("attr1"); - attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name); - if (attr->ob_ival != value->ob_ival) + attr = obj->ob_type->tp_getattro(obj, name); + if (PyInt_AsLong(attr) != PyInt_AsLong(value)) { PyErr_SetString(PyExc_ValueError, "tp_getattro returned wrong value"); @@ -629,7 +638,7 @@ Py_DECREF(name); Py_DECREF(attr); name = PyString_FromString("attr2"); - attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name); + attr = obj->ob_type->tp_getattro(obj, name); if (attr == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -652,6 +661,9 @@ module = self.import_extension('foo', [ ("get_foo", "METH_O", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #endif char* name = "foo"; PyTypeObject *tp = Py_TYPE(args); PyObject *res; @@ -836,6 +848,10 @@ ''' )], prologue=''' static int + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #define PyInt_Check PyLong_Check + #endif mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value) { if (PyInt_Check(key)) { @@ -898,6 +914,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) { @@ -935,6 +955,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { @@ -983,6 +1007,9 @@ ), ("tp_iternext", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #endif PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0); PyObject *obj = PyTuple_GET_ITEM(args, 1); PyObject *result; @@ -1002,7 +1029,7 @@ it = module.tp_iter(list, l) assert type(it) is type(iter([])) assert module.tp_iternext(type(it), it) == 1 - assert module.tp_iternext(type(it), it) == "stop!" + assert module.tp_iternext(type(it), it) == b"stop!" # class LL(list): def __iter__(self): @@ -1132,7 +1159,11 @@ PyObject_HEAD long ival; } IntLikeObject; - + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #define PyInt_FromLong PyLong_FromLong + #endif static PyObject * intlike_nb_add(PyObject *self, PyObject *other) { @@ -1476,7 +1507,6 @@ )]) # used to segfault after some iterations for i in range(11): - print i class A(object): pass class B: @@ -1488,10 +1518,10 @@ def test_getattr_getattro(self): module = self.import_module(name='foo') - assert module.gettype2.dcba == 'getattro:dcba' + assert module.gettype2.dcba == b'getattro:dcba' assert (type(module.gettype2).__getattribute__(module.gettype2, 'dcBA') - == 'getattro:dcBA') - assert module.gettype1.abcd == 'getattr:abcd' + == b'getattro:dcBA') + assert module.gettype1.abcd == b'getattr:abcd' # GetType1 objects have a __getattribute__ method, but this # doesn't call tp_getattr at all, also on CPython raises(AttributeError, type(module.gettype1).__getattribute__, @@ -1533,6 +1563,9 @@ return PyInt_FromLong(42); ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #endif static PyTypeObject Foo_Type = { PyVarObject_HEAD_INIT(NULL, 0) "foo.foo", @@ -1635,8 +1668,10 @@ (int, Py_TPFLAGS_INT_SUBCLASS), (list, Py_TPFLAGS_LIST_SUBCLASS), (tuple, Py_TPFLAGS_TUPLE_SUBCLASS), + (bytes, Py_TPFLAGS_STRING_SUBCLASS), (str, Py_TPFLAGS_STRING_SUBCLASS), (unicode, Py_TPFLAGS_UNICODE_SUBCLASS), + (dict, Py_TPFLAGS_DICT_SUBCLASS), (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS), (type, Py_TPFLAGS_TYPE_SUBCLASS), ): @@ -1664,7 +1699,7 @@ return PyLong_FromLong(0); '''),]) # copied from object.h - Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1<<0) class MyFloat(float): pass diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py --- a/pypy/module/cpyext/userslot.py +++ b/pypy/module/cpyext/userslot.py @@ -49,6 +49,11 @@ w_stararg=w_args, w_starstararg=w_kwds) return space.call_args(w_impl, args) + at slot_function([PyObject, PyObject, PyObject], PyObject) +def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(w_self, args) + # unary functions @slot_function([PyObject], PyObject) From pypy.commits at gmail.com Mon Mar 26 18:13:48 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Mar 2018 15:13:48 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix merge, minimize diff to default Message-ID: <5ab9709c.ee85df0a.b2a0b.74d4@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94141:fcce464367d8 Date: 2018-03-27 01:05 +0300 http://bitbucket.org/pypy/pypy/changeset/fcce464367d8/ Log: fix merge, minimize diff to default diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1,3 +1,4 @@ +import pytest from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import rffi from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -6,8 +7,6 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr -import sys -import pytest class AppTestTypeObject(AppTestCpythonExtensionBase): @@ -243,6 +242,29 @@ module = self.import_module(name='foo') raises(TypeError, module.MetaType, 'other', (module.fooType,), {}) + def test_sre(self): + import sys + for m in ['_sre', 'sre_compile', 'sre_constants', 'sre_parse', 're']: + # clear out these modules + try: + del sys.modules[m] + except KeyError: + pass + module = self.import_module(name='_sre') + import re + assert re.sre_compile._sre is module + s = u"Foo " * 1000 + u"Bar" + prog = re.compile(u"Foo.*Bar") + assert prog.match(s) + m = re.search(u"xyz", u"xyzxyz") + assert m + m = re.search("xyz", "xyzxyz") + assert m + assert "groupdict" in dir(m) + re._cache.clear() + re._cache_repl.clear() + del prog, m + def test_init_error(self): module = self.import_module("foo") raises(ValueError, module.InitErrType) @@ -519,7 +541,7 @@ py_type = rffi.cast(PyTypeObjectPtr, ref) w_dict = from_ref(space, py_type.c_tp_dict) - w_name = space.newunicode(u'a') + w_name = space.newtext('a') space.setitem(w_dict, w_name, space.wrap(1)) assert space.int_w(space.getattr(w_class, w_name)) == 1 space.delitem(w_dict, w_name) @@ -549,6 +571,7 @@ def test_typeslots(self, space): assert cts.macros['Py_tp_doc'] == 56 + class AppTestSlots(AppTestCpythonExtensionBase): def setup_class(cls): AppTestCpythonExtensionBase.setup_class.im_func(cls) @@ -603,16 +626,21 @@ module = self.import_extension('foo', [ ("test_tp_getattro", "METH_VARARGS", ''' - PyObject *obj = PyTuple_GET_ITEM(args, 0); - PyObject *value = PyTuple_GET_ITEM(args, 1); + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #define PyIntObject PyLongObject + #define PyInt_AsLong PyLong_AsLong + #endif + PyObject *name, *obj = PyTuple_GET_ITEM(args, 0); + PyObject *attr, *value = PyTuple_GET_ITEM(args, 1); if (!obj->ob_type->tp_getattro) { PyErr_SetString(PyExc_ValueError, "missing tp_getattro"); return NULL; } - PyObject *name = PyUnicode_FromString("attr1"); - PyObject *attr = obj->ob_type->tp_getattro(obj, name); - if (PyLong_AsLong(attr) != PyLong_AsLong(value)) + name = PyString_FromString("attr1"); + attr = obj->ob_type->tp_getattro(obj, name); + if (PyInt_AsLong(attr) != PyInt_AsLong(value)) { PyErr_SetString(PyExc_ValueError, "tp_getattro returned wrong value"); @@ -620,7 +648,7 @@ } Py_DECREF(name); Py_DECREF(attr); - name = PyUnicode_FromString("attr2"); + name = PyString_FromString("attr2"); attr = obj->ob_type->tp_getattro(obj, name); if (attr == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -644,6 +672,9 @@ module = self.import_extension('foo', [ ("get_foo", "METH_O", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #endif char* name = "foo"; PyTypeObject *tp = Py_TYPE(args); PyObject *res; @@ -651,7 +682,7 @@ res = (*tp->tp_getattr)(args, name); } else if (tp->tp_getattro != NULL) { - PyObject *w = PyUnicode_FromString(name); + PyObject *w = PyString_FromString(name); res = (*tp->tp_getattro)(args, w); Py_DECREF(w); } @@ -736,17 +767,23 @@ module = self.import_extension('foo', [ ("tp_call", "METH_VARARGS", ''' - PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0); - PyObject *obj = PyTuple_GET_ITEM(args, 1); - PyObject *c_args = PyTuple_GET_ITEM(args, 2); - if (!type->tp_call) - { - PyErr_SetNone(PyExc_ValueError); - return NULL; - } - return type->tp_call(obj, c_args, NULL); - ''')]) - + PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0); + PyObject *obj = PyTuple_GET_ITEM(args, 1); + PyObject *c_args = PyTuple_GET_ITEM(args, 2); + if (!type->tp_call) + { + PyErr_SetNone(PyExc_ValueError); + return NULL; + } + return type->tp_call(obj, c_args, NULL); + ''' + ) + ]) + class C: + def __call__(self, *args): + return args + ret = module.tp_call(C, C(), ('x', 2)) + assert ret == ('x', 2) class D(type): def __call__(self, *args): return "foo! %r" % (args,) @@ -823,9 +860,13 @@ ''' )], prologue=''' static int + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #define PyInt_Check PyLong_Check + #endif mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value) { - if (PyLong_Check(key)) { + if (PyInt_Check(key)) { PyErr_SetNone(PyExc_ZeroDivisionError); return -1; } @@ -876,43 +917,6 @@ res = "foo" in obj assert res is True - def test_sq_ass_slice(self): - module = self.import_extension('foo', [ - ("new_obj", "METH_NOARGS", - ''' - PyObject *obj; - obj = PyObject_New(PyObject, &Foo_Type); - return obj; - ''' - )], prologue=''' - static int - sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) - { - int expected = (a == 10 && b == 20 && - PyInt_Check(o) && PyInt_AsLong(o) == 42); - if (!expected) { - PyErr_SetString(PyExc_ValueError, "test failed"); - return -1; - } - return 0; - } - PySequenceMethods tp_as_sequence; - static PyTypeObject Foo_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "foo.foo", - }; - ''', more_init=''' - Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; - Foo_Type.tp_as_sequence = &tp_as_sequence; - tp_as_sequence.sq_ass_slice = sq_ass_slice; - if (PyType_Ready(&Foo_Type) < 0) INITERROR; - ''') - obj = module.new_obj() - obj[10:20] = 42 - raises(ValueError, "obj[10:20] = 43") - raises(ValueError, "obj[11:20] = 42") - raises(ValueError, "obj[10:21] = 42") - def test_sq_ass_item(self): module = self.import_extension('foo', [ ("new_obj", "METH_NOARGS", @@ -922,6 +926,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { @@ -965,9 +973,14 @@ PyErr_SetNone(PyExc_ValueError); return NULL; } - return type->tp_iter(obj);'''), + return type->tp_iter(obj); + ''' + ), ("tp_iternext", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #endif PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0); PyObject *obj = PyTuple_GET_ITEM(args, 1); PyObject *result; @@ -980,13 +993,16 @@ /* In py3, returning NULL from tp_iternext means the iterator * is exhausted */ if (!result && !PyErr_Occurred()) - result = PyBytes_FromString("stop!"); - return result;''')]) + result = PyString_FromString("stop!"); + return result; + ''' + ) + ]) l = [1] it = module.tp_iter(list, l) assert type(it) is type(iter([])) assert module.tp_iternext(type(it), it) == 1 - assert module.tp_iternext(type(it), it) == b'stop!' + assert module.tp_iternext(type(it), it) == b"stop!" # class LL(list): def __iter__(self): @@ -1116,31 +1132,35 @@ PyObject_HEAD long ival; } IntLikeObject; - + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #define PyInt_FromLong PyLong_FromLong + #endif static PyObject * intlike_nb_add(PyObject *self, PyObject *other) { long val2, val1 = ((IntLikeObject *)(self))->ival; - if (PyLong_Check(other)) { - long val2 = PyLong_AsLong(other); - return PyLong_FromLong(val1+val2); + if (PyInt_Check(other)) { + long val2 = PyInt_AsLong(other); + return PyInt_FromLong(val1+val2); } val2 = ((IntLikeObject *)(other))->ival; - return PyLong_FromLong(val1+val2); + return PyInt_FromLong(val1+val2); } static PyObject * intlike_nb_pow(PyObject *self, PyObject *other, PyObject * z) { long val2, val1 = ((IntLikeObject *)(self))->ival; - if (PyLong_Check(other)) { - long val2 = PyLong_AsLong(other); - return PyLong_FromLong(val1+val2); + if (PyInt_Check(other)) { + long val2 = PyInt_AsLong(other); + return PyInt_FromLong(val1+val2); } val2 = ((IntLikeObject *)(other))->ival; - return PyLong_FromLong((int)pow(val1,val2)); + return PyInt_FromLong((int)pow(val1,val2)); } PyTypeObject IntLike_Type = { @@ -1493,9 +1513,12 @@ ''' ), ("forty_two", "METH_O", ''' - return PyLong_FromLong(42); + return PyInt_FromLong(42); ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #endif static PyTypeObject Foo_Type = { PyVarObject_HEAD_INIT(NULL, 0) "foo.foo", @@ -1626,9 +1649,10 @@ return PyLong_FromLong(0); '''),]) # copied from object.h - Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1<<0) class MyFloat(float): pass assert module.test_pypy_flags(float, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 - assert module.test_pypy_flags(MyFloat, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 \ No newline at end of file + assert module.test_pypy_flags(MyFloat, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -19,7 +19,7 @@ Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS, Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS, Py_TPFLAGS_TYPE_SUBCLASS, - Py_TPFLAGS_BYTES_SUBCLASS + Py_TPFLAGS_BYTES_SUBCLASS, Py_TPPYPYFLAGS_FLOAT_SUBCLASS, ) From pypy.commits at gmail.com Mon Mar 26 18:13:50 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Mar 2018 15:13:50 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: fix test, implement userslot for __call__ needed for python-defined class Message-ID: <5ab9709e.deafdf0a.1414.c970@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94142:c9ce25047019 Date: 2018-03-27 01:10 +0300 http://bitbucket.org/pypy/pypy/changeset/c9ce25047019/ Log: fix test, implement userslot for __call__ needed for python-defined class diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -364,7 +364,7 @@ self.unimport_module(name) self.cleanup() state = self.space.fromcache(State) - assert not state.operror + assert 'operror' not in dir(state) class AppTestCpythonExtension(AppTestCpythonExtensionBase): diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py --- a/pypy/module/cpyext/userslot.py +++ b/pypy/module/cpyext/userslot.py @@ -49,6 +49,11 @@ w_stararg=w_args, w_starstararg=w_kwds) return space.call_args(w_impl, args) + at slot_function([PyObject, PyObject, PyObject], PyObject) +def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(w_self, args) + # unary functions @slot_function([PyObject], PyObject) From pypy.commits at gmail.com Mon Mar 26 18:13:46 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Mar 2018 15:13:46 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: merge default into py3.5 Message-ID: <5ab9709a.11e3df0a.17297.b424@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94140:96851b8e1c6a Date: 2018-03-26 14:29 +0300 http://bitbucket.org/pypy/pypy/changeset/96851b8e1c6a/ Log: merge default into py3.5 diff too long, truncating to 2000 out of 2028 lines diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,42 +4,40 @@ Welcome to PyPy! -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. +PyPy is an interperter that implements the Python programming language, based +on the RPython compiler framework for dynamic language implementations. -The home page is: +The home page for the interpreter is: http://pypy.org/ -If you want to help developing PyPy, this document might help you: +If you want to help developing PyPy, this documentation might help you: http://doc.pypy.org/ -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! +More documentation about the RPython framework can be found here - the pypy-dev team + http://rpython.readthedocs.io +The source for the documentation is in the pypy/doc directory + +Using PyPy instead of CPython +============================= + +Please read the information at http://pypy.org to find the correct way to +download and use PyPy as an alternative to CPython. Building ======== -First switch to or download the correct branch. The basic choices are -``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X -branch (e.g. ``py3.5``). +Building PyPy is not the recommended way to obtain the PyPy alternative python +interpreter. It is time-consuming and requires significant computing resources. +More information can be found here -Build with: + http://doc.pypy.org/en/latest/build.html -.. code-block:: console +Enjoy and send us feedback! - $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + the pypy-dev team -This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy -directory. We suggest to use virtualenv with the resulting -pypy-c/pypy3-c as the interpreter; you can find more details about -various installation schemes here: - http://doc.pypy.org/en/latest/install.html diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -17,13 +17,18 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~ The quickest way to start using PyPy is to download a prebuilt binary for your -OS and architecture. You can either use the `most recent release`_ or one of -our `development nightly build`_. Please note that the nightly builds are not +OS and architecture. You may be able to use either use the +`most recent release`_ or one of our `development nightly build`_. These +builds depend on dynamically linked libraries that may not be available on your +OS. See the section about `Linux binaries` for more info and alternatives that +may work on your system. + +Please note that the nightly builds are not guaranteed to be as stable as official releases, use them at your own risk. .. _most recent release: http://pypy.org/download.html .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/ - +.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions Installing PyPy ~~~~~~~~~~~~~~~ @@ -69,9 +74,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is often convenient to run pypy inside a virtualenv. To do this -you need a recent version of virtualenv -- 1.6.1 or greater. You can +you need a version of virtualenv -- 1.6.1 or greater. You can then install PyPy both from a precompiled tarball or from a mercurial -checkout:: +checkout after translation:: # from a tarball $ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -357,12 +357,13 @@ sublist_w = [] n = last_pos = 0 while not count or n < count: + pattern = ctx.pattern sub_jitdriver.jit_merge_point( self=self, use_builder=use_builder, filter_is_callable=filter_is_callable, filter_type=type(w_filter), - ctx=ctx, + ctx=ctx, pattern=pattern, w_filter=w_filter, strbuilder=strbuilder, unicodebuilder=unicodebuilder, @@ -437,7 +438,7 @@ filter_as_unicode w_string sublist_w self""".split(), - greens=["filter_is_callable", "use_builder", "filter_type", "ctx.pattern"]) + greens=["filter_is_callable", "use_builder", "filter_type", "pattern"]) def _sub_append_slice(ctx, space, use_builder, sublist_w, diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -135,6 +135,11 @@ 'TYPE', 'BYTES'): constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name) +# PyPy-specific flags +for name in ('FLOAT',): + constant_names.append('Py_TPPYPYFLAGS_%s_SUBCLASS' % name) + + for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) globals().update(rffi_platform.configure(CConfig_constants)) diff --git a/pypy/module/cpyext/boolobject.py b/pypy/module/cpyext/boolobject.py --- a/pypy/module/cpyext/boolobject.py +++ b/pypy/module/cpyext/boolobject.py @@ -1,9 +1,5 @@ -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, - build_type_checkers) - -# Inheriting from bool isn't actually possible. -PyBool_Check = build_type_checkers("Bool")[1] +from rpython.rtyper.lltypesystem import rffi +from pypy.module.cpyext.api import cpython_api, PyObject @cpython_api([rffi.LONG], PyObject) def PyBool_FromLong(space, value): diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py --- a/pypy/module/cpyext/floatobject.py +++ b/pypy/module/cpyext/floatobject.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (PyObjectFields, bootstrap_function, cpython_struct, - CANNOT_FAIL, cpython_api, PyObject, build_type_checkers, CONST_STRING) + CANNOT_FAIL, cpython_api, PyObject, CONST_STRING) from pypy.module.cpyext.pyobject import ( make_typedescr, track_reference, from_ref) from pypy.interpreter.error import OperationError @@ -38,8 +38,6 @@ track_reference(space, obj, w_obj) return w_obj -PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") - @cpython_api([lltype.Float], PyObject) def PyFloat_FromDouble(space, value): return space.newfloat(value) diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -82,10 +82,10 @@ def PyTraceBack_Here(space, w_frame): from pypy.interpreter.pytraceback import record_application_traceback state = space.fromcache(State) - if state.operror is None: + if state.get_exception() is None: return -1 frame = space.interp_w(PyFrame, w_frame) - record_application_traceback(space, state.operror, frame, 0) + record_application_traceback(space, state.get_exception(), frame, 0) return 0 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -14,6 +14,8 @@ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h --- a/pypy/module/cpyext/include/floatobject.h +++ b/pypy/module/cpyext/include/floatobject.h @@ -32,6 +32,11 @@ return PyFloat_FromDouble(-Py_HUGE_VAL); \ } while(0) +#define PyFloat_Check(op) \ + _PyPy_Type_FastSubclass((op)->ob_type, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) +#define PyFloat_CheckExact(op) ((op)->ob_type == &PyFloat_Type) + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -218,6 +218,12 @@ #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30) #define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31) + +/* These are conceptually the same as the flags above, but they are + PyPy-specific and are stored inside tp_pypy_flags */ +#define Py_TPPYPYFLAGS_FLOAT_SUBCLASS (1L<<0) + + #define Py_TPFLAGS_DEFAULT ( \ Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \ Py_TPFLAGS_HAVE_VERSION_TAG | \ @@ -236,6 +242,8 @@ #endif #define PyType_FastSubclass(t,f) PyType_HasFeature(t,f) +#define _PyPy_Type_FastSubclass(t,f) (((t)->tp_pypy_flags & (f)) != 0) + #define PyType_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) #define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type) diff --git a/pypy/module/cpyext/include/sliceobject.h b/pypy/module/cpyext/include/sliceobject.h --- a/pypy/module/cpyext/include/sliceobject.h +++ b/pypy/module/cpyext/include/sliceobject.h @@ -17,6 +17,8 @@ PyObject *step; } PySliceObject; +#define PySlice_Check(op) ((op)->ob_type == &PySlice_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -45,6 +45,18 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) +def w_kwargs_from_args(space, __args__): + w_kwargs = None + if __args__.keywords: + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) + return w_kwargs + def undotted_name(name): """Return the last component of a dotted name""" dotpos = name.rfind('.') @@ -136,15 +148,7 @@ def call_keywords(self, space, w_self, __args__): func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) py_args = tuple_from_args_w(space, __args__.arguments_w) - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + w_kwargs = w_kwargs_from_args(space, __args__) try: return generic_cpy_call(space, func, w_self, py_args, w_kwargs) finally: @@ -243,14 +247,15 @@ (self.name.decode('utf-8'), self.w_objclass.getname(self.space))) +class W_PyCWrapperObject(W_Root): + """ + Abstract class; for concrete subclasses, see slotdefs.py + """ + _immutable_fields_ = ['offset[*]'] -class W_PyCWrapperObject(W_Root): - def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func, offset=None): + def __init__(self, space, pto, method_name, doc, func, offset): self.space = space self.method_name = method_name - self.wrapper_func = wrapper_func - self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func self.offset = offset @@ -259,10 +264,17 @@ assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type - def call(self, space, w_self, w_args, w_kw): + def descr_call(self, space, w_self, __args__): + return self.call(space, w_self, __args__) + + def call(self, space, w_self, __args__): + raise NotImplementedError + + @jit.unroll_safe + def get_func_to_call(self): func_to_call = self.func if self.offset: - pto = as_pyobj(space, self.w_objclass) + pto = as_pyobj(self.space, self.w_objclass) # make ptr the equivalent of this, using the offsets #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) if pto: @@ -276,31 +288,33 @@ assert False, "failed to convert w_type %s to PyObject" % str( self.w_objclass) assert func_to_call - if self.wrapper_func is None: - assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, - w_kw) - if space.is_true(w_kw): - raise oefmt(space.w_TypeError, + return func_to_call + + def check_args(self, __args__, arity): + length = len(__args__.arguments_w) + if length != arity: + raise oefmt(self.space.w_TypeError, "expected %d arguments, got %d", + arity, length) + if __args__.keywords: + raise oefmt(self.space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - return self.wrapper_func(space, w_self, w_args, func_to_call) + + def check_argsv(self, __args__, min, max): + length = len(__args__.arguments_w) + if not min <= length <= max: + raise oefmt(self.space.w_TypeError, "expected %d-%d arguments, got %d", + min, max, length) + if __args__.keywords: + raise oefmt(self.space.w_TypeError, + "wrapper %s doesn't take any keyword arguments", + self.method_name) def descr_method_repr(self): return self.space.newtext("" % (self.method_name, self.w_objclass.name)) - at jit.dont_look_inside -def cwrapper_descr_call(space, w_self, __args__): - self = space.interp_w(W_PyCWrapperObject, w_self) - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w[1:]) - w_self = args_w[0] - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - return self.call(space, w_self, w_args, w_kw) def cmethod_descr_get(space, w_function, w_obj, w_cls=None): if w_obj is None or space.is_w(w_obj, space.w_None): @@ -351,7 +365,7 @@ W_PyCWrapperObject.typedef = TypeDef( 'wrapper_descriptor', - __call__ = interp2app(cwrapper_descr_call), + __call__ = interp2app(W_PyCWrapperObject.descr_call), __get__ = interp2app(cmethod_descr_get), __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject, wrapfn="newtext_or_none"), diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -288,6 +288,11 @@ unsigned int tp_version_tag; destructor tp_finalize; + + /* PyPy specific extra fields: make sure that they are ALWAYS at the end, + for compatibility with CPython */ + long tp_pypy_flags; + } PyTypeObject; typedef struct{ diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -67,9 +67,10 @@ @cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) - if state.operror is None: + operror = state.get_exception() + if operror is None: return None - return state.operror.w_type # borrowed ref + return operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -47,7 +47,6 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) -PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @cpython_api([PyObject, PyObject, PyObject], PyObject) def PySlice_New(space, w_start, w_stop, w_step): @@ -75,9 +74,8 @@ normal slices. Returns 0 on success and -1 on error with exception set.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \ w_slice.indices4(space, length) return 0 @@ -97,9 +95,8 @@ objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, in the source of your extension.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0] = \ w_slice.indices3(space, length) return 0 diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -19,6 +19,8 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext import userslot from pypy.module.cpyext.buffer import CBuffer, CPyBuffer, fq +from pypy.module.cpyext.methodobject import (W_PyCWrapperObject, tuple_from_args_w, + w_kwargs_from_args) from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable @@ -38,29 +40,6 @@ Py_GT = 4 Py_GE = 5 - -def check_num_args(space, w_ob, n): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if n == space.len_w(w_ob): - return - raise oefmt(space.w_TypeError, - "expected %d arguments, got %d", - n, space.len_w(w_ob)) - -def check_num_argsv(space, w_ob, low, high): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if low <=space.len_w(w_ob) <= high: - return - raise oefmt(space.w_TypeError, - "expected %d-%d arguments, got %d", - low, high, space.len_w(w_ob)) - @not_rpython def llslot(space, func): return func.api_func.get_llhelper(space) @@ -71,325 +50,401 @@ get_llhelper = v_func.value.api_func.get_llhelper return ctx.appcall(get_llhelper, v_space) +# NOTE: the following wrap_* are subclasses of W_PyCWrapperObject, even if +# they don't follow the usual W_* naming convention for subclasses of W_Root: +# we do this because we automatically generate most of the slots from the +# CPython code copy&pasted inside slotdefs_str, and thus we need to keep the +# same names as they are used in C. -def wrap_init(space, w_self, w_args, func, w_kwargs): - func_init = rffi.cast(initproc, func) - res = generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return None +class wrap_init(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_init = rffi.cast(initproc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return None -def wrap_unaryfunc(space, w_self, w_args, func): - func_unary = rffi.cast(unaryfunc, func) - check_num_args(space, w_args, 0) - return generic_cpy_call(space, func_unary, w_self) +class wrap_unaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_unary = rffi.cast(unaryfunc, func) + return generic_cpy_call(space, func_unary, w_self) -def wrap_binaryfunc(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_binary, w_self, args_w[0]) +class wrap_binaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_x = __args__.arguments_w[0] + return generic_cpy_call(space, func_binary, w_self, w_x) def _get_ob_type(space, w_obj): # please ensure that w_obj stays alive ob_type = as_pyobj(space, space.type(w_obj)) return rffi.cast(PyTypeObjectPtr, ob_type) -def wrap_binaryfunc_l(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_binary, w_self, args_w[0]) +class wrap_binaryfunc_l(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_value = __args__.arguments_w[0] + return generic_cpy_call(space, func_binary, w_self, w_value) -def wrap_binaryfunc_r(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_binary, args_w[0], w_self) +class wrap_binaryfunc_r(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_value = __args__.arguments_w[0] + return generic_cpy_call(space, func_binary, w_value, w_self) -def wrap_ternaryfunc(space, w_self, w_args, func): - # The third argument is optional - func_ternary = rffi.cast(ternaryfunc, func) - check_num_argsv(space, w_args, 1, 2) - args_w = space.fixedview(w_args) - arg3 = space.w_None - if len(args_w) > 1: - arg3 = args_w[1] - return generic_cpy_call(space, func_ternary, w_self, args_w[0], arg3) +class wrap_ternaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + # The third argument is optional + self.check_argsv(__args__, 1, 2) + func = self.get_func_to_call() + func_ternary = rffi.cast(ternaryfunc, func) + w_arg0 = __args__.arguments_w[0] + if len(__args__.arguments_w) == 2: + w_arg1 = __args__.arguments_w[1] + else: + w_arg1 = space.w_None + return generic_cpy_call(space, func_ternary, w_self, w_arg0, w_arg1) -def wrap_ternaryfunc_r(space, w_self, w_args, func): - # The third argument is optional - func_ternary = rffi.cast(ternaryfunc, func) - check_num_argsv(space, w_args, 1, 2) - args_w = space.fixedview(w_args) - arg3 = space.w_None - if len(args_w) > 1: - arg3 = args_w[1] - return generic_cpy_call(space, func_ternary, args_w[0], w_self, arg3) +class wrap_ternaryfunc_r(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + # The third argument is optional + self.check_argsv(__args__, 1, 2) + func = self.get_func_to_call() + func_ternary = rffi.cast(ternaryfunc, func) + w_arg0 = __args__.arguments_w[0] + if len(__args__.arguments_w) == 2: + w_arg1 = __args__.arguments_w[1] + else: + w_arg1 = space.w_None + return generic_cpy_call(space, func_ternary, w_arg0, w_self, w_arg1) +class wrap_inquirypred(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_inquiry = rffi.cast(inquiry, func) + res = generic_cpy_call(space, func_inquiry, w_self) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) -def wrap_inquirypred(space, w_self, w_args, func): - func_inquiry = rffi.cast(inquiry, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_inquiry, w_self) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class wrap_getattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrfunc, func) + w_name = __args__.arguments_w[0] + name_ptr = rffi.str2charp(space.text_w(w_name)) + try: + return generic_cpy_call(space, func_target, w_self, name_ptr) + finally: + rffi.free_charp(name_ptr) -def wrap_getattr(space, w_self, w_args, func): - func_target = rffi.cast(getattrfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - name_ptr = rffi.str2charp(space.text_w(args_w[0])) - try: - return generic_cpy_call(space, func_target, w_self, name_ptr) - finally: - rffi.free_charp(name_ptr) +class wrap_getattro(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrofunc, func) + w_name = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, w_self, w_name) -def wrap_getattro(space, w_self, w_args, func): - func_target = rffi.cast(getattrofunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_target, w_self, args_w[0]) +class wrap_setattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_setattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 2) - w_name, w_value = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_delattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_delattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 1) - w_name, = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_descr_get(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(descrgetfunc, func) + length = len(__args__.arguments_w) + if length == 1: + w_obj = __args__.arguments_w[0] + w_type = None + elif length == 2: + w_obj = __args__.arguments_w[0] + w_type = __args__.arguments_w[1] + else: + raise oefmt(space.w_TypeError, + "expected 1 or 2 arguments, got %d", len(__args__.arguments_w)) + if w_obj is space.w_None: + w_obj = None + if w_type is space.w_None: + w_type = None + if w_obj is None and w_type is None: + raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") + return generic_cpy_call(space, func_target, w_self, w_obj, w_type) -def wrap_descr_get(space, w_self, w_args, func): - func_target = rffi.cast(descrgetfunc, func) - args_w = space.fixedview(w_args) - if len(args_w) == 1: - w_obj, = args_w - w_type = None - elif len(args_w) == 2: - w_obj, w_type = args_w - else: - raise oefmt(space.w_TypeError, - "expected 1 or 2 arguments, got %d", len(args_w)) - if w_obj is space.w_None: - w_obj = None - if w_type is space.w_None: - w_type = None - if w_obj is None and w_type is None: - raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") - return generic_cpy_call(space, func_target, w_self, w_obj, w_type) +class wrap_descr_set(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(descrsetfunc, func) + w_obj = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + res = generic_cpy_call(space, func_target, w_self, w_obj, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_descr_set(space, w_self, w_args, func): - func_target = rffi.cast(descrsetfunc, func) - check_num_args(space, w_args, 2) - w_obj, w_value = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_obj, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_descr_delete(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(descrsetfunc, func) + w_obj = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_obj, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_descr_delete(space, w_self, w_args, func): - func_target = rffi.cast(descrsetfunc, func) - check_num_args(space, w_args, 1) - w_obj, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_obj, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_call(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(ternaryfunc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) -def wrap_call(space, w_self, w_args, func, w_kwds): - func_target = rffi.cast(ternaryfunc, func) - return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) +class wrap_ssizessizeobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 3) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeobjargproc, func) + i = space.int_w(space.index(__args__.arguments_w[0])) + j = space.int_w(space.index(__args__.arguments_w[1])) + w_y = __args__.arguments_w[2] + res = generic_cpy_call(space, func_target, w_self, i, j, w_y) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_ssizessizeobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeobjargproc, func) - check_num_args(space, w_args, 3) - args_w = space.fixedview(w_args) - i = space.int_w(space.index(args_w[0])) - j = space.int_w(space.index(args_w[1])) - w_y = args_w[2] - res = generic_cpy_call(space, func_target, w_self, i, j, w_y) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_lenfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_len = rffi.cast(lenfunc, func) + res = generic_cpy_call(space, func_len, w_self) + if widen(res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) -def wrap_lenfunc(space, w_self, w_args, func): - func_len = rffi.cast(lenfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_len, w_self) - if widen(res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_sq_item(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeargfunc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + return generic_cpy_call(space, func_target, w_self, index) -def wrap_sq_item(space, w_self, w_args, func): - func_target = rffi.cast(ssizeargfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - return generic_cpy_call(space, func_target, w_self, index) +class wrap_sq_setitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + index = space.int_w(space.index(w_index)) + res = generic_cpy_call(space, func_target, w_self, index, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_sq_setitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - -def wrap_sq_delitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, index, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_sq_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, index, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) # Warning, confusing function name (like CPython). Used only for sq_contains. -def wrap_objobjproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjproc, func) - check_num_args(space, w_args, 1) - w_value, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_value) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class wrap_objobjproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjproc, func) + w_value = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_value) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) -def wrap_objobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 2) - w_key, w_value = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_key, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_objobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + res = generic_cpy_call(space, func_target, w_self, w_key, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None -def wrap_delitem(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 1) - w_key, = space.fixedview(w_args) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, w_key, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, w_key, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None -def wrap_ssizessizeargfunc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeargfunc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - start = space.int_w(args_w[0]) - end = space.int_w(args_w[1]) - return generic_cpy_call(space, func_target, w_self, start, end) +class wrap_ssizessizeargfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeargfunc, func) + start = space.int_w(__args__.arguments_w[0]) + end = space.int_w(__args__.arguments_w[1]) + return generic_cpy_call(space, func_target, w_self, start, end) -def wrap_next(space, w_self, w_args, func): - from pypy.module.cpyext.api import generic_cpy_call_expect_null - func_target = rffi.cast(iternextfunc, func) - check_num_args(space, w_args, 0) - w_res = generic_cpy_call_expect_null(space, func_target, w_self) - if not w_res and not PyErr_Occurred(space): - raise OperationError(space.w_StopIteration, space.w_None) - return w_res +class wrap_next(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + from pypy.module.cpyext.api import generic_cpy_call_expect_null + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(iternextfunc, func) + w_res = generic_cpy_call_expect_null(space, func_target, w_self) + if not w_res and not PyErr_Occurred(space): + raise OperationError(space.w_StopIteration, space.w_None) + return w_res -def wrap_hashfunc(space, w_self, w_args, func): - func_target = rffi.cast(hashfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_target, w_self) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_hashfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(hashfunc, func) + res = generic_cpy_call(space, func_target, w_self) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) -def wrap_getreadbuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getreadbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) -def wrap_getwritebuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getwritebuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) -def wrap_getbuffer(space, w_self, w_args, func): - func_target = rffi.cast(getbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(Py_buffer) as pybuf: - _flags = 0 - if space.len_w(w_args) > 0: - _flags = space.int_w(space.listview(w_args)[0]) - flags = rffi.cast(rffi.INT_real,_flags) - size = generic_cpy_call(space, func_target, w_self, pybuf, flags) - if widen(size) < 0: - space.fromcache(State).check_and_raise_exception(always=True) - ptr = pybuf.c_buf - size = pybuf.c_len - ndim = widen(pybuf.c_ndim) - shape = None - if pybuf.c_shape: - shape = [pybuf.c_shape[i] for i in range(ndim)] - strides = None - if pybuf.c_strides: - strides = [pybuf.c_strides[i] for i in range(ndim)] - if pybuf.c_format: - format = rffi.charp2str(pybuf.c_format) - else: - format = 'B' - # the CPython docs mandates that you do an incref whenever you call - # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't - # leak we release the buffer: - # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer - buf = CPyBuffer(space, ptr, size, w_self, format=format, - ndim=ndim, shape=shape, strides=strides, - itemsize=pybuf.c_itemsize, - readonly=widen(pybuf.c_readonly), - needs_decref=True, - releasebufferproc = rbp) - fq.register_finalizer(buf) - return buf.wrap(space) + +class wrap_getbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(getbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(Py_buffer) as pybuf: + _flags = 0 + if len(__args__.arguments_w) > 0: + _flags = space.int_w(__args__.arguments_w[0]) + flags = rffi.cast(rffi.INT_real,_flags) + size = generic_cpy_call(space, func_target, w_self, pybuf, flags) + if widen(size) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + ptr = pybuf.c_buf + size = pybuf.c_len + ndim = widen(pybuf.c_ndim) + shape = None + if pybuf.c_shape: + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None + if pybuf.c_strides: + strides = [pybuf.c_strides[i] for i in range(ndim)] + if pybuf.c_format: + format = rffi.charp2str(pybuf.c_format) + else: + format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer + buf = CPyBuffer(space, ptr, size, w_self, format=format, + ndim=ndim, shape=shape, strides=strides, + itemsize=pybuf.c_itemsize, + readonly=widen(pybuf.c_readonly), + needs_decref=True, + releasebufferproc = rbp) + fq.register_finalizer(buf) + return buf.wrap(space) def get_richcmp_func(OP_CONST): - def inner(space, w_self, w_args, func): - func_target = rffi.cast(richcmpfunc, func) - check_num_args(space, w_args, 1) - w_other, = space.fixedview(w_args) - return generic_cpy_call(space, func_target, - w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST)) - return inner + class wrap_richcmp(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(richcmpfunc, func) + w_other = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, + w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST)) + return wrap_richcmp richcmp_eq = get_richcmp_func(Py_EQ) richcmp_ne = get_richcmp_func(Py_NE) @@ -398,17 +453,19 @@ richcmp_gt = get_richcmp_func(Py_GT) richcmp_ge = get_richcmp_func(Py_GE) -def wrap_cmpfunc(space, w_self, w_args, func): - func_target = rffi.cast(cmpfunc, func) - check_num_args(space, w_args, 1) - w_other, = space.fixedview(w_args) +class wrap_cmpfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(cmpfunc, func) + w_other = __args__.arguments_w[0] - if not space.issubtype_w(space.type(w_self), space.type(w_other)): - raise oefmt(space.w_TypeError, - "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'", - w_self, w_self, w_other) + if not space.issubtype_w(space.type(w_self), space.type(w_other)): + raise oefmt(space.w_TypeError, + "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'", + w_self, w_self, w_other) - return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) + return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) SLOT_FACTORIES = {} def slot_factory(tp_name): @@ -776,9 +833,10 @@ missing_wrappers = ['wrap_indexargfunc', 'wrap_del'] for name in missing_wrappers: assert name not in globals() - def missing_wrapper(space, w_self, w_args, func): - print "cpyext: missing slot wrapper " + name - raise NotImplementedError("Slot wrapper " + name) + class missing_wrapper(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + print "cpyext: missing slot wrapper " + name + raise NotImplementedError("Slot wrapper " + name) missing_wrapper.__name__ = name globals()[name] = missing_wrapper @@ -811,13 +869,12 @@ PyWrapperFlag_KEYWORDS = 1 class TypeSlot: - def __init__(self, method_name, slot_name, function, wrapper1, wrapper2, doc): + def __init__(self, method_name, slot_name, function, wrapper, doc): self.method_name = method_name self.slot_name = slot_name self.slot_names = tuple(("c_" + slot_name).split(".")) self.slot_func = function - self.wrapper_func = wrapper1 - self.wrapper_func_kwds = wrapper2 + self.wrapper_class = wrapper self.doc = doc # adapted from typeobject.c @@ -838,13 +895,7 @@ function = getattr(userslot, FUNCTION or '!missing', None) assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS - if FLAGS: - wrapper1 = None - wrapper2 = wrapper - else: - wrapper1 = wrapper - wrapper2 = None - return TypeSlot(NAME, SLOT, function, wrapper1, wrapper2, DOC) + return TypeSlot(NAME, SLOT, function, wrapper, DOC) def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC): return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0) @@ -1127,7 +1178,7 @@ x.slot_func.api_func if x.slot_func else None) for x in slotdefs]) slotdefs_for_wrappers = unrolling_iterable( - [(x.method_name, x.slot_names, x.wrapper_func, x.wrapper_func_kwds, x.doc) + [(x.method_name, x.slot_names, x.wrapper_class, x.doc) for x in slotdefs]) if __name__ == "__main__": diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -2,11 +2,18 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter import executioncontext +from pypy.interpreter.executioncontext import ExecutionContext from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rdynload import DLLHANDLE from rpython.rlib import rawrefcount import sys + +# Keep track of exceptions raised in cpyext for a particular execution +# context. +ExecutionContext.cpyext_operror = None + + class State: def __init__(self, space): self.space = space @@ -18,7 +25,8 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.operror = None + ec = self.space.getexecutioncontext() + ec.cpyext_operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track @@ -37,17 +45,24 @@ def set_exception(self, operror): self.clear_exception() - self.operror = operror + ec = self.space.getexecutioncontext() + ec.cpyext_operror = operror def clear_exception(self): """Clear the current exception state, and return the operror.""" - operror = self.operror - self.operror = None + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror + ec.cpyext_operror = None return operror + def get_exception(self): + ec = self.space.getexecutioncontext() + return ec.cpyext_operror + @specialize.arg(1) def check_and_raise_exception(self, always=False): - operror = self.operror + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror if operror: self.clear_exception() raise operror diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -39,7 +39,7 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - operror = state.operror + operror = state.get_exception() if not operror: raise Exception("DID NOT RAISE") if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py --- a/pypy/module/cpyext/test/test_boolobject.py +++ b/pypy/module/cpyext/test/test_boolobject.py @@ -1,7 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong -from pypy.module.cpyext.floatobject import PyFloat_FromDouble +from pypy.module.cpyext.boolobject import PyBool_FromLong class TestBoolObject(BaseApiTest): def test_fromlong(self, space): @@ -12,12 +11,6 @@ else: assert obj is space.w_False - def test_check(self, space): - assert PyBool_Check(space, space.w_True) - assert PyBool_Check(space, space.w_False) - assert not PyBool_Check(space, space.w_None) - assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0)) - class AppTestBoolMacros(AppTestCpythonExtensionBase): def test_macros(self): module = self.import_extension('foo', [ @@ -42,4 +35,14 @@ assert module.to_int(False) == 0 assert module.to_int(True) == 1 - + def test_check(self): + module = self.import_extension('foo', [ + ("type_check", "METH_O", + ''' + return PyLong_FromLong(PyBool_Check(args)); + ''')]) + assert module.type_check(True) + assert module.type_check(False) + assert not module.type_check(None) + assert not module.type_check(1.0) + diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -613,7 +613,8 @@ Py_ssize_t refcnt_after; Py_INCREF(true_obj); Py_INCREF(true_obj); - PyBool_Check(true_obj); + if (!PyBool_Check(true_obj)) + Py_RETURN_NONE; refcnt_after = true_obj->ob_refcnt; Py_DECREF(true_obj); Py_DECREF(true_obj); diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -102,9 +102,11 @@ """ PyObject* pyobj = PyFloat_FromDouble(1.0); PyFloatObject* pfo = (PyFloatObject*)pyobj; - int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) && - PyFloat_Check(pfo) && PyFloat_CheckExact(pfo); + int res = (PyFloat_Check(pyobj) + + PyFloat_CheckExact(pyobj) * 10 + + PyFloat_Check(pfo) * 100 + + PyFloat_CheckExact(pfo) * 1000); Py_DecRef(pyobj); return PyLong_FromLong(res);"""), ]) - assert module.test() == 1 + assert module.test() == 1111 diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py --- a/pypy/module/cpyext/test/test_number.py +++ b/pypy/module/cpyext/test/test_number.py @@ -8,7 +8,6 @@ PyNumber_Index, PyNumber_Add, PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute, PyNumber_Power, PyNumber_InPlacePower) -from pypy.module.cpyext.floatobject import PyFloat_Check from pypy.module.cpyext.longobject import PyLong_CheckExact from pypy.module.cpyext.object import PyObject_Size diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py --- a/pypy/module/cpyext/test/test_pyerrors.py +++ b/pypy/module/cpyext/test/test_pyerrors.py @@ -43,7 +43,8 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.operror.get_w_value(space), + operror = state.get_exception() + assert space.eq_w(operror.get_w_value(space), space.wrap("a value")) api.PyErr_Clear() @@ -51,12 +52,14 @@ def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.operror.w_type, space.w_KeyError) - assert space.eq_w(state.operror.get_w_value(space), space.w_None) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_KeyError) + assert space.eq_w(operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.operror.w_type, space.w_MemoryError) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_Warning(self, space, api, capfd): @@ -475,3 +478,59 @@ '''), ]) raises(SystemError, module.oops) + + def test_error_thread_race(self): + # Check race condition: thread 0 returns from cpyext with error set, + # after thread 1 has set an error but before it returns. + module = self.import_extension('foo', [ + ("emit_error", "METH_VARARGS", + ''' + PyThreadState *save = NULL; + PyGILState_STATE gilsave; + + /* NB. synchronization due to GIL */ + static volatile int flag = 0; + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + return NULL; + + /* Proceed in thread 1 first */ + save = PyEval_SaveThread(); + while (id == 0 && flag == 0); + gilsave = PyGILState_Ensure(); + + PyErr_Format(PyExc_ValueError, "%d", id); + + /* Proceed in thread 0 first */ + if (id == 1) flag = 1; + PyGILState_Release(gilsave); + while (id == 1 && flag == 1); + PyEval_RestoreThread(save); + + if (id == 0) flag = 0; + return NULL; + ''' + ), + ]) + + import threading + + failures = [] + + def worker(arg): + try: + module.emit_error(arg) + failures.append(True) + except Exception as exc: + if str(exc) != str(arg): + failures.append(exc) + + threads = [threading.Thread(target=worker, args=(j,)) + for j in (0, 1)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not failures diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py --- a/pypy/module/cpyext/test/test_sliceobject.py +++ b/pypy/module/cpyext/test/test_sliceobject.py @@ -2,14 +2,8 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP -from pypy.module.cpyext.sliceobject import PySlice_Check class TestSliceObject(BaseApiTest): - def test_slice(self, space): - w_i = space.wrap(10) - w_slice = space.newslice(w_i, w_i, w_i) - assert PySlice_Check(space, w_slice) - assert not PySlice_Check(space, w_i) def test_GetIndicesEx(self, space, api): w = space.wrap @@ -79,3 +73,14 @@ """), ]) assert module.get_ellipsis() is Ellipsis + + def test_typecheck(self): + module = self.import_extension('foo', [ + ("check", "METH_O", + """ + PySliceObject *slice = (PySliceObject *)args; + return PyLong_FromLong(PySlice_Check(slice)); + """), + ]) + s = slice(10, 20, 30) + assert module.check(s) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -783,6 +783,36 @@ assert module.tp_init(list, x, ("hi",)) is None assert x == ["h", "i"] + def test_mp_subscript(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static PyObject* + mp_subscript(PyObject *self, PyObject *key) + { + return Py_BuildValue("i", 42); + } + PyMappingMethods tp_as_mapping; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init = ''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_mapping = &tp_as_mapping; + tp_as_mapping.mp_subscript = (binaryfunc)mp_subscript; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + assert obj[100] == 42 + raises(TypeError, "obj.__getitem__(100, 101)") + raises(TypeError, "obj.__getitem__(100, a=42)") + def test_mp_ass_subscript(self): module = self.import_extension('foo', [ ("new_obj", "METH_NOARGS", @@ -846,6 +876,84 @@ res = "foo" in obj assert res is True + def test_sq_ass_slice(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) + { + int expected = (a == 10 && b == 20 && + PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_slice = sq_ass_slice; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10:20] = 42 + raises(ValueError, "obj[10:20] = 43") + raises(ValueError, "obj[11:20] = 42") + raises(ValueError, "obj[10:21] = 42") + + def test_sq_ass_item(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) + { + int expected; + if (o == NULL) // delitem + expected = (i == 12); + else // setitem + expected = (i == 10 && PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_item = sq_ass_item; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10] = 42 + raises(ValueError, "obj[10] = 43") + raises(ValueError, "obj[11] = 42") + del obj[12] + raises(ValueError, "del obj[13]") + def test_tp_iter(self): module = self.import_extension('foo', [ ("tp_iter", "METH_VARARGS", @@ -1498,3 +1606,29 @@ class MyList(list): pass assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0 + + def test_has_pypy_subclass_flag(self): + module = self.import_extension('foo', [ + ("test_pypy_flags", "METH_VARARGS", + ''' + long long in_flag, my_flag; + PyObject * obj; + if (!PyArg_ParseTuple(args, "OL", &obj, &in_flag)) + return NULL; + if (!PyType_Check(obj)) + { + PyErr_SetString(PyExc_ValueError, "input must be type"); + return NULL; + } + my_flag = ((PyTypeObject*)obj)->tp_pypy_flags; + if ((my_flag & in_flag) != in_flag) + return PyLong_FromLong(-1); + return PyLong_FromLong(0); + '''),]) + # copied from object.h + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + + class MyFloat(float): + pass + assert module.test_pypy_flags(float, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + assert module.test_pypy_flags(MyFloat, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 \ No newline at end of file diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -19,11 +19,15 @@ Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS, Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS, Py_TPFLAGS_TYPE_SUBCLASS, - Py_TPFLAGS_BYTES_SUBCLASS) + Py_TPFLAGS_BYTES_SUBCLASS + Py_TPPYPYFLAGS_FLOAT_SUBCLASS, + ) + from pypy.module.cpyext.cparser import CTypeSpace from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, - W_PyCMethodObject, W_PyCFunctionObject, extract_doc, extract_txtsig) + PyCFunction_NewEx, PyCFunction, PyMethodDef, + W_PyCMethodObject, W_PyCFunctionObject, extract_doc, extract_txtsig, + W_PyCWrapperObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, get_typedescr, make_typedescr, @@ -311,7 +315,7 @@ def add_operators(space, dict_w, pto, name): from pypy.module.cpyext.object import PyObject_HashNotImplemented hash_not_impl = llslot(space, PyObject_HashNotImplemented) - for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: + for method_name, slot_names, wrapper_class, doc in slotdefs_for_wrappers: if method_name in dict_w: continue offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] @@ -336,10 +340,11 @@ func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue - if wrapper_func is None and wrapper_func_kwds is None: + if wrapper_class is None: continue - w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func_voidp, offset=offset) + + assert issubclass(wrapper_class, W_PyCWrapperObject) + w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset[:]) dict_w[method_name] = w_obj if pto.c_tp_doc: raw_doc = rffi.charp2str(cts.cast('char*', pto.c_tp_doc)) @@ -424,6 +429,9 @@ pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS elif space.issubtype_w(w_obj, space.w_dict): pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS + # the following types are a pypy-specific extensions, using tp_pypy_flags + elif space.issubtype_w(w_obj, space.w_float): + pto.c_tp_pypy_flags |= Py_TPPYPYFLAGS_FLOAT_SUBCLASS def check_descr(space, w_self, w_type): if not space.isinstance_w(w_self, w_type): diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -103,7 +103,7 @@ elif bookkeeper and not hasattr(t, '_freeze_'): return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: - raise AssertionError("annotationoftype(%r)" % (t,)) + raise TypeError("Annotation of type %r not supported" % (t,)) class Sig(object): diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -259,6 +259,26 @@ intmask(). +Type Enforcing and Checking +--------------------------- + +RPython provides a helper decorator to force RPython-level types on function +arguments. The decorator, called ``enforceargs()``, accepts as parameters the +types expected to match the arguments of the function. + +Functions decorated with ``enforceargs()`` have their function signature +analyzed and their RPython-level type inferred at import time (for further +details about the flavor of translation performed in RPython, see the +`Annotation pass documentation`_). Encountering types not supported by RPython +will raise a ``TypeError``. + +``enforceargs()`` by default also performs type checking of parameter types +each time the function is invoked. To disable this behavior, it's possible to +pass the ``typecheck=False`` parameter to the decorator. + +.. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator + + Exception rules --------------- diff --git a/rpython/doc/translation.rst b/rpython/doc/translation.rst --- a/rpython/doc/translation.rst +++ b/rpython/doc/translation.rst @@ -48,7 +48,7 @@ be present in memory as a form that is "static enough" in the sense of :doc:`RPython `. -2. The Annotator_ performs a global analysis starting from an specified +2. The Annotator_ performs a global analysis starting from a specified entry point to deduce type and other information about what each variable can contain at run-time, :ref:`building flow graphs ` as it encounters them. diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -1592,8 +1592,7 @@ index = self.get_finalizer_queue_index(hop) c_index = rmodel.inputconst(lltype.Signed, index) v_ptr = hop.spaceop.args[1] - v_ptr = hop.genop("cast_opaque_ptr", [v_ptr], - resulttype=llmemory.GCREF) + assert v_ptr.concretetype == llmemory.GCREF hop.genop("direct_call", [self.register_finalizer_ptr, self.c_const_gc, c_index, v_ptr]) diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py --- a/rpython/memory/gcwrapper.py +++ b/rpython/memory/gcwrapper.py @@ -235,11 +235,11 @@ obj = deque.popleft() else: obj = llmemory.NULL - return llmemory.cast_adr_to_ptr(obj, rclass.OBJECTPTR) + return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) def gc_fq_register(self, fq_tag, ptr): index = self.get_finalizer_queue_index(fq_tag) - ptr = lltype.cast_opaque_ptr(llmemory.GCREF, ptr) + assert lltype.typeOf(ptr) == llmemory.GCREF self.gc.register_finalizer(index, ptr) # ____________________________________________________________ diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -120,7 +120,7 @@ """ Decorate a function with forcing of RPython-level types on arguments. None means no enforcing. - When not translated, the type of the actual arguments are checked against + When not translated, the type of the actual arguments is checked against the enforced types every time the function is called. You can disable the typechecking by passing ``typecheck=False`` to @enforceargs. """ @@ -147,8 +147,7 @@ # they are already homogeneous, so we only check the first # item. The case of empty list/dict is handled inside typecheck() if isinstance(arg, list): - item = arg[0] - return [get_type_descr_of_argument(item)] + return [get_type_descr_of_argument(arg[0])] elif isinstance(arg, dict): key, value = next(arg.iteritems()) return {get_type_descr_of_argument(key): get_type_descr_of_argument(value)} diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -378,13 +378,12 @@ class FinalizerQueue(object): """A finalizer queue. See pypy/doc/discussion/finalizer-order.rst. - Note: only works with the framework GCs (like minimark). It is - ignored with Boehm or with refcounting (used by tests). """ # Must be subclassed, and the subclass needs these attributes: # # Class: # the class (or base class) of finalized objects + # --or-- None to handle low-level GCREFs directly # # def finalizer_trigger(self): # called to notify that new items have been put in the queue @@ -397,11 +396,13 @@ def next_dead(self): if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance + from rpython.rtyper.lltypesystem.llmemory import GCREF + from rpython.rtyper.annlowlevel import cast_gcref_to_instance tag = FinalizerQueue._get_tag(self) - ptr = llop.gc_fq_next_dead(OBJECTPTR, tag) - return cast_base_ptr_to_instance(self.Class, ptr) + ptr = llop.gc_fq_next_dead(GCREF, tag) + if self.Class is not None: + ptr = cast_gcref_to_instance(self.Class, ptr) + return ptr try: return self._queue.popleft() except (AttributeError, IndexError): @@ -410,14 +411,18 @@ @specialize.arg(0) @jit.dont_look_inside def register_finalizer(self, obj): - assert isinstance(obj, self.Class) + from rpython.rtyper.lltypesystem.llmemory import GCREF + if self.Class is None: + assert lltype.typeOf(obj) == GCREF + else: + assert isinstance(obj, self.Class) if we_are_translated(): from rpython.rtyper.lltypesystem.lloperation import llop - from rpython.rtyper.rclass import OBJECTPTR - from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr + from rpython.rtyper.annlowlevel import cast_instance_to_gcref tag = FinalizerQueue._get_tag(self) - ptr = cast_instance_to_base_ptr(obj) - llop.gc_fq_register(lltype.Void, tag, ptr) + if self.Class is not None: + obj = cast_instance_to_gcref(obj) + llop.gc_fq_register(lltype.Void, tag, obj) return else: self._untranslated_register_finalizer(obj) diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py --- a/rpython/rlib/test/test_rgc.py +++ b/rpython/rlib/test/test_rgc.py @@ -599,3 +599,94 @@ e = py.test.raises(TyperError, gengraph, f, []) assert str(e.value).startswith('the RPython-level __del__() method in') + + def test_translated_boehm(self): + self._test_translated(use_gc="boehm", llcase=False) + + def test_translated_boehm_ll(self): + self._test_translated(use_gc="boehm", llcase=True) + + def test_translated_incminimark(self): + self._test_translated(use_gc="incminimark", llcase=False) + + def test_translated_incminimark_ll(self): + self._test_translated(use_gc="incminimark", llcase=True) + + def _test_translated(self, use_gc, llcase): + import subprocess + from rpython.rlib import objectmodel + from rpython.translator.interactive import Translation + # + class Seen: + count = 0 + class MySimpleFQ(rgc.FinalizerQueue): + if not llcase: + Class = T_Root + else: + Class = None + def finalizer_trigger(self): + seen.count += 1 + seen = Seen() + fq = MySimpleFQ() + if not llcase: + EMPTY = None + llbuilder = T_Int + else: + from rpython.rtyper.annlowlevel import llstr + EMPTY = lltype.nullptr(llmemory.GCREF.TO) + def llbuilder(n): + return lltype.cast_opaque_ptr(llmemory.GCREF, llstr(str(n))) + + def subfunc(): + w0 = llbuilder(40); fq.register_finalizer(w0) + w1 = llbuilder(41); fq.register_finalizer(w1) + w2 = llbuilder(42); fq.register_finalizer(w2) + w3 = llbuilder(43); fq.register_finalizer(w3) + w4 = llbuilder(44); fq.register_finalizer(w4) + w5 = llbuilder(45); fq.register_finalizer(w5) + w6 = llbuilder(46); fq.register_finalizer(w6) + w7 = llbuilder(47); fq.register_finalizer(w7) + w8 = llbuilder(48); fq.register_finalizer(w8) + w9 = llbuilder(49); fq.register_finalizer(w9) + gc.collect() + assert seen.count == 0 + assert fq.next_dead() is EMPTY + objectmodel.keepalive_until_here(w0) + objectmodel.keepalive_until_here(w1) + objectmodel.keepalive_until_here(w2) + objectmodel.keepalive_until_here(w3) + objectmodel.keepalive_until_here(w4) + objectmodel.keepalive_until_here(w5) + objectmodel.keepalive_until_here(w6) + objectmodel.keepalive_until_here(w7) + objectmodel.keepalive_until_here(w8) + objectmodel.keepalive_until_here(w9) + From pypy.commits at gmail.com Mon Mar 26 18:16:18 2018 From: pypy.commits at gmail.com (mattip) Date: Mon, 26 Mar 2018 15:16:18 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5ab97132.77a9df0a.4d889.ee45@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r94143:c564ff0e3d7d Date: 2018-03-27 01:15 +0300 http://bitbucket.org/pypy/pypy/changeset/c564ff0e3d7d/ Log: merge default into branch diff too long, truncating to 2000 out of 2249 lines diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -4,42 +4,40 @@ Welcome to PyPy! -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. +PyPy is an interperter that implements the Python programming language, based +on the RPython compiler framework for dynamic language implementations. -The home page is: +The home page for the interpreter is: http://pypy.org/ -If you want to help developing PyPy, this document might help you: +If you want to help developing PyPy, this documentation might help you: http://doc.pypy.org/ -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! +More documentation about the RPython framework can be found here - the pypy-dev team + http://rpython.readthedocs.io +The source for the documentation is in the pypy/doc directory + +Using PyPy instead of CPython +============================= + +Please read the information at http://pypy.org to find the correct way to +download and use PyPy as an alternative to CPython. Building ======== -First switch to or download the correct branch. The basic choices are -``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X -branch (e.g. ``py3.5``). +Building PyPy is not the recommended way to obtain the PyPy alternative python +interpreter. It is time-consuming and requires significant computing resources. +More information can be found here -Build with: + http://doc.pypy.org/en/latest/build.html -.. code-block:: console +Enjoy and send us feedback! - $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py + the pypy-dev team -This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy -directory. We suggest to use virtualenv with the resulting -pypy-c/pypy3-c as the interpreter; you can find more details about -various installation schemes here: - http://doc.pypy.org/en/latest/install.html diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst --- a/pypy/doc/install.rst +++ b/pypy/doc/install.rst @@ -17,13 +17,18 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~ The quickest way to start using PyPy is to download a prebuilt binary for your -OS and architecture. You can either use the `most recent release`_ or one of -our `development nightly build`_. Please note that the nightly builds are not +OS and architecture. You may be able to use either use the +`most recent release`_ or one of our `development nightly build`_. These +builds depend on dynamically linked libraries that may not be available on your +OS. See the section about `Linux binaries` for more info and alternatives that +may work on your system. + +Please note that the nightly builds are not guaranteed to be as stable as official releases, use them at your own risk. .. _most recent release: http://pypy.org/download.html .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/ - +.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions Installing PyPy ~~~~~~~~~~~~~~~ @@ -69,9 +74,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is often convenient to run pypy inside a virtualenv. To do this -you need a recent version of virtualenv -- 1.6.1 or greater. You can +you need a version of virtualenv -- 1.6.1 or greater. You can then install PyPy both from a precompiled tarball or from a mercurial -checkout:: +checkout after translation:: # from a tarball $ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -55,6 +55,10 @@ Speed up branchy code that does a lot of function inlining by saving one call to read the TLS in most bridges. +.. branch: rpython-sprint + +Refactor in rpython signatures + .. branch: unicode-utf8-re .. branch: utf8-io Utf8 handling for unicode diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -319,12 +319,13 @@ n = 0 last_pos = ctx.ZERO while not count or n < count: + pattern = ctx.pattern sub_jitdriver.jit_merge_point( self=self, use_builder=use_builder, filter_is_callable=filter_is_callable, filter_type=type(w_filter), - ctx=ctx, + ctx=ctx, pattern=pattern, w_filter=w_filter, strbuilder=strbuilder, filter_as_string=filter_as_string, @@ -401,7 +402,7 @@ filter_as_string w_string sublist_w self""".split(), - greens=["filter_is_callable", "use_builder", "filter_type", "ctx.pattern"]) + greens=["filter_is_callable", "use_builder", "filter_type", "pattern"]) def _sub_append_slice(ctx, space, use_builder, sublist_w, diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -133,6 +133,11 @@ 'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3 constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name) +# PyPy-specific flags +for name in ('FLOAT',): + constant_names.append('Py_TPPYPYFLAGS_%s_SUBCLASS' % name) + + for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) globals().update(rffi_platform.configure(CConfig_constants)) diff --git a/pypy/module/cpyext/boolobject.py b/pypy/module/cpyext/boolobject.py --- a/pypy/module/cpyext/boolobject.py +++ b/pypy/module/cpyext/boolobject.py @@ -1,9 +1,5 @@ -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL, - build_type_checkers) - -# Inheriting from bool isn't actually possible. -PyBool_Check = build_type_checkers("Bool")[1] +from rpython.rtyper.lltypesystem import rffi +from pypy.module.cpyext.api import cpython_api, PyObject @cpython_api([rffi.LONG], PyObject) def PyBool_FromLong(space, value): diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py --- a/pypy/module/cpyext/floatobject.py +++ b/pypy/module/cpyext/floatobject.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (PyObjectFields, bootstrap_function, cpython_struct, - CANNOT_FAIL, cpython_api, PyObject, build_type_checkers, CONST_STRING) + CANNOT_FAIL, cpython_api, PyObject, CONST_STRING) from pypy.module.cpyext.pyobject import ( make_typedescr, track_reference, from_ref) from pypy.interpreter.error import OperationError @@ -38,8 +38,6 @@ track_reference(space, obj, w_obj) return w_obj -PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float") - @cpython_api([lltype.Float], PyObject) def PyFloat_FromDouble(space, value): return space.newfloat(value) diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -82,10 +82,10 @@ def PyTraceBack_Here(space, w_frame): from pypy.interpreter.pytraceback import record_application_traceback state = space.fromcache(State) - if state.operror is None: + if state.get_exception() is None: return -1 frame = space.interp_w(PyFrame, w_frame) - record_application_traceback(space, state.operror, frame, 0) + record_application_traceback(space, state.get_exception(), frame, 0) return 0 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h --- a/pypy/module/cpyext/include/boolobject.h +++ b/pypy/module/cpyext/include/boolobject.h @@ -16,6 +16,8 @@ #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h --- a/pypy/module/cpyext/include/floatobject.h +++ b/pypy/module/cpyext/include/floatobject.h @@ -32,6 +32,11 @@ return PyFloat_FromDouble(-Py_HUGE_VAL); \ } while(0) +#define PyFloat_Check(op) \ + _PyPy_Type_FastSubclass((op)->ob_type, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) +#define PyFloat_CheckExact(op) ((op)->ob_type == &PyFloat_Type) + + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -228,6 +228,11 @@ #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1L<<30) #define Py_TPFLAGS_TYPE_SUBCLASS (1L<<31) +/* These are conceptually the same as the flags above, but they are + PyPy-specific and are stored inside tp_pypy_flags */ +#define Py_TPPYPYFLAGS_FLOAT_SUBCLASS (1L<<0) + + #define Py_TPFLAGS_DEFAULT_EXTERNAL ( \ Py_TPFLAGS_HAVE_GETCHARBUFFER | \ Py_TPFLAGS_HAVE_SEQUENCE_IN | \ @@ -247,6 +252,8 @@ #define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) #define PyType_FastSubclass(t,f) PyType_HasFeature(t,f) +#define _PyPy_Type_FastSubclass(t,f) (((t)->tp_pypy_flags & (f)) != 0) + #define PyType_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS) #define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type) diff --git a/pypy/module/cpyext/include/sliceobject.h b/pypy/module/cpyext/include/sliceobject.h --- a/pypy/module/cpyext/include/sliceobject.h +++ b/pypy/module/cpyext/include/sliceobject.h @@ -17,6 +17,8 @@ PyObject *step; } PySliceObject; +#define PySlice_Check(op) ((op)->ob_type == &PySlice_Type) + #ifdef __cplusplus } #endif diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -45,6 +45,18 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) +def w_kwargs_from_args(space, __args__): + w_kwargs = None + if __args__.keywords: + # CCC: we should probably have a @jit.look_inside_iff if the + # keyword count is constant, as we do in Arguments.unpack + w_kwargs = space.newdict() + for i in range(len(__args__.keywords)): + key = __args__.keywords[i] + w_obj = __args__.keywords_w[i] + space.setitem(w_kwargs, space.newtext(key), w_obj) + return w_kwargs + class W_PyCFunctionObject(W_Root): _immutable_fields_ = ["flags"] @@ -103,15 +115,7 @@ def call_keywords(self, space, w_self, __args__): func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth) py_args = tuple_from_args_w(space, __args__.arguments_w) - w_kwargs = None - if __args__.keywords: - # CCC: we should probably have a @jit.look_inside_iff if the - # keyword count is constant, as we do in Arguments.unpack - w_kwargs = space.newdict() - for i in range(len(__args__.keywords)): - key = __args__.keywords[i] - w_obj = __args__.keywords_w[i] - space.setitem(w_kwargs, space.newtext(key), w_obj) + w_kwargs = w_kwargs_from_args(space, __args__) try: return generic_cpy_call(space, func, w_self, py_args, w_kwargs) finally: @@ -213,14 +217,15 @@ (self.name, self.w_objclass.getname(self.space))) +class W_PyCWrapperObject(W_Root): + """ + Abstract class; for concrete subclasses, see slotdefs.py + """ + _immutable_fields_ = ['offset[*]'] -class W_PyCWrapperObject(W_Root): - def __init__(self, space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func, offset=None): + def __init__(self, space, pto, method_name, doc, func, offset): self.space = space self.method_name = method_name - self.wrapper_func = wrapper_func - self.wrapper_func_kwds = wrapper_func_kwds self.doc = doc self.func = func self.offset = offset @@ -229,10 +234,17 @@ assert isinstance(w_type, W_TypeObject) self.w_objclass = w_type - def call(self, space, w_self, w_args, w_kw): + def descr_call(self, space, w_self, __args__): + return self.call(space, w_self, __args__) + + def call(self, space, w_self, __args__): + raise NotImplementedError + + @jit.unroll_safe + def get_func_to_call(self): func_to_call = self.func if self.offset: - pto = as_pyobj(space, self.w_objclass) + pto = as_pyobj(self.space, self.w_objclass) # make ptr the equivalent of this, using the offsets #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply) if pto: @@ -246,31 +258,33 @@ assert False, "failed to convert w_type %s to PyObject" % str( self.w_objclass) assert func_to_call - if self.wrapper_func is None: - assert self.wrapper_func_kwds is not None - return self.wrapper_func_kwds(space, w_self, w_args, func_to_call, - w_kw) - if space.is_true(w_kw): - raise oefmt(space.w_TypeError, + return func_to_call + + def check_args(self, __args__, arity): + length = len(__args__.arguments_w) + if length != arity: + raise oefmt(self.space.w_TypeError, "expected %d arguments, got %d", + arity, length) + if __args__.keywords: + raise oefmt(self.space.w_TypeError, "wrapper %s doesn't take any keyword arguments", self.method_name) - return self.wrapper_func(space, w_self, w_args, func_to_call) + + def check_argsv(self, __args__, min, max): + length = len(__args__.arguments_w) + if not min <= length <= max: + raise oefmt(self.space.w_TypeError, "expected %d-%d arguments, got %d", + min, max, length) + if __args__.keywords: + raise oefmt(self.space.w_TypeError, + "wrapper %s doesn't take any keyword arguments", + self.method_name) def descr_method_repr(self): return self.space.newtext("" % (self.method_name, self.w_objclass.name)) - at jit.dont_look_inside -def cwrapper_descr_call(space, w_self, __args__): - self = space.interp_w(W_PyCWrapperObject, w_self) - args_w, kw_w = __args__.unpack() - w_args = space.newtuple(args_w[1:]) - w_self = args_w[0] - w_kw = space.newdict() - for key, w_obj in kw_w.items(): - space.setitem(w_kw, space.newtext(key), w_obj) - return self.call(space, w_self, w_args, w_kw) def cmethod_descr_get(space, w_function, w_obj, w_cls=None): asking_for_bound = (space.is_none(w_cls) or @@ -323,7 +337,7 @@ W_PyCWrapperObject.typedef = TypeDef( 'wrapper_descriptor', - __call__ = interp2app(cwrapper_descr_call), + __call__ = interp2app(W_PyCWrapperObject.descr_call), __get__ = interp2app(cmethod_descr_get), __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject, wrapfn="newtext_or_none"), diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -311,6 +311,10 @@ /* Type attribute cache version tag. Added in version 2.6 */ unsigned int tp_version_tag; + /* PyPy specific extra fields: make sure that they are ALWAYS at the end, + for compatibility with CPython */ + long tp_pypy_flags; + } PyTypeObject; typedef struct _heaptypeobject { diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -31,9 +31,10 @@ @cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) - if state.operror is None: + operror = state.get_exception() + if operror is None: return None - return state.operror.w_type # borrowed ref + return operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -47,7 +47,6 @@ from pypy.module.cpyext.object import _dealloc _dealloc(space, py_obj) -PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice") @cpython_api([PyObject, PyObject, PyObject], PyObject) def PySlice_New(space, w_start, w_stop, w_step): @@ -75,9 +74,8 @@ normal slices. Returns 0 on success and -1 on error with exception set.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \ w_slice.indices4(space, length) return 0 @@ -97,9 +95,8 @@ objects in versions of Python prior to 2.3, you would probably do well to incorporate the source of PySlice_GetIndicesEx(), suitably renamed, in the source of your extension.""" - if not PySlice_Check(space, w_slice): + if not isinstance(w_slice, W_SliceObject): PyErr_BadInternalCall(space) - assert isinstance(w_slice, W_SliceObject) start_p[0], stop_p[0], step_p[0] = \ w_slice.indices3(space, length) return 0 diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -19,6 +19,8 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext import userslot from pypy.module.cpyext.buffer import CBuffer, CPyBuffer, fq +from pypy.module.cpyext.methodobject import (W_PyCWrapperObject, tuple_from_args_w, + w_kwargs_from_args) from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable @@ -38,29 +40,6 @@ Py_GT = 4 Py_GE = 5 - -def check_num_args(space, w_ob, n): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if n == space.len_w(w_ob): - return - raise oefmt(space.w_TypeError, - "expected %d arguments, got %d", - n, space.len_w(w_ob)) - -def check_num_argsv(space, w_ob, low, high): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact - if not PyTuple_CheckExact(space, w_ob): - raise oefmt(space.w_SystemError, - "PyArg_UnpackTuple() argument list is not a tuple") - if low <=space.len_w(w_ob) <= high: - return - raise oefmt(space.w_TypeError, - "expected %d-%d arguments, got %d", - low, high, space.len_w(w_ob)) - @not_rpython def llslot(space, func): return func.api_func.get_llhelper(space) @@ -71,337 +50,413 @@ get_llhelper = v_func.value.api_func.get_llhelper return ctx.appcall(get_llhelper, v_space) +# NOTE: the following wrap_* are subclasses of W_PyCWrapperObject, even if +# they don't follow the usual W_* naming convention for subclasses of W_Root: +# we do this because we automatically generate most of the slots from the +# CPython code copy&pasted inside slotdefs_str, and thus we need to keep the +# same names as they are used in C. -def wrap_init(space, w_self, w_args, func, w_kwargs): - func_init = rffi.cast(initproc, func) - res = generic_cpy_call(space, func_init, w_self, w_args, w_kwargs) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return None +class wrap_init(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_init = rffi.cast(initproc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return None -def wrap_unaryfunc(space, w_self, w_args, func): - func_unary = rffi.cast(unaryfunc, func) - check_num_args(space, w_args, 0) - return generic_cpy_call(space, func_unary, w_self) +class wrap_unaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_unary = rffi.cast(unaryfunc, func) + return generic_cpy_call(space, func_unary, w_self) -def wrap_binaryfunc(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_binary, w_self, args_w[0]) +class wrap_binaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_x = __args__.arguments_w[0] + return generic_cpy_call(space, func_binary, w_self, w_x) def _get_ob_type(space, w_obj): # please ensure that w_obj stays alive ob_type = as_pyobj(space, space.type(w_obj)) return rffi.cast(PyTypeObjectPtr, ob_type) -def wrap_binaryfunc_l(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - ob_type = _get_ob_type(space, w_self) - if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and - not space.issubtype_w(space.type(args_w[0]), space.type(w_self))): - return space.w_NotImplemented - return generic_cpy_call(space, func_binary, w_self, args_w[0]) +class wrap_binaryfunc_l(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_value = __args__.arguments_w[0] + ob_type = _get_ob_type(space, w_self) + if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.issubtype_w(space.type(w_value), space.type(w_self))): + return space.w_NotImplemented + return generic_cpy_call(space, func_binary, w_self, w_value) -def wrap_binaryfunc_r(space, w_self, w_args, func): - func_binary = rffi.cast(binaryfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - ob_type = _get_ob_type(space, w_self) - if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and - not space.issubtype_w(space.type(args_w[0]), space.type(w_self))): - return space.w_NotImplemented - return generic_cpy_call(space, func_binary, args_w[0], w_self) +class wrap_binaryfunc_r(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_binary = rffi.cast(binaryfunc, func) + w_value = __args__.arguments_w[0] + ob_type = _get_ob_type(space, w_self) + if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.issubtype_w(space.type(w_value), space.type(w_self))): + return space.w_NotImplemented + return generic_cpy_call(space, func_binary, w_value, w_self) -def wrap_ternaryfunc(space, w_self, w_args, func): - # The third argument is optional - func_ternary = rffi.cast(ternaryfunc, func) - check_num_argsv(space, w_args, 1, 2) - args_w = space.fixedview(w_args) - arg3 = space.w_None - if len(args_w) > 1: - arg3 = args_w[1] - return generic_cpy_call(space, func_ternary, w_self, args_w[0], arg3) +class wrap_ternaryfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + # The third argument is optional + self.check_argsv(__args__, 1, 2) + func = self.get_func_to_call() + func_ternary = rffi.cast(ternaryfunc, func) + w_arg0 = __args__.arguments_w[0] + if len(__args__.arguments_w) == 2: + w_arg1 = __args__.arguments_w[1] + else: + w_arg1 = space.w_None + return generic_cpy_call(space, func_ternary, w_self, w_arg0, w_arg1) -def wrap_ternaryfunc_r(space, w_self, w_args, func): - # The third argument is optional - func_ternary = rffi.cast(ternaryfunc, func) - check_num_argsv(space, w_args, 1, 2) - args_w = space.fixedview(w_args) - ob_type = _get_ob_type(space, w_self) - if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and - not space.issubtype_w(space.type(args_w[0]), space.type(w_self))): - return space.w_NotImplemented - arg3 = space.w_None - if len(args_w) > 1: - arg3 = args_w[1] - return generic_cpy_call(space, func_ternary, args_w[0], w_self, arg3) +class wrap_ternaryfunc_r(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + # The third argument is optional + self.check_argsv(__args__, 1, 2) + func = self.get_func_to_call() + func_ternary = rffi.cast(ternaryfunc, func) + w_arg0 = __args__.arguments_w[0] + if len(__args__.arguments_w) == 2: + w_arg1 = __args__.arguments_w[1] + else: + w_arg1 = space.w_None + ob_type = _get_ob_type(space, w_self) + if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and + not space.issubtype_w(space.type(w_arg0), space.type(w_self))): + return space.w_NotImplemented + return generic_cpy_call(space, func_ternary, w_arg0, w_self, w_arg1) +class wrap_inquirypred(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_inquiry = rffi.cast(inquiry, func) + res = generic_cpy_call(space, func_inquiry, w_self) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) -def wrap_inquirypred(space, w_self, w_args, func): - func_inquiry = rffi.cast(inquiry, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_inquiry, w_self) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class wrap_getattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrfunc, func) + w_name = __args__.arguments_w[0] + name_ptr = rffi.str2charp(space.text_w(w_name)) + try: + return generic_cpy_call(space, func_target, w_self, name_ptr) + finally: + rffi.free_charp(name_ptr) -def wrap_getattr(space, w_self, w_args, func): - func_target = rffi.cast(getattrfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - name_ptr = rffi.str2charp(space.text_w(args_w[0])) - try: - return generic_cpy_call(space, func_target, w_self, name_ptr) - finally: - rffi.free_charp(name_ptr) +class wrap_getattro(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(getattrofunc, func) + w_name = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, w_self, w_name) -def wrap_getattro(space, w_self, w_args, func): - func_target = rffi.cast(getattrofunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - return generic_cpy_call(space, func_target, w_self, args_w[0]) +class wrap_setattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_setattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 2) - w_name, w_value = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_delattr(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(setattrofunc, func) + w_name = __args__.arguments_w[0] + # XXX "Carlo Verre hack"? + res = generic_cpy_call(space, func_target, w_self, w_name, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_delattr(space, w_self, w_args, func): - func_target = rffi.cast(setattrofunc, func) - check_num_args(space, w_args, 1) - w_name, = space.fixedview(w_args) - # XXX "Carlo Verre hack"? - res = generic_cpy_call(space, func_target, w_self, w_name, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_descr_get(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(descrgetfunc, func) + length = len(__args__.arguments_w) + if length == 1: + w_obj = __args__.arguments_w[0] + w_type = None + elif length == 2: + w_obj = __args__.arguments_w[0] + w_type = __args__.arguments_w[1] + else: + raise oefmt(space.w_TypeError, + "expected 1 or 2 arguments, got %d", len(__args__.arguments_w)) + if w_obj is space.w_None: + w_obj = None + if w_type is space.w_None: + w_type = None + if w_obj is None and w_type is None: + raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") + return generic_cpy_call(space, func_target, w_self, w_obj, w_type) -def wrap_descr_get(space, w_self, w_args, func): - func_target = rffi.cast(descrgetfunc, func) - args_w = space.fixedview(w_args) - if len(args_w) == 1: - w_obj, = args_w - w_type = None - elif len(args_w) == 2: - w_obj, w_type = args_w - else: - raise oefmt(space.w_TypeError, - "expected 1 or 2 arguments, got %d", len(args_w)) - if w_obj is space.w_None: - w_obj = None - if w_type is space.w_None: - w_type = None - if w_obj is None and w_type is None: - raise oefmt(space.w_TypeError, "__get__(None, None) is invalid") - return generic_cpy_call(space, func_target, w_self, w_obj, w_type) +class wrap_descr_set(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(descrsetfunc, func) + w_obj = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + res = generic_cpy_call(space, func_target, w_self, w_obj, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_descr_set(space, w_self, w_args, func): - func_target = rffi.cast(descrsetfunc, func) - check_num_args(space, w_args, 2) - w_obj, w_value = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_obj, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_descr_delete(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(descrsetfunc, func) + w_obj = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_obj, None) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_descr_delete(space, w_self, w_args, func): - func_target = rffi.cast(descrsetfunc, func) - check_num_args(space, w_args, 1) - w_obj, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_obj, None) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_call(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(ternaryfunc, func) + py_args = tuple_from_args_w(space, __args__.arguments_w) + w_kwargs = w_kwargs_from_args(space, __args__) + return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs) -def wrap_call(space, w_self, w_args, func, w_kwds): - func_target = rffi.cast(ternaryfunc, func) - return generic_cpy_call(space, func_target, w_self, w_args, w_kwds) +class wrap_ssizessizeobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 3) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeobjargproc, func) + i = space.int_w(space.index(__args__.arguments_w[0])) + j = space.int_w(space.index(__args__.arguments_w[1])) + w_y = __args__.arguments_w[2] + res = generic_cpy_call(space, func_target, w_self, i, j, w_y) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_ssizessizeobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeobjargproc, func) - check_num_args(space, w_args, 3) - args_w = space.fixedview(w_args) - i = space.int_w(space.index(args_w[0])) - j = space.int_w(space.index(args_w[1])) - w_y = args_w[2] - res = generic_cpy_call(space, func_target, w_self, i, j, w_y) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_lenfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_len = rffi.cast(lenfunc, func) + res = generic_cpy_call(space, func_len, w_self) + if widen(res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) -def wrap_lenfunc(space, w_self, w_args, func): - func_len = rffi.cast(lenfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_len, w_self) - if widen(res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_sq_item(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeargfunc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + return generic_cpy_call(space, func_target, w_self, index) -def wrap_sq_item(space, w_self, w_args, func): - func_target = rffi.cast(ssizeargfunc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - return generic_cpy_call(space, func_target, w_self, index) +class wrap_sq_setitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + index = space.int_w(space.index(w_index)) + res = generic_cpy_call(space, func_target, w_self, index, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) -def wrap_sq_setitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - res = generic_cpy_call(space, func_target, w_self, index, args_w[1]) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - -def wrap_sq_delitem(space, w_self, w_args, func): - func_target = rffi.cast(ssizeobjargproc, func) - check_num_args(space, w_args, 1) - args_w = space.fixedview(w_args) - index = space.int_w(space.index(args_w[0])) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, index, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) +class wrap_sq_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(ssizeobjargproc, func) + w_index = __args__.arguments_w[0] + index = space.int_w(space.index(w_index)) + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, index, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) # Warning, confusing function name (like CPython). Used only for sq_contains. -def wrap_objobjproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjproc, func) - check_num_args(space, w_args, 1) - w_value, = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_value) - res = rffi.cast(lltype.Signed, res) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newbool(bool(res)) +class wrap_objobjproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjproc, func) + w_value = __args__.arguments_w[0] + res = generic_cpy_call(space, func_target, w_self, w_value) + res = rffi.cast(lltype.Signed, res) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newbool(bool(res)) -def wrap_objobjargproc(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 2) - w_key, w_value = space.fixedview(w_args) - res = generic_cpy_call(space, func_target, w_self, w_key, w_value) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_objobjargproc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + w_value = __args__.arguments_w[1] + res = generic_cpy_call(space, func_target, w_self, w_key, w_value) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None -def wrap_delitem(space, w_self, w_args, func): - func_target = rffi.cast(objobjargproc, func) - check_num_args(space, w_args, 1) - w_key, = space.fixedview(w_args) - null = rffi.cast(PyObject, 0) - res = generic_cpy_call(space, func_target, w_self, w_key, null) - if rffi.cast(lltype.Signed, res) == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.w_None +class wrap_delitem(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(objobjargproc, func) + w_key = __args__.arguments_w[0] + null = rffi.cast(PyObject, 0) + res = generic_cpy_call(space, func_target, w_self, w_key, null) + if rffi.cast(lltype.Signed, res) == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.w_None -def wrap_ssizessizeargfunc(space, w_self, w_args, func): - func_target = rffi.cast(ssizessizeargfunc, func) - check_num_args(space, w_args, 2) - args_w = space.fixedview(w_args) - start = space.int_w(args_w[0]) - end = space.int_w(args_w[1]) - return generic_cpy_call(space, func_target, w_self, start, end) +class wrap_ssizessizeargfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 2) + func = self.get_func_to_call() + func_target = rffi.cast(ssizessizeargfunc, func) + start = space.int_w(__args__.arguments_w[0]) + end = space.int_w(__args__.arguments_w[1]) + return generic_cpy_call(space, func_target, w_self, start, end) -def wrap_next(space, w_self, w_args, func): - from pypy.module.cpyext.api import generic_cpy_call_expect_null - func_target = rffi.cast(iternextfunc, func) - check_num_args(space, w_args, 0) - w_res = generic_cpy_call_expect_null(space, func_target, w_self) - if not w_res and not PyErr_Occurred(space): - raise OperationError(space.w_StopIteration, space.w_None) - return w_res +class wrap_next(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + from pypy.module.cpyext.api import generic_cpy_call_expect_null + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(iternextfunc, func) + w_res = generic_cpy_call_expect_null(space, func_target, w_self) + if not w_res and not PyErr_Occurred(space): + raise OperationError(space.w_StopIteration, space.w_None) + return w_res -def wrap_hashfunc(space, w_self, w_args, func): - func_target = rffi.cast(hashfunc, func) - check_num_args(space, w_args, 0) - res = generic_cpy_call(space, func_target, w_self) - if res == -1: - space.fromcache(State).check_and_raise_exception(always=True) - return space.newint(res) +class wrap_hashfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 0) + func = self.get_func_to_call() + func_target = rffi.cast(hashfunc, func) + res = generic_cpy_call(space, func_target, w_self) + if res == -1: + space.fromcache(State).check_and_raise_exception(always=True) + return space.newint(res) -def wrap_getreadbuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getreadbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) -def wrap_getwritebuffer(space, w_self, w_args, func): - func_target = rffi.cast(readbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: - index = rffi.cast(Py_ssize_t, 0) - size = generic_cpy_call(space, func_target, w_self, index, ptr) - if size < 0: - space.fromcache(State).check_and_raise_exception(always=True) - view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, - releasebufferproc=rbp) - fq.register_finalizer(view) - return space.newbuffer(CBuffer(view)) +class wrap_getwritebuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(readbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr: + index = rffi.cast(Py_ssize_t, 0) + size = generic_cpy_call(space, func_target, w_self, index, ptr) + if size < 0: + space.fromcache(State).check_and_raise_exception(always=True) + view = CPyBuffer(space, ptr[0], size, w_self, readonly=False, + releasebufferproc=rbp) + fq.register_finalizer(view) + return space.newbuffer(CBuffer(view)) -def wrap_getbuffer(space, w_self, w_args, func): - func_target = rffi.cast(getbufferproc, func) - py_type = _get_ob_type(space, w_self) - rbp = rffi.cast(rffi.VOIDP, 0) - if py_type.c_tp_as_buffer: - rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) - with lltype.scoped_alloc(Py_buffer) as pybuf: - _flags = 0 - if space.len_w(w_args) > 0: - _flags = space.int_w(space.listview(w_args)[0]) - flags = rffi.cast(rffi.INT_real,_flags) - size = generic_cpy_call(space, func_target, w_self, pybuf, flags) - if widen(size) < 0: - space.fromcache(State).check_and_raise_exception(always=True) - ptr = pybuf.c_buf - size = pybuf.c_len - ndim = widen(pybuf.c_ndim) - shape = None - if pybuf.c_shape: - shape = [pybuf.c_shape[i] for i in range(ndim)] - strides = None - if pybuf.c_strides: - strides = [pybuf.c_strides[i] for i in range(ndim)] - if pybuf.c_format: - format = rffi.charp2str(pybuf.c_format) - else: - format = 'B' - # the CPython docs mandates that you do an incref whenever you call - # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't - # leak we release the buffer: - # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer - buf = CPyBuffer(space, ptr, size, w_self, format=format, - ndim=ndim, shape=shape, strides=strides, - itemsize=pybuf.c_itemsize, - readonly=widen(pybuf.c_readonly), - needs_decref=True, - releasebufferproc = rbp) - fq.register_finalizer(buf) - return buf.wrap(space) + +class wrap_getbuffer(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + func = self.get_func_to_call() + func_target = rffi.cast(getbufferproc, func) + py_type = _get_ob_type(space, w_self) + rbp = rffi.cast(rffi.VOIDP, 0) + if py_type.c_tp_as_buffer: + rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer) + with lltype.scoped_alloc(Py_buffer) as pybuf: + _flags = 0 + if len(__args__.arguments_w) > 0: + _flags = space.int_w(__args__.arguments_w[0]) + flags = rffi.cast(rffi.INT_real,_flags) + size = generic_cpy_call(space, func_target, w_self, pybuf, flags) + if widen(size) < 0: + space.fromcache(State).check_and_raise_exception(always=True) + ptr = pybuf.c_buf + size = pybuf.c_len + ndim = widen(pybuf.c_ndim) + shape = None + if pybuf.c_shape: + shape = [pybuf.c_shape[i] for i in range(ndim)] + strides = None + if pybuf.c_strides: + strides = [pybuf.c_strides[i] for i in range(ndim)] + if pybuf.c_format: + format = rffi.charp2str(pybuf.c_format) + else: + format = 'B' + # the CPython docs mandates that you do an incref whenever you call + # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't + # leak we release the buffer: + # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer + buf = CPyBuffer(space, ptr, size, w_self, format=format, + ndim=ndim, shape=shape, strides=strides, + itemsize=pybuf.c_itemsize, + readonly=widen(pybuf.c_readonly), + needs_decref=True, + releasebufferproc = rbp) + fq.register_finalizer(buf) + return buf.wrap(space) def get_richcmp_func(OP_CONST): - def inner(space, w_self, w_args, func): - func_target = rffi.cast(richcmpfunc, func) - check_num_args(space, w_args, 1) - w_other, = space.fixedview(w_args) - return generic_cpy_call(space, func_target, - w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST)) - return inner + class wrap_richcmp(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(richcmpfunc, func) + w_other = __args__.arguments_w[0] + return generic_cpy_call(space, func_target, + w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST)) + return wrap_richcmp richcmp_eq = get_richcmp_func(Py_EQ) richcmp_ne = get_richcmp_func(Py_NE) @@ -410,17 +465,19 @@ richcmp_gt = get_richcmp_func(Py_GT) richcmp_ge = get_richcmp_func(Py_GE) -def wrap_cmpfunc(space, w_self, w_args, func): - func_target = rffi.cast(cmpfunc, func) - check_num_args(space, w_args, 1) - w_other, = space.fixedview(w_args) +class wrap_cmpfunc(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + self.check_args(__args__, 1) + func = self.get_func_to_call() + func_target = rffi.cast(cmpfunc, func) + w_other = __args__.arguments_w[0] - if not space.issubtype_w(space.type(w_self), space.type(w_other)): - raise oefmt(space.w_TypeError, - "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'", - w_self, w_self, w_other) + if not space.issubtype_w(space.type(w_self), space.type(w_other)): + raise oefmt(space.w_TypeError, + "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'", + w_self, w_self, w_other) - return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) + return space.newint(generic_cpy_call(space, func_target, w_self, w_other)) SLOT_FACTORIES = {} def slot_factory(tp_name): @@ -804,9 +861,10 @@ missing_wrappers = ['wrap_indexargfunc', 'wrap_delslice', 'wrap_coercefunc'] for name in missing_wrappers: assert name not in globals() - def missing_wrapper(space, w_self, w_args, func): - print "cpyext: missing slot wrapper " + name - raise NotImplementedError("Slot wrapper " + name) + class missing_wrapper(W_PyCWrapperObject): + def call(self, space, w_self, __args__): + print "cpyext: missing slot wrapper " + name + raise NotImplementedError("Slot wrapper " + name) missing_wrapper.__name__ = name globals()[name] = missing_wrapper @@ -836,13 +894,12 @@ PyWrapperFlag_KEYWORDS = 1 class TypeSlot: - def __init__(self, method_name, slot_name, function, wrapper1, wrapper2, doc): + def __init__(self, method_name, slot_name, function, wrapper, doc): self.method_name = method_name self.slot_name = slot_name self.slot_names = tuple(("c_" + slot_name).split(".")) self.slot_func = function - self.wrapper_func = wrapper1 - self.wrapper_func_kwds = wrapper2 + self.wrapper_class = wrapper self.doc = doc # adapted from typeobject.c @@ -863,13 +920,7 @@ function = getattr(userslot, FUNCTION or '!missing', None) assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS - if FLAGS: - wrapper1 = None - wrapper2 = wrapper - else: - wrapper1 = wrapper - wrapper2 = None - return TypeSlot(NAME, SLOT, function, wrapper1, wrapper2, DOC) + return TypeSlot(NAME, SLOT, function, wrapper, DOC) def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC): return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0) @@ -1158,7 +1209,7 @@ x.slot_func.api_func if x.slot_func else None) for x in slotdefs]) slotdefs_for_wrappers = unrolling_iterable( - [(x.method_name, x.slot_names, x.wrapper_func, x.wrapper_func_kwds, x.doc) + [(x.method_name, x.slot_names, x.wrapper_class, x.doc) for x in slotdefs]) if __name__ == "__main__": diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -2,11 +2,18 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter import executioncontext +from pypy.interpreter.executioncontext import ExecutionContext from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rdynload import DLLHANDLE from rpython.rlib import rawrefcount import sys + +# Keep track of exceptions raised in cpyext for a particular execution +# context. +ExecutionContext.cpyext_operror = None + + class State: def __init__(self, space): self.space = space @@ -18,7 +25,8 @@ def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef - self.operror = None + ec = self.space.getexecutioncontext() + ec.cpyext_operror = None self.new_method_def = lltype.nullptr(PyMethodDef) # When importing a package, use this to keep track @@ -37,17 +45,24 @@ def set_exception(self, operror): self.clear_exception() - self.operror = operror + ec = self.space.getexecutioncontext() + ec.cpyext_operror = operror def clear_exception(self): """Clear the current exception state, and return the operror.""" - operror = self.operror - self.operror = None + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror + ec.cpyext_operror = None return operror + def get_exception(self): + ec = self.space.getexecutioncontext() + return ec.cpyext_operror + @specialize.arg(1) def check_and_raise_exception(self, always=False): - operror = self.operror + ec = self.space.getexecutioncontext() + operror = ec.cpyext_operror if operror: self.clear_exception() raise operror diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -39,7 +39,7 @@ raise Exception("%s is not callable" % (f,)) f(*args) state = space.fromcache(State) - operror = state.operror + operror = state.get_exception() if not operror: raise Exception("DID NOT RAISE") if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type: diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py --- a/pypy/module/cpyext/test/test_boolobject.py +++ b/pypy/module/cpyext/test/test_boolobject.py @@ -1,7 +1,6 @@ from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong -from pypy.module.cpyext.floatobject import PyFloat_FromDouble +from pypy.module.cpyext.boolobject import PyBool_FromLong class TestBoolObject(BaseApiTest): def test_fromlong(self, space): @@ -12,12 +11,6 @@ else: assert obj is space.w_False - def test_check(self, space): - assert PyBool_Check(space, space.w_True) - assert PyBool_Check(space, space.w_False) - assert not PyBool_Check(space, space.w_None) - assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0)) - class AppTestBoolMacros(AppTestCpythonExtensionBase): def test_macros(self): module = self.import_extension('foo', [ @@ -42,4 +35,14 @@ assert module.to_int(False) == 0 assert module.to_int(True) == 1 - + def test_check(self): + module = self.import_extension('foo', [ + ("type_check", "METH_O", + ''' + return PyLong_FromLong(PyBool_Check(args)); + ''')]) + assert module.type_check(True) + assert module.type_check(False) + assert not module.type_check(None) + assert not module.type_check(1.0) + diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -636,7 +636,8 @@ Py_ssize_t refcnt_after; Py_INCREF(true_obj); Py_INCREF(true_obj); - PyBool_Check(true_obj); + if (!PyBool_Check(true_obj)) + Py_RETURN_NONE; refcnt_after = true_obj->ob_refcnt; Py_DECREF(true_obj); Py_DECREF(true_obj); diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -102,9 +102,11 @@ """ PyObject* pyobj = PyFloat_FromDouble(1.0); PyFloatObject* pfo = (PyFloatObject*)pyobj; - int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) && - PyFloat_Check(pfo) && PyFloat_CheckExact(pfo); + int res = (PyFloat_Check(pyobj) + + PyFloat_CheckExact(pyobj) * 10 + + PyFloat_Check(pfo) * 100 + + PyFloat_CheckExact(pfo) * 1000); Py_DecRef(pyobj); return PyLong_FromLong(res);"""), ]) - assert module.test() == 1 + assert module.test() == 1111 diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py --- a/pypy/module/cpyext/test/test_number.py +++ b/pypy/module/cpyext/test/test_number.py @@ -11,7 +11,6 @@ PyNumber_Index, PyNumber_Coerce, PyNumber_CoerceEx, PyNumber_Add, PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute, PyNumber_Power, PyNumber_InPlacePower) -from pypy.module.cpyext.floatobject import PyFloat_Check from pypy.module.cpyext.intobject import PyInt_CheckExact from pypy.module.cpyext.longobject import PyLong_CheckExact from pypy.module.cpyext.object import PyObject_Size @@ -86,7 +85,7 @@ w_res = from_ref(space, ppl[0]) - assert PyFloat_Check(space, w_res) + assert space.isinstance_w(w_res, space.w_float) assert space.unwrap(w_res) == 123. decref(space, pl) decref(space, pf) diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py --- a/pypy/module/cpyext/test/test_pyerrors.py +++ b/pypy/module/cpyext/test/test_pyerrors.py @@ -52,7 +52,8 @@ api.PyErr_SetObject(space.w_ValueError, space.wrap("a value")) assert api.PyErr_Occurred() is space.w_ValueError state = space.fromcache(State) - assert space.eq_w(state.operror.get_w_value(space), + operror = state.get_exception() + assert space.eq_w(operror.get_w_value(space), space.wrap("a value")) api.PyErr_Clear() @@ -60,12 +61,14 @@ def test_SetNone(self, space, api): api.PyErr_SetNone(space.w_KeyError) state = space.fromcache(State) - assert space.eq_w(state.operror.w_type, space.w_KeyError) - assert space.eq_w(state.operror.get_w_value(space), space.w_None) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_KeyError) + assert space.eq_w(operror.get_w_value(space), space.w_None) api.PyErr_Clear() api.PyErr_NoMemory() - assert space.eq_w(state.operror.w_type, space.w_MemoryError) + operror = state.get_exception() + assert space.eq_w(operror.w_type, space.w_MemoryError) api.PyErr_Clear() def test_Warning(self, space, api, capfd): @@ -437,3 +440,59 @@ '''), ]) raises(SystemError, module.oops) + + def test_error_thread_race(self): + # Check race condition: thread 0 returns from cpyext with error set, + # after thread 1 has set an error but before it returns. + module = self.import_extension('foo', [ + ("emit_error", "METH_VARARGS", + ''' + PyThreadState *save = NULL; + PyGILState_STATE gilsave; + + /* NB. synchronization due to GIL */ + static volatile int flag = 0; + int id; + + if (!PyArg_ParseTuple(args, "i", &id)) + return NULL; + + /* Proceed in thread 1 first */ + save = PyEval_SaveThread(); + while (id == 0 && flag == 0); + gilsave = PyGILState_Ensure(); + + PyErr_Format(PyExc_ValueError, "%d", id); + + /* Proceed in thread 0 first */ + if (id == 1) flag = 1; + PyGILState_Release(gilsave); + while (id == 1 && flag == 1); + PyEval_RestoreThread(save); + + if (id == 0) flag = 0; + return NULL; + ''' + ), + ]) + + import threading + + failures = [] + + def worker(arg): + try: + module.emit_error(arg) + failures.append(True) + except Exception as exc: + if str(exc) != str(arg): + failures.append(exc) + + threads = [threading.Thread(target=worker, args=(j,)) + for j in (0, 1)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not failures diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py --- a/pypy/module/cpyext/test/test_sliceobject.py +++ b/pypy/module/cpyext/test/test_sliceobject.py @@ -2,14 +2,8 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP -from pypy.module.cpyext.sliceobject import PySlice_Check class TestSliceObject(BaseApiTest): - def test_slice(self, space): - w_i = space.wrap(10) - w_slice = space.newslice(w_i, w_i, w_i) - assert PySlice_Check(space, w_slice) - assert not PySlice_Check(space, w_i) def test_GetIndicesEx(self, space, api): w = space.wrap @@ -79,3 +73,14 @@ """), ]) assert module.get_ellipsis() is Ellipsis + + def test_typecheck(self): + module = self.import_extension('foo', [ + ("check", "METH_O", + """ + PySliceObject *slice = (PySliceObject *)args; + return PyLong_FromLong(PySlice_Check(slice)); + """), + ]) + s = slice(10, 20, 30) + assert module.check(s) diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -796,6 +796,36 @@ assert module.tp_init(list, x, ("hi",)) is None assert x == ["h", "i"] + def test_mp_subscript(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static PyObject* + mp_subscript(PyObject *self, PyObject *key) + { + return Py_BuildValue("i", 42); + } + PyMappingMethods tp_as_mapping; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init = ''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_mapping = &tp_as_mapping; + tp_as_mapping.mp_subscript = (binaryfunc)mp_subscript; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + assert obj[100] == 42 + raises(TypeError, "obj.__getitem__(100, 101)") + raises(TypeError, "obj.__getitem__(100, a=42)") + def test_mp_ass_subscript(self): module = self.import_extension('foo', [ ("new_obj", "METH_NOARGS", @@ -859,6 +889,84 @@ res = "foo" in obj assert res is True + def test_sq_ass_slice(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) + { + int expected = (a == 10 && b == 20 && + PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_slice = sq_ass_slice; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10:20] = 42 + raises(ValueError, "obj[10:20] = 43") + raises(ValueError, "obj[11:20] = 42") + raises(ValueError, "obj[10:21] = 42") + + def test_sq_ass_item(self): + module = self.import_extension('foo', [ + ("new_obj", "METH_NOARGS", + ''' + PyObject *obj; + obj = PyObject_New(PyObject, &Foo_Type); + return obj; + ''' + )], prologue=''' + static int + sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) + { + int expected; + if (o == NULL) // delitem + expected = (i == 12); + else // setitem + expected = (i == 10 && PyInt_Check(o) && PyInt_AsLong(o) == 42); + if (!expected) { + PyErr_SetString(PyExc_ValueError, "test failed"); + return -1; + } + return 0; + } + PySequenceMethods tp_as_sequence; + static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "foo.foo", + }; + ''', more_init=''' + Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT; + Foo_Type.tp_as_sequence = &tp_as_sequence; + tp_as_sequence.sq_ass_item = sq_ass_item; + if (PyType_Ready(&Foo_Type) < 0) INITERROR; + ''') + obj = module.new_obj() + obj[10] = 42 + raises(ValueError, "obj[10] = 43") + raises(ValueError, "obj[11] = 42") + del obj[12] + raises(ValueError, "del obj[13]") + def test_tp_iter(self): module = self.import_extension('foo', [ ("tp_iter", "METH_VARARGS", @@ -1537,4 +1645,29 @@ pass assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0 + def test_has_pypy_subclass_flag(self): + module = self.import_extension('foo', [ + ("test_pypy_flags", "METH_VARARGS", + ''' + long long in_flag, my_flag; + PyObject * obj; + if (!PyArg_ParseTuple(args, "OL", &obj, &in_flag)) + return NULL; + if (!PyType_Check(obj)) + { + PyErr_SetString(PyExc_ValueError, "input must be type"); + return NULL; + } + my_flag = ((PyTypeObject*)obj)->tp_pypy_flags; + if ((my_flag & in_flag) != in_flag) + return PyLong_FromLong(-1); + return PyLong_FromLong(0); + '''),]) + # copied from object.h + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + class MyFloat(float): + pass + assert module.test_pypy_flags(float, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + assert module.test_pypy_flags(MyFloat, Py_TPPYPYFLAGS_FLOAT_SUBCLASS) == 0 + diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -22,10 +22,11 @@ Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS, Py_TPFLAGS_TYPE_SUBCLASS, Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3 + Py_TPPYPYFLAGS_FLOAT_SUBCLASS, ) from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject, - W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef, - W_PyCMethodObject, W_PyCFunctionObject) + PyCFunction_NewEx, PyCFunction, PyMethodDef, + W_PyCMethodObject, W_PyCFunctionObject, W_PyCWrapperObject) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, from_ref, get_typedescr, make_typedescr, @@ -310,7 +311,7 @@ def add_operators(space, dict_w, pto): from pypy.module.cpyext.object import PyObject_HashNotImplemented hash_not_impl = llslot(space, PyObject_HashNotImplemented) - for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers: + for method_name, slot_names, wrapper_class, doc in slotdefs_for_wrappers: if method_name in dict_w: continue offset = [rffi.offsetof(lltype.typeOf(pto).TO, slot_names[0])] @@ -335,10 +336,11 @@ func_voidp = rffi.cast(rffi.VOIDP, func) if not func: continue - if wrapper_func is None and wrapper_func_kwds is None: + if wrapper_class is None: continue - w_obj = W_PyCWrapperObject(space, pto, method_name, wrapper_func, - wrapper_func_kwds, doc, func_voidp, offset=offset) + + assert issubclass(wrapper_class, W_PyCWrapperObject) + w_obj = wrapper_class(space, pto, method_name, doc, func_voidp, offset=offset[:]) dict_w[method_name] = w_obj if pto.c_tp_doc: dict_w['__doc__'] = space.newtext( @@ -426,6 +428,9 @@ pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS elif space.issubtype_w(w_obj, space.w_dict): pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS + # the following types are a pypy-specific extensions, using tp_pypy_flags + elif space.issubtype_w(w_obj, space.w_float): + pto.c_tp_pypy_flags |= Py_TPPYPYFLAGS_FLOAT_SUBCLASS def check_descr(space, w_self, w_type): if not space.isinstance_w(w_self, w_type): diff --git a/pypy/module/unicodedata/test/test_hyp.py b/pypy/module/unicodedata/test/test_hyp.py --- a/pypy/module/unicodedata/test/test_hyp.py +++ b/pypy/module/unicodedata/test/test_hyp.py @@ -1,7 +1,7 @@ import sys import pytest try: - from hypothesis import given, strategies as st, example, settings + from hypothesis import given, strategies as st, example, settings, assume except ImportError: pytest.skip("hypothesis required") @@ -43,11 +43,14 @@ @pytest.mark.parametrize('NF1, NF2, NF3', compositions) @example(s=u'---\uafb8\u11a7---') # issue 2289 - at example(s=u'\ufacf') @settings(max_examples=1000) @given(s=st.text()) def test_composition(s, space, NF1, NF2, NF3): - if s == u'\ufacf' and sys.maxunicode == 65535: - pytest.skip('chr(0xfacf) normalizes to chr(0x2284a), which is too big') + # 'chr(0xfacf) normalizes to chr(0x2284a), which is too big') + assume(not (s == u'\ufacf' and sys.maxunicode == 65535)) norm1, norm2, norm3 = [make_normalization(space, form) for form in [NF1, NF2, NF3]] assert norm2(norm1(s)) == norm3(s) + +if sys.maxunicode != 65535: + # conditionally generate the example via an unwrapped decorator + test_composition = example(s=u'\ufacf')(test_composition) diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -15,6 +15,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.objectmodel import ( import_from_mixin, instantiate, newlist_hint, resizelist_hint, specialize) +from rpython.rlib.rarithmetic import ovfcheck from rpython.rlib import longlong2float from rpython.tool.sourcetools import func_with_new_name @@ -872,7 +873,12 @@ """Extend w_list from a generic iterable""" length_hint = self.space.length_hint(w_iterable, 0) if length_hint: - w_list._resize_hint(w_list.length() + length_hint) + try: + newsize_hint = ovfcheck(w_list.length() + length_hint) + except OverflowError: + pass + else: + w_list._resize_hint(newsize_hint) extended = _do_extend_from_iterable(self.space, w_list, w_iterable) diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -619,6 +619,18 @@ assert l == [1.2, 2.3, 3.4, 4.5] assert l is l0 + def test_extend_iterable_length_hint_overflow(self): + import sys + class CustomIterable(object): + def __iter__(self): + if False: + yield + def __length_hint__(self): + return sys.maxsize + a = [1, 2, 3, 4] + a.extend(CustomIterable()) + assert a == [1, 2, 3, 4] + def test_sort(self): l = l0 = [1, 5, 3, 0] l.sort() diff --git a/rpython/annotator/signature.py b/rpython/annotator/signature.py --- a/rpython/annotator/signature.py +++ b/rpython/annotator/signature.py @@ -14,16 +14,16 @@ def _annotation_key(t): from rpython.rtyper import extregistry - if type(t) is list: + if isinstance(t, list): assert len(t) == 1 return ('list', _annotation_key(t[0])) - elif type(t) is dict: + elif isinstance(t, dict): assert len(t.keys()) == 1 return ('dict', _annotation_key(t.items()[0])) elif isinstance(t, tuple): return tuple([_annotation_key(i) for i in t]) elif extregistry.is_registered(t): - # XXX should it really be always different? + # XXX do we want to do something in this case? return t return t @@ -38,24 +38,36 @@ return t return _compute_annotation(t, bookkeeper) + +def _validate_annotation_size(t): + try: + _ = iter(t) + except TypeError: # if it's not an iterable, just return + return t # (size does not matter) + if isinstance(t, tuple): # we accept tuples with any length, because + return t # their in-memory representation is predictable + if len(t) > 1: + raise TypeError("Cannot specify multiple types in a %s (try using tuple)", type(t)) + + def _compute_annotation(t, bookkeeper=None): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llannotation import lltype_to_annotation + _validate_annotation_size(t) if isinstance(t, SomeObject): return t elif isinstance(t, lltype.LowLevelType): return lltype_to_annotation(t) elif isinstance(t, list): - assert len(t) == 1, "We do not support type joining in list" - listdef = ListDef(bookkeeper, annotation(t[0]), mutated=True, resized=True) - return SomeList(listdef) + return SomeList( + ListDef(bookkeeper, annotation(t[0]), + mutated=True, resized=True)) elif isinstance(t, tuple): return SomeTuple(tuple([annotation(i) for i in t])) elif isinstance(t, dict): - assert len(t) == 1, "We do not support type joining in dict" - result = SomeDict(DictDef(bookkeeper, annotation(t.keys()[0]), - annotation(t.values()[0]))) - return result + return SomeDict( + DictDef(bookkeeper, + annotation(t.keys()[0]), annotation(t.values()[0]))) elif type(t) is types.NoneType: return s_None elif extregistry.is_registered(t): @@ -84,15 +96,14 @@ elif t is types.NoneType: return s_None elif bookkeeper and extregistry.is_registered_type(t): - entry = extregistry.lookup_type(t) - return entry.compute_annotation_bk(bookkeeper) + return (extregistry.lookup_type(t) + .compute_annotation_bk(bookkeeper)) elif t is type: return SomeType() elif bookkeeper and not hasattr(t, '_freeze_'): - classdef = bookkeeper.getuniqueclassdef(t) - return SomeInstance(classdef) + return SomeInstance(bookkeeper.getuniqueclassdef(t)) else: - raise AssertionError("annotationoftype(%r)" % (t,)) + raise TypeError("Annotation of type %r not supported" % (t,)) class Sig(object): diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -259,6 +259,26 @@ intmask(). +Type Enforcing and Checking +--------------------------- + +RPython provides a helper decorator to force RPython-level types on function +arguments. The decorator, called ``enforceargs()``, accepts as parameters the +types expected to match the arguments of the function. + +Functions decorated with ``enforceargs()`` have their function signature +analyzed and their RPython-level type inferred at import time (for further +details about the flavor of translation performed in RPython, see the +`Annotation pass documentation`_). Encountering types not supported by RPython +will raise a ``TypeError``. + +``enforceargs()`` by default also performs type checking of parameter types +each time the function is invoked. To disable this behavior, it's possible to +pass the ``typecheck=False`` parameter to the decorator. + +.. _Annotation pass documentation: http://rpython.readthedocs.io/en/latest/translation.html#annotator + + Exception rules --------------- diff --git a/rpython/doc/translation.rst b/rpython/doc/translation.rst --- a/rpython/doc/translation.rst +++ b/rpython/doc/translation.rst From pypy.commits at gmail.com Tue Mar 27 05:46:45 2018 From: pypy.commits at gmail.com (antocuni) Date: Tue, 27 Mar 2018 02:46:45 -0700 (PDT) Subject: [pypy-commit] pypy memory-accounting: this branch has been merged long ago Message-ID: <5aba1305.1ca4df0a.85f65.48f4@mx.google.com> Author: Antonio Cuni Branch: memory-accounting Changeset: r94144:d53e1ebe51e1 Date: 2018-03-27 10:45 +0100 http://bitbucket.org/pypy/pypy/changeset/d53e1ebe51e1/ Log: this branch has been merged long ago From pypy.commits at gmail.com Tue Mar 27 06:50:19 2018 From: pypy.commits at gmail.com (Julian Berman) Date: Tue, 27 Mar 2018 03:50:19 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: Regenerating HTML Message-ID: <5aba21eb.b4abdf0a.6f475.6b30@mx.google.com> Author: Julian Berman Branch: extradoc Changeset: r918:913450bf91e9 Date: 2018-03-27 11:48 +0100 http://bitbucket.org/pypy/pypy.org/changeset/913450bf91e9/ Log: Regenerating HTML diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -132,9 +132,7 @@

    Python 3.5.3 compatible PyPy3.5 v5.10.1

    -

    Warning: PyPy3.5 is known to be rarely much slower than -PyPy 2. You are welcome to use it anyway

    -
      +
      • Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 16.04) (see [1] below)
      • Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04) (see [1] below)
      • ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian) (see [1] below)
        • From pypy.commits at gmail.com Tue Mar 27 06:50:16 2018 From: pypy.commits at gmail.com (Julian Berman) Date: Tue, 27 Mar 2018 03:50:16 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: This warning is apparently not too relevant anymore, at least in this spot. Message-ID: <5aba21e8.4fa1df0a.14a28.529a@mx.google.com> Author: Julian Berman Branch: extradoc Changeset: r917:e7d9800ec5ac Date: 2018-03-27 11:47 +0100 http://bitbucket.org/pypy/pypy.org/changeset/e7d9800ec5ac/ Log: This warning is apparently not too relevant anymore, at least in this spot. diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -120,9 +120,6 @@ .. class:: download_menu - Warning: PyPy3.5 is known to be rarely much slower than - PyPy 2. You are welcome to use it anyway - * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below) * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below) From pypy.commits at gmail.com Tue Mar 27 07:53:15 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 04:53:15 -0700 (PDT) Subject: [pypy-commit] pypy py3.5: add missing files for tests Message-ID: <5aba30ab.5b88df0a.be1c9.c085@mx.google.com> Author: Matti Picus Branch: py3.5 Changeset: r94145:fce0870ca033 Date: 2018-03-27 14:25 +0300 http://bitbucket.org/pypy/pypy/changeset/fce0870ca033/ Log: add missing files for tests diff too long, truncating to 2000 out of 4018 lines diff --git a/pypy/module/cpyext/test/_sre.c b/pypy/module/cpyext/test/_sre.c new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/test/_sre.c @@ -0,0 +1,3914 @@ +/* + * Secret Labs' Regular Expression Engine + * + * regular expression matching engine + * + * partial history: + * 1999-10-24 fl created (based on existing template matcher code) + * 2000-03-06 fl first alpha, sort of + * 2000-08-01 fl fixes for 1.6b1 + * 2000-08-07 fl use PyOS_CheckStack() if available + * 2000-09-20 fl added expand method + * 2001-03-20 fl lots of fixes for 2.1b2 + * 2001-04-15 fl export copyright as Python attribute, not global + * 2001-04-28 fl added __copy__ methods (work in progress) + * 2001-05-14 fl fixes for 1.5.2 compatibility + * 2001-07-01 fl added BIGCHARSET support (from Martin von Loewis) + * 2001-10-18 fl fixed group reset issue (from Matthew Mueller) + * 2001-10-20 fl added split primitive; reenable unicode for 1.6/2.0/2.1 + * 2001-10-21 fl added sub/subn primitive + * 2001-10-24 fl added finditer primitive (for 2.2 only) + * 2001-12-07 fl fixed memory leak in sub/subn (Guido van Rossum) + * 2002-11-09 fl fixed empty sub/subn return type + * 2003-04-18 mvl fully support 4-byte codes + * 2003-10-17 gn implemented non recursive scheme + * + * Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. + * + * This version of the SRE library can be redistributed under CNRI's + * Python 1.6 license. For any other use, please contact Secret Labs + * AB (info at pythonware.com). + * + * Portions of this engine have been developed in cooperation with + * CNRI. Hewlett-Packard provided funding for 1.6 integration and + * other compatibility work. + */ + +#ifndef SRE_RECURSIVE + +static char copyright[] = + " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" +#include "structmember.h" /* offsetof */ + +#include "sre.h" + +#include + +/* name of this module, minus the leading underscore */ +#if !defined(SRE_MODULE) +#define SRE_MODULE "sre" +#endif + +#define SRE_PY_MODULE "re" + +/* defining this one enables tracing */ +#undef VERBOSE + +#if PY_VERSION_HEX >= 0x01060000 +#if PY_VERSION_HEX < 0x02020000 || defined(Py_USING_UNICODE) +/* defining this enables unicode support (default under 1.6a1 and later) */ +#define HAVE_UNICODE +#endif +#endif + +/* -------------------------------------------------------------------- */ +/* optional features */ + +/* enables fast searching */ +#define USE_FAST_SEARCH + +/* enables aggressive inlining (always on for Visual C) */ +#undef USE_INLINE + +/* enables copy/deepcopy handling (work in progress) */ +#undef USE_BUILTIN_COPY + +#if PY_VERSION_HEX < 0x01060000 +#define PyObject_DEL(op) PyMem_DEL((op)) +#endif + +/* -------------------------------------------------------------------- */ + +#if defined(_MSC_VER) +#pragma optimize("agtw", on) /* doesn't seem to make much difference... */ +#pragma warning(disable: 4710) /* who cares if functions are not inlined ;-) */ +/* fastest possible local call under MSVC */ +#define LOCAL(type) static __inline type __fastcall +#elif defined(USE_INLINE) +#define LOCAL(type) static inline type +#else +#define LOCAL(type) static type +#endif + +/* error codes */ +#define SRE_ERROR_ILLEGAL -1 /* illegal opcode */ +#define SRE_ERROR_STATE -2 /* illegal state */ +#define SRE_ERROR_RECURSION_LIMIT -3 /* runaway recursion */ +#define SRE_ERROR_MEMORY -9 /* out of memory */ +#define SRE_ERROR_INTERRUPTED -10 /* signal handler raised exception */ + +#if defined(VERBOSE) +#define TRACE(v) printf v +#else +#define TRACE(v) +#endif + +/* -------------------------------------------------------------------- */ +/* search engine state */ + +/* default character predicates (run sre_chars.py to regenerate tables) */ + +#define SRE_DIGIT_MASK 1 +#define SRE_SPACE_MASK 2 +#define SRE_LINEBREAK_MASK 4 +#define SRE_ALNUM_MASK 8 +#define SRE_WORD_MASK 16 + +/* FIXME: this assumes ASCII. create tables in init_sre() instead */ + +static char sre_char_info[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 2, +2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, +0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 25, 25, 25, 25, 25, 25, 25, +25, 25, 0, 0, 0, 0, 0, 0, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, +0, 0, 16, 0, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 0, 0, 0, 0, 0 }; + +static char sre_char_lower[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, +10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, +27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, +44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, +61, 62, 63, 64, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, +108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, +122, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, +106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, +120, 121, 122, 123, 124, 125, 126, 127 }; + +#define SRE_IS_DIGIT(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_DIGIT_MASK) : 0) +#define SRE_IS_SPACE(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_SPACE_MASK) : 0) +#define SRE_IS_LINEBREAK(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_LINEBREAK_MASK) : 0) +#define SRE_IS_ALNUM(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_ALNUM_MASK) : 0) +#define SRE_IS_WORD(ch)\ + ((ch) < 128 ? (sre_char_info[(ch)] & SRE_WORD_MASK) : 0) + +static unsigned int sre_lower(unsigned int ch) +{ + return ((ch) < 128 ? (unsigned int)sre_char_lower[ch] : ch); +} + +/* locale-specific character predicates */ +/* !(c & ~N) == (c < N+1) for any unsigned c, this avoids + * warnings when c's type supports only numbers < N+1 */ +#define SRE_LOC_IS_DIGIT(ch) (!((ch) & ~255) ? isdigit((ch)) : 0) +#define SRE_LOC_IS_SPACE(ch) (!((ch) & ~255) ? isspace((ch)) : 0) +#define SRE_LOC_IS_LINEBREAK(ch) ((ch) == '\n') +#define SRE_LOC_IS_ALNUM(ch) (!((ch) & ~255) ? isalnum((ch)) : 0) +#define SRE_LOC_IS_WORD(ch) (SRE_LOC_IS_ALNUM((ch)) || (ch) == '_') + +static unsigned int sre_lower_locale(unsigned int ch) +{ + return ((ch) < 256 ? (unsigned int)tolower((ch)) : ch); +} + +/* unicode-specific character predicates */ + +#if defined(HAVE_UNICODE) + +#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL((Py_UNICODE)(ch)) +#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch)) +#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch)) +#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM((Py_UNICODE)(ch)) +#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM((ch)) || (ch) == '_') + +static unsigned int sre_lower_unicode(unsigned int ch) +{ + return (unsigned int) Py_UNICODE_TOLOWER((Py_UNICODE)(ch)); +} + +#endif + +LOCAL(int) +sre_category(SRE_CODE category, unsigned int ch) +{ + switch (category) { + + case SRE_CATEGORY_DIGIT: + return SRE_IS_DIGIT(ch); + case SRE_CATEGORY_NOT_DIGIT: + return !SRE_IS_DIGIT(ch); + case SRE_CATEGORY_SPACE: + return SRE_IS_SPACE(ch); + case SRE_CATEGORY_NOT_SPACE: + return !SRE_IS_SPACE(ch); + case SRE_CATEGORY_WORD: + return SRE_IS_WORD(ch); + case SRE_CATEGORY_NOT_WORD: + return !SRE_IS_WORD(ch); + case SRE_CATEGORY_LINEBREAK: + return SRE_IS_LINEBREAK(ch); + case SRE_CATEGORY_NOT_LINEBREAK: + return !SRE_IS_LINEBREAK(ch); + + case SRE_CATEGORY_LOC_WORD: + return SRE_LOC_IS_WORD(ch); + case SRE_CATEGORY_LOC_NOT_WORD: + return !SRE_LOC_IS_WORD(ch); + +#if defined(HAVE_UNICODE) + case SRE_CATEGORY_UNI_DIGIT: + return SRE_UNI_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_NOT_DIGIT: + return !SRE_UNI_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_SPACE: + return SRE_UNI_IS_SPACE(ch); + case SRE_CATEGORY_UNI_NOT_SPACE: + return !SRE_UNI_IS_SPACE(ch); + case SRE_CATEGORY_UNI_WORD: + return SRE_UNI_IS_WORD(ch); + case SRE_CATEGORY_UNI_NOT_WORD: + return !SRE_UNI_IS_WORD(ch); + case SRE_CATEGORY_UNI_LINEBREAK: + return SRE_UNI_IS_LINEBREAK(ch); + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + return !SRE_UNI_IS_LINEBREAK(ch); +#else + case SRE_CATEGORY_UNI_DIGIT: + return SRE_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_NOT_DIGIT: + return !SRE_IS_DIGIT(ch); + case SRE_CATEGORY_UNI_SPACE: + return SRE_IS_SPACE(ch); + case SRE_CATEGORY_UNI_NOT_SPACE: + return !SRE_IS_SPACE(ch); + case SRE_CATEGORY_UNI_WORD: + return SRE_LOC_IS_WORD(ch); + case SRE_CATEGORY_UNI_NOT_WORD: + return !SRE_LOC_IS_WORD(ch); + case SRE_CATEGORY_UNI_LINEBREAK: + return SRE_IS_LINEBREAK(ch); + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + return !SRE_IS_LINEBREAK(ch); +#endif + } + return 0; +} + +/* helpers */ + +static void +data_stack_dealloc(SRE_STATE* state) +{ + if (state->data_stack) { + PyMem_FREE(state->data_stack); + state->data_stack = NULL; + } + state->data_stack_size = state->data_stack_base = 0; +} + +static int +data_stack_grow(SRE_STATE* state, Py_ssize_t size) +{ + Py_ssize_t minsize, cursize; + minsize = state->data_stack_base+size; + cursize = state->data_stack_size; + if (cursize < minsize) { + void* stack; + cursize = minsize+minsize/4+1024; + TRACE(("allocate/grow stack %d\n", cursize)); + stack = PyMem_REALLOC(state->data_stack, cursize); + if (!stack) { + data_stack_dealloc(state); + return SRE_ERROR_MEMORY; + } + state->data_stack = (char *)stack; + state->data_stack_size = cursize; + } + return 0; +} + +/* generate 8-bit version */ + +#define SRE_CHAR unsigned char +#define SRE_AT sre_at +#define SRE_COUNT sre_count +#define SRE_CHARSET sre_charset +#define SRE_INFO sre_info +#define SRE_MATCH sre_match +#define SRE_MATCH_CONTEXT sre_match_context +#define SRE_SEARCH sre_search +#define SRE_LITERAL_TEMPLATE sre_literal_template + +#if defined(HAVE_UNICODE) + +#define SRE_RECURSIVE +#include "_sre.c" +#undef SRE_RECURSIVE + +#undef SRE_LITERAL_TEMPLATE +#undef SRE_SEARCH +#undef SRE_MATCH +#undef SRE_MATCH_CONTEXT +#undef SRE_INFO +#undef SRE_CHARSET +#undef SRE_COUNT +#undef SRE_AT +#undef SRE_CHAR + +/* generate 16-bit unicode version */ + +#define SRE_CHAR Py_UNICODE +#define SRE_AT sre_uat +#define SRE_COUNT sre_ucount +#define SRE_CHARSET sre_ucharset +#define SRE_INFO sre_uinfo +#define SRE_MATCH sre_umatch +#define SRE_MATCH_CONTEXT sre_umatch_context +#define SRE_SEARCH sre_usearch +#define SRE_LITERAL_TEMPLATE sre_uliteral_template +#endif + +#endif /* SRE_RECURSIVE */ + +/* -------------------------------------------------------------------- */ +/* String matching engine */ + +/* the following section is compiled twice, with different character + settings */ + +LOCAL(int) +SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) +{ + /* check if pointer is at given position */ + + Py_ssize_t thisp, thatp; + + switch (at) { + + case SRE_AT_BEGINNING: + case SRE_AT_BEGINNING_STRING: + return ((void*) ptr == state->beginning); + + case SRE_AT_BEGINNING_LINE: + return ((void*) ptr == state->beginning || + SRE_IS_LINEBREAK((int) ptr[-1])); + + case SRE_AT_END: + return (((void*) (ptr+1) == state->end && + SRE_IS_LINEBREAK((int) ptr[0])) || + ((void*) ptr == state->end)); + + case SRE_AT_END_LINE: + return ((void*) ptr == state->end || + SRE_IS_LINEBREAK((int) ptr[0])); + + case SRE_AT_END_STRING: + return ((void*) ptr == state->end); + + case SRE_AT_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_IS_WORD((int) ptr[0]) : 0; + return thisp != thatp; + + case SRE_AT_NON_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_IS_WORD((int) ptr[0]) : 0; + return thisp == thatp; + + case SRE_AT_LOC_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_LOC_IS_WORD((int) ptr[0]) : 0; + return thisp != thatp; + + case SRE_AT_LOC_NON_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_LOC_IS_WORD((int) ptr[0]) : 0; + return thisp == thatp; + +#if defined(HAVE_UNICODE) + case SRE_AT_UNI_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_UNI_IS_WORD((int) ptr[0]) : 0; + return thisp != thatp; + + case SRE_AT_UNI_NON_BOUNDARY: + if (state->beginning == state->end) + return 0; + thatp = ((void*) ptr > state->beginning) ? + SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + thisp = ((void*) ptr < state->end) ? + SRE_UNI_IS_WORD((int) ptr[0]) : 0; + return thisp == thatp; +#endif + + } + + return 0; +} + +LOCAL(int) +SRE_CHARSET(SRE_CODE* set, SRE_CODE ch) +{ + /* check if character is a member of the given set */ + + int ok = 1; + + for (;;) { + switch (*set++) { + + case SRE_OP_FAILURE: + return !ok; + + case SRE_OP_LITERAL: + /* */ + if (ch == set[0]) + return ok; + set++; + break; + + case SRE_OP_CATEGORY: + /* */ + if (sre_category(set[0], (int) ch)) + return ok; + set += 1; + break; + + case SRE_OP_CHARSET: + if (sizeof(SRE_CODE) == 2) { + /* (16 bits per code word) */ + if (ch < 256 && (set[ch >> 4] & (1 << (ch & 15)))) + return ok; + set += 16; + } + else { + /* (32 bits per code word) */ + if (ch < 256 && (set[ch >> 5] & (1 << (ch & 31)))) + return ok; + set += 8; + } + break; + + case SRE_OP_RANGE: + /* */ + if (set[0] <= ch && ch <= set[1]) + return ok; + set += 2; + break; + + case SRE_OP_NEGATE: + ok = !ok; + break; + + case SRE_OP_BIGCHARSET: + /* <256 blockindices> */ + { + Py_ssize_t count, block; + count = *(set++); + + if (sizeof(SRE_CODE) == 2) { + block = ((unsigned char*)set)[ch >> 8]; + set += 128; + if (set[block*16 + ((ch & 255)>>4)] & (1 << (ch & 15))) + return ok; + set += count*16; + } + else { + /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids + * warnings when c's type supports only numbers < N+1 */ + if (!(ch & ~65535)) + block = ((unsigned char*)set)[ch >> 8]; + else + block = -1; + set += 64; + if (block >=0 && + (set[block*8 + ((ch & 255)>>5)] & (1 << (ch & 31)))) + return ok; + set += count*8; + } + break; + } + + default: + /* internal error -- there's not much we can do about it + here, so let's just pretend it didn't match... */ + return 0; + } + } +} + +LOCAL(Py_ssize_t) SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern); + +LOCAL(Py_ssize_t) +SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) +{ + SRE_CODE chr; + SRE_CHAR* ptr = (SRE_CHAR *)state->ptr; + SRE_CHAR* end = (SRE_CHAR *)state->end; + Py_ssize_t i; + + /* adjust end */ + if (maxcount < end - ptr && maxcount != 65535) + end = ptr + maxcount; + + switch (pattern[0]) { + + case SRE_OP_IN: + /* repeated set */ + TRACE(("|%p|%p|COUNT IN\n", pattern, ptr)); + while (ptr < end && SRE_CHARSET(pattern + 2, *ptr)) + ptr++; + break; + + case SRE_OP_ANY: + /* repeated dot wildcard. */ + TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr)); + while (ptr < end && !SRE_IS_LINEBREAK(*ptr)) + ptr++; + break; + + case SRE_OP_ANY_ALL: + /* repeated dot wildcard. skip to the end of the target + string, and backtrack from there */ + TRACE(("|%p|%p|COUNT ANY_ALL\n", pattern, ptr)); + ptr = end; + break; + + case SRE_OP_LITERAL: + /* repeated literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) *ptr == chr) + ptr++; + break; + + case SRE_OP_LITERAL_IGNORE: + /* repeated literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr) + ptr++; + break; + + case SRE_OP_NOT_LITERAL: + /* repeated non-literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) *ptr != chr) + ptr++; + break; + + case SRE_OP_NOT_LITERAL_IGNORE: + /* repeated non-literal */ + chr = pattern[1]; + TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr)); + while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr) + ptr++; + break; + + default: + /* repeated single character pattern */ + TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr)); + while ((SRE_CHAR*) state->ptr < end) { + i = SRE_MATCH(state, pattern); + if (i < 0) + return i; + if (!i) + break; + } + TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + (SRE_CHAR*) state->ptr - ptr)); + return (SRE_CHAR*) state->ptr - ptr; + } + + TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); + return ptr - (SRE_CHAR*) state->ptr; +} + +#if 0 /* not used in this release */ +LOCAL(int) +SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) +{ + /* check if an SRE_OP_INFO block matches at the current position. + returns the number of SRE_CODE objects to skip if successful, 0 + if no match */ + + SRE_CHAR* end = state->end; + SRE_CHAR* ptr = state->ptr; + Py_ssize_t i; + + /* check minimal length */ + if (pattern[3] && (end - ptr) < pattern[3]) + return 0; + + /* check known prefix */ + if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) { + /* */ + for (i = 0; i < pattern[5]; i++) + if ((SRE_CODE) ptr[i] != pattern[7 + i]) + return 0; + return pattern[0] + 2 * pattern[6]; + } + return pattern[0]; +} +#endif + +/* The macros below should be used to protect recursive SRE_MATCH() + * calls that *failed* and do *not* return immediately (IOW, those + * that will backtrack). Explaining: + * + * - Recursive SRE_MATCH() returned true: that's usually a success + * (besides atypical cases like ASSERT_NOT), therefore there's no + * reason to restore lastmark; + * + * - Recursive SRE_MATCH() returned false but the current SRE_MATCH() + * is returning to the caller: If the current SRE_MATCH() is the + * top function of the recursion, returning false will be a matching + * failure, and it doesn't matter where lastmark is pointing to. + * If it's *not* the top function, it will be a recursive SRE_MATCH() + * failure by itself, and the calling SRE_MATCH() will have to deal + * with the failure by the same rules explained here (it will restore + * lastmark by itself if necessary); + * + * - Recursive SRE_MATCH() returned false, and will continue the + * outside 'for' loop: must be protected when breaking, since the next + * OP could potentially depend on lastmark; + * + * - Recursive SRE_MATCH() returned false, and will be called again + * inside a local for/while loop: must be protected between each + * loop iteration, since the recursive SRE_MATCH() could do anything, + * and could potentially depend on lastmark. + * + * For more information, check the discussion at SF patch #712900. + */ +#define LASTMARK_SAVE() \ + do { \ + ctx->lastmark = state->lastmark; \ + ctx->lastindex = state->lastindex; \ + } while (0) +#define LASTMARK_RESTORE() \ + do { \ + state->lastmark = ctx->lastmark; \ + state->lastindex = ctx->lastindex; \ + } while (0) + +#define RETURN_ERROR(i) do { return i; } while(0) +#define RETURN_FAILURE do { ret = 0; goto exit; } while(0) +#define RETURN_SUCCESS do { ret = 1; goto exit; } while(0) + +#define RETURN_ON_ERROR(i) \ + do { if (i < 0) RETURN_ERROR(i); } while (0) +#define RETURN_ON_SUCCESS(i) \ + do { RETURN_ON_ERROR(i); if (i > 0) RETURN_SUCCESS; } while (0) +#define RETURN_ON_FAILURE(i) \ + do { RETURN_ON_ERROR(i); if (i == 0) RETURN_FAILURE; } while (0) + +#define SFY(x) #x + +#define DATA_STACK_ALLOC(state, type, ptr) \ +do { \ + alloc_pos = state->data_stack_base; \ + TRACE(("allocating %s in %d (%d)\n", \ + SFY(type), alloc_pos, sizeof(type))); \ + if (state->data_stack_size < alloc_pos+sizeof(type)) { \ + int j = data_stack_grow(state, sizeof(type)); \ + if (j < 0) return j; \ + if (ctx_pos != -1) \ + DATA_STACK_LOOKUP_AT(state, SRE_MATCH_CONTEXT, ctx, ctx_pos); \ + } \ + ptr = (type*)(state->data_stack+alloc_pos); \ + state->data_stack_base += sizeof(type); \ +} while (0) + +#define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ +do { \ + TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + ptr = (type*)(state->data_stack+pos); \ +} while (0) + +#define DATA_STACK_PUSH(state, data, size) \ +do { \ + TRACE(("copy data in %p to %d (%d)\n", \ + data, state->data_stack_base, size)); \ + if (state->data_stack_size < state->data_stack_base+size) { \ + int j = data_stack_grow(state, size); \ + if (j < 0) return j; \ + if (ctx_pos != -1) \ + DATA_STACK_LOOKUP_AT(state, SRE_MATCH_CONTEXT, ctx, ctx_pos); \ + } \ + memcpy(state->data_stack+state->data_stack_base, data, size); \ + state->data_stack_base += size; \ +} while (0) + +#define DATA_STACK_POP(state, data, size, discard) \ +do { \ + TRACE(("copy data to %p from %d (%d)\n", \ + data, state->data_stack_base-size, size)); \ + memcpy(data, state->data_stack+state->data_stack_base-size, size); \ + if (discard) \ + state->data_stack_base -= size; \ +} while (0) + +#define DATA_STACK_POP_DISCARD(state, size) \ +do { \ + TRACE(("discard data from %d (%d)\n", \ + state->data_stack_base-size, size)); \ + state->data_stack_base -= size; \ +} while(0) + +#define DATA_PUSH(x) \ + DATA_STACK_PUSH(state, (x), sizeof(*(x))) +#define DATA_POP(x) \ + DATA_STACK_POP(state, (x), sizeof(*(x)), 1) +#define DATA_POP_DISCARD(x) \ + DATA_STACK_POP_DISCARD(state, sizeof(*(x))) +#define DATA_ALLOC(t,p) \ + DATA_STACK_ALLOC(state, t, p) +#define DATA_LOOKUP_AT(t,p,pos) \ + DATA_STACK_LOOKUP_AT(state,t,p,pos) + +#define MARK_PUSH(lastmark) \ + do if (lastmark > 0) { \ + i = lastmark; /* ctx->lastmark may change if reallocated */ \ + DATA_STACK_PUSH(state, state->mark, (i+1)*sizeof(void*)); \ + } while (0) +#define MARK_POP(lastmark) \ + do if (lastmark > 0) { \ + DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 1); \ + } while (0) +#define MARK_POP_KEEP(lastmark) \ + do if (lastmark > 0) { \ + DATA_STACK_POP(state, state->mark, (lastmark+1)*sizeof(void*), 0); \ + } while (0) +#define MARK_POP_DISCARD(lastmark) \ + do if (lastmark > 0) { \ + DATA_STACK_POP_DISCARD(state, (lastmark+1)*sizeof(void*)); \ + } while (0) + +#define JUMP_NONE 0 +#define JUMP_MAX_UNTIL_1 1 +#define JUMP_MAX_UNTIL_2 2 +#define JUMP_MAX_UNTIL_3 3 +#define JUMP_MIN_UNTIL_1 4 +#define JUMP_MIN_UNTIL_2 5 +#define JUMP_MIN_UNTIL_3 6 +#define JUMP_REPEAT 7 +#define JUMP_REPEAT_ONE_1 8 +#define JUMP_REPEAT_ONE_2 9 +#define JUMP_MIN_REPEAT_ONE 10 +#define JUMP_BRANCH 11 +#define JUMP_ASSERT 12 +#define JUMP_ASSERT_NOT 13 + +#define DO_JUMP(jumpvalue, jumplabel, nextpattern) \ + DATA_ALLOC(SRE_MATCH_CONTEXT, nextctx); \ + nextctx->last_ctx_pos = ctx_pos; \ + nextctx->jump = jumpvalue; \ + nextctx->pattern = nextpattern; \ + ctx_pos = alloc_pos; \ + ctx = nextctx; \ + goto entrance; \ + jumplabel: \ + while (0) /* gcc doesn't like labels at end of scopes */ \ + +typedef struct { + Py_ssize_t last_ctx_pos; + Py_ssize_t jump; + SRE_CHAR* ptr; + SRE_CODE* pattern; + Py_ssize_t count; + Py_ssize_t lastmark; + Py_ssize_t lastindex; + union { + SRE_CODE chr; + SRE_REPEAT* rep; + } u; +} SRE_MATCH_CONTEXT; + +/* check if string matches the given pattern. returns <0 for + error, 0 for failure, and 1 for success */ +LOCAL(Py_ssize_t) +SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) +{ + SRE_CHAR* end = (SRE_CHAR *)state->end; + Py_ssize_t alloc_pos, ctx_pos = -1; + Py_ssize_t i, ret = 0; + Py_ssize_t jump; + unsigned int sigcount=0; + + SRE_MATCH_CONTEXT* ctx; + SRE_MATCH_CONTEXT* nextctx; + + TRACE(("|%p|%p|ENTER\n", pattern, state->ptr)); + + DATA_ALLOC(SRE_MATCH_CONTEXT, ctx); + ctx->last_ctx_pos = -1; + ctx->jump = JUMP_NONE; + ctx->pattern = pattern; + ctx_pos = alloc_pos; + +entrance: + + ctx->ptr = (SRE_CHAR *)state->ptr; + + if (ctx->pattern[0] == SRE_OP_INFO) { + /* optimization info block */ + /* <1=skip> <2=flags> <3=min> ... */ + if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { + TRACE(("reject (got %d chars, need %d)\n", + (end - ctx->ptr), ctx->pattern[3])); + RETURN_FAILURE; + } + ctx->pattern += ctx->pattern[1] + 1; + } + + for (;;) { + ++sigcount; + if ((0 == (sigcount & 0xfff)) && PyErr_CheckSignals()) + RETURN_ERROR(SRE_ERROR_INTERRUPTED); + + switch (*ctx->pattern++) { + + case SRE_OP_MARK: + /* set mark */ + /* */ + TRACE(("|%p|%p|MARK %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + i = ctx->pattern[0]; + if (i & 1) + state->lastindex = i/2 + 1; + if (i > state->lastmark) { + /* state->lastmark is the highest valid index in the + state->mark array. If it is increased by more than 1, + the intervening marks must be set to NULL to signal + that these marks have not been encountered. */ + Py_ssize_t j = state->lastmark + 1; + while (j < i) + state->mark[j++] = NULL; + state->lastmark = i; + } + state->mark[i] = ctx->ptr; + ctx->pattern++; + break; + + case SRE_OP_LITERAL: + /* match literal string */ + /* */ + TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern, + ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0]) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_NOT_LITERAL: + /* match anything that is not literal character */ + /* */ + TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern, + ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0]) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_SUCCESS: + /* end of pattern */ + TRACE(("|%p|%p|SUCCESS\n", ctx->pattern, ctx->ptr)); + state->ptr = ctx->ptr; + RETURN_SUCCESS; + + case SRE_OP_AT: + /* match at given position */ + /* */ + TRACE(("|%p|%p|AT %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); + if (!SRE_AT(state, ctx->ptr, *ctx->pattern)) + RETURN_FAILURE; + ctx->pattern++; + break; + + case SRE_OP_CATEGORY: + /* match at given category */ + /* */ + TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern, + ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0])) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_ANY: + /* match anything (except a newline) */ + /* */ + TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0])) + RETURN_FAILURE; + ctx->ptr++; + break; + + case SRE_OP_ANY_ALL: + /* match anything */ + /* */ + TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end) + RETURN_FAILURE; + ctx->ptr++; + break; + + case SRE_OP_IN: + /* match set member (or non_member) */ + /* */ + TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr)) + RETURN_FAILURE; + ctx->pattern += ctx->pattern[0]; + ctx->ptr++; + break; + + case SRE_OP_LITERAL_IGNORE: + TRACE(("|%p|%p|LITERAL_IGNORE %d\n", + ctx->pattern, ctx->ptr, ctx->pattern[0])); + if (ctx->ptr >= end || + state->lower(*ctx->ptr) != state->lower(*ctx->pattern)) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_NOT_LITERAL_IGNORE: + TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", + ctx->pattern, ctx->ptr, *ctx->pattern)); + if (ctx->ptr >= end || + state->lower(*ctx->ptr) == state->lower(*ctx->pattern)) + RETURN_FAILURE; + ctx->pattern++; + ctx->ptr++; + break; + + case SRE_OP_IN_IGNORE: + TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr)); + if (ctx->ptr >= end + || !SRE_CHARSET(ctx->pattern+1, + (SRE_CODE)state->lower(*ctx->ptr))) + RETURN_FAILURE; + ctx->pattern += ctx->pattern[0]; + ctx->ptr++; + break; + + case SRE_OP_JUMP: + case SRE_OP_INFO: + /* jump forward */ + /* */ + TRACE(("|%p|%p|JUMP %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + ctx->pattern += ctx->pattern[0]; + break; + + case SRE_OP_BRANCH: + /* alternation */ + /* <0=skip> code ... */ + TRACE(("|%p|%p|BRANCH\n", ctx->pattern, ctx->ptr)); + LASTMARK_SAVE(); + ctx->u.rep = state->repeat; + if (ctx->u.rep) + MARK_PUSH(ctx->lastmark); + for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { + if (ctx->pattern[1] == SRE_OP_LITERAL && + (ctx->ptr >= end || + (SRE_CODE) *ctx->ptr != ctx->pattern[2])) + continue; + if (ctx->pattern[1] == SRE_OP_IN && + (ctx->ptr >= end || + !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr))) + continue; + state->ptr = ctx->ptr; + DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); + if (ret) { + if (ctx->u.rep) + MARK_POP_DISCARD(ctx->lastmark); + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + if (ctx->u.rep) + MARK_POP_KEEP(ctx->lastmark); + LASTMARK_RESTORE(); + } + if (ctx->u.rep) + MARK_POP_DISCARD(ctx->lastmark); + RETURN_FAILURE; + + case SRE_OP_REPEAT_ONE: + /* match repeated sequence (maximizing regexp) */ + + /* this operator only works if the repeated item is + exactly one character wide, and we're not already + collecting backtracking points. for other cases, + use the MAX_REPEAT operator */ + + /* <1=min> <2=max> item tail */ + + TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2])); + + if (ctx->ptr + ctx->pattern[1] > end) + RETURN_FAILURE; /* cannot match */ + + state->ptr = ctx->ptr; + + ret = SRE_COUNT(state, ctx->pattern+3, ctx->pattern[2]); + RETURN_ON_ERROR(ret); + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + ctx->count = ret; + ctx->ptr += ctx->count; + + /* when we arrive here, count contains the number of + matches, and ctx->ptr points to the tail of the target + string. check if the rest of the pattern matches, + and backtrack if not. */ + + if (ctx->count < (Py_ssize_t) ctx->pattern[1]) + RETURN_FAILURE; + + if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { + /* tail is empty. we're finished */ + state->ptr = ctx->ptr; + RETURN_SUCCESS; + } + + LASTMARK_SAVE(); + + if (ctx->pattern[ctx->pattern[0]] == SRE_OP_LITERAL) { + /* tail starts with a literal. skip positions where + the rest of the pattern cannot possibly match */ + ctx->u.chr = ctx->pattern[ctx->pattern[0]+1]; + for (;;) { + while (ctx->count >= (Py_ssize_t) ctx->pattern[1] && + (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) { + ctx->ptr--; + ctx->count--; + } + if (ctx->count < (Py_ssize_t) ctx->pattern[1]) + break; + state->ptr = ctx->ptr; + DO_JUMP(JUMP_REPEAT_ONE_1, jump_repeat_one_1, + ctx->pattern+ctx->pattern[0]); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + + LASTMARK_RESTORE(); + + ctx->ptr--; + ctx->count--; + } + + } else { + /* general case */ + while (ctx->count >= (Py_ssize_t) ctx->pattern[1]) { + state->ptr = ctx->ptr; + DO_JUMP(JUMP_REPEAT_ONE_2, jump_repeat_one_2, + ctx->pattern+ctx->pattern[0]); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->ptr--; + ctx->count--; + LASTMARK_RESTORE(); + } + } + RETURN_FAILURE; + + case SRE_OP_MIN_REPEAT_ONE: + /* match repeated sequence (minimizing regexp) */ + + /* this operator only works if the repeated item is + exactly one character wide, and we're not already + collecting backtracking points. for other cases, + use the MIN_REPEAT operator */ + + /* <1=min> <2=max> item tail */ + + TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2])); + + if (ctx->ptr + ctx->pattern[1] > end) + RETURN_FAILURE; /* cannot match */ + + state->ptr = ctx->ptr; + + if (ctx->pattern[1] == 0) + ctx->count = 0; + else { + /* count using pattern min as the maximum */ + ret = SRE_COUNT(state, ctx->pattern+3, ctx->pattern[1]); + RETURN_ON_ERROR(ret); + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + if (ret < (Py_ssize_t) ctx->pattern[1]) + /* didn't match minimum number of times */ + RETURN_FAILURE; + /* advance past minimum matches of repeat */ + ctx->count = ret; + ctx->ptr += ctx->count; + } + + if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { + /* tail is empty. we're finished */ + state->ptr = ctx->ptr; + RETURN_SUCCESS; + + } else { + /* general case */ + LASTMARK_SAVE(); + while ((Py_ssize_t)ctx->pattern[2] == 65535 + || ctx->count <= (Py_ssize_t)ctx->pattern[2]) { + state->ptr = ctx->ptr; + DO_JUMP(JUMP_MIN_REPEAT_ONE,jump_min_repeat_one, + ctx->pattern+ctx->pattern[0]); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + state->ptr = ctx->ptr; + ret = SRE_COUNT(state, ctx->pattern+3, 1); + RETURN_ON_ERROR(ret); + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + if (ret == 0) + break; + assert(ret == 1); + ctx->ptr++; + ctx->count++; + LASTMARK_RESTORE(); + } + } + RETURN_FAILURE; + + case SRE_OP_REPEAT: + /* create repeat context. all the hard work is done + by the UNTIL operator (MAX_UNTIL, MIN_UNTIL) */ + /* <1=min> <2=max> item tail */ + TRACE(("|%p|%p|REPEAT %d %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[1], ctx->pattern[2])); + + /* install new repeat context */ + ctx->u.rep = (SRE_REPEAT*) PyObject_MALLOC(sizeof(*ctx->u.rep)); + if (!ctx->u.rep) { + PyErr_NoMemory(); + RETURN_FAILURE; + } + ctx->u.rep->count = -1; + ctx->u.rep->pattern = ctx->pattern; + ctx->u.rep->prev = state->repeat; + ctx->u.rep->last_ptr = NULL; + state->repeat = ctx->u.rep; + + state->ptr = ctx->ptr; + DO_JUMP(JUMP_REPEAT, jump_repeat, ctx->pattern+ctx->pattern[0]); + state->repeat = ctx->u.rep->prev; + PyObject_FREE(ctx->u.rep); + + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + RETURN_FAILURE; + + case SRE_OP_MAX_UNTIL: + /* maximizing repeat */ + /* <1=min> <2=max> item tail */ + + /* FIXME: we probably need to deal with zero-width + matches in here... */ + + ctx->u.rep = state->repeat; + if (!ctx->u.rep) + RETURN_ERROR(SRE_ERROR_STATE); + + state->ptr = ctx->ptr; + + ctx->count = ctx->u.rep->count+1; + + TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + ctx->ptr, ctx->count)); + + if (ctx->count < ctx->u.rep->pattern[1]) { + /* not enough matches */ + ctx->u.rep->count = ctx->count; + DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, + ctx->u.rep->pattern+3); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + RETURN_FAILURE; + } + + if ((ctx->count < ctx->u.rep->pattern[2] || + ctx->u.rep->pattern[2] == 65535) && + state->ptr != ctx->u.rep->last_ptr) { + /* we may have enough matches, but if we can + match another item, do so */ + ctx->u.rep->count = ctx->count; + LASTMARK_SAVE(); + MARK_PUSH(ctx->lastmark); + /* zero-width match protection */ + DATA_PUSH(&ctx->u.rep->last_ptr); + ctx->u.rep->last_ptr = state->ptr; + DO_JUMP(JUMP_MAX_UNTIL_2, jump_max_until_2, + ctx->u.rep->pattern+3); + DATA_POP(&ctx->u.rep->last_ptr); + if (ret) { + MARK_POP_DISCARD(ctx->lastmark); + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + MARK_POP(ctx->lastmark); + LASTMARK_RESTORE(); + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + } + + /* cannot match more repeated items here. make sure the + tail matches */ + state->repeat = ctx->u.rep->prev; + DO_JUMP(JUMP_MAX_UNTIL_3, jump_max_until_3, ctx->pattern); + RETURN_ON_SUCCESS(ret); + state->repeat = ctx->u.rep; + state->ptr = ctx->ptr; + RETURN_FAILURE; + + case SRE_OP_MIN_UNTIL: + /* minimizing repeat */ + /* <1=min> <2=max> item tail */ + + ctx->u.rep = state->repeat; + if (!ctx->u.rep) + RETURN_ERROR(SRE_ERROR_STATE); + + state->ptr = ctx->ptr; + + ctx->count = ctx->u.rep->count+1; + + TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + ctx->ptr, ctx->count, ctx->u.rep->pattern)); + + if (ctx->count < ctx->u.rep->pattern[1]) { + /* not enough matches */ + ctx->u.rep->count = ctx->count; + DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, + ctx->u.rep->pattern+3); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + RETURN_FAILURE; + } + + LASTMARK_SAVE(); + + /* see if the tail matches */ + state->repeat = ctx->u.rep->prev; + DO_JUMP(JUMP_MIN_UNTIL_2, jump_min_until_2, ctx->pattern); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + + state->repeat = ctx->u.rep; + state->ptr = ctx->ptr; + + LASTMARK_RESTORE(); + + if (ctx->count >= ctx->u.rep->pattern[2] + && ctx->u.rep->pattern[2] != 65535) + RETURN_FAILURE; + + ctx->u.rep->count = ctx->count; + DO_JUMP(JUMP_MIN_UNTIL_3,jump_min_until_3, + ctx->u.rep->pattern+3); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_SUCCESS; + } + ctx->u.rep->count = ctx->count-1; + state->ptr = ctx->ptr; + RETURN_FAILURE; + + case SRE_OP_GROUPREF: + /* match backreference */ + TRACE(("|%p|%p|GROUPREF %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + i = ctx->pattern[0]; + { + Py_ssize_t groupref = i+i; + if (groupref >= state->lastmark) { + RETURN_FAILURE; + } else { + SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; + SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + if (!p || !e || e < p) + RETURN_FAILURE; + while (p < e) { + if (ctx->ptr >= end || *ctx->ptr != *p) + RETURN_FAILURE; + p++; ctx->ptr++; + } + } + } + ctx->pattern++; + break; + + case SRE_OP_GROUPREF_IGNORE: + /* match backreference */ + TRACE(("|%p|%p|GROUPREF_IGNORE %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + i = ctx->pattern[0]; + { + Py_ssize_t groupref = i+i; + if (groupref >= state->lastmark) { + RETURN_FAILURE; + } else { + SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; + SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + if (!p || !e || e < p) + RETURN_FAILURE; + while (p < e) { + if (ctx->ptr >= end || + state->lower(*ctx->ptr) != state->lower(*p)) + RETURN_FAILURE; + p++; ctx->ptr++; + } + } + } + ctx->pattern++; + break; + + case SRE_OP_GROUPREF_EXISTS: + TRACE(("|%p|%p|GROUPREF_EXISTS %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[0])); + /* codeyes codeno ... */ + i = ctx->pattern[0]; + { + Py_ssize_t groupref = i+i; + if (groupref >= state->lastmark) { + ctx->pattern += ctx->pattern[1]; + break; + } else { + SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; + SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + if (!p || !e || e < p) { + ctx->pattern += ctx->pattern[1]; + break; + } + } + } + ctx->pattern += 2; + break; + + case SRE_OP_ASSERT: + /* assert subpattern */ + /* */ + TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[1])); + state->ptr = ctx->ptr - ctx->pattern[1]; + if (state->ptr < state->beginning) + RETURN_FAILURE; + DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2); + RETURN_ON_FAILURE(ret); + ctx->pattern += ctx->pattern[0]; + break; + + case SRE_OP_ASSERT_NOT: + /* assert not subpattern */ + /* */ + TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern, + ctx->ptr, ctx->pattern[1])); + state->ptr = ctx->ptr - ctx->pattern[1]; + if (state->ptr >= state->beginning) { + DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); + if (ret) { + RETURN_ON_ERROR(ret); + RETURN_FAILURE; + } + } + ctx->pattern += ctx->pattern[0]; + break; + + case SRE_OP_FAILURE: + /* immediate failure */ + TRACE(("|%p|%p|FAILURE\n", ctx->pattern, ctx->ptr)); + RETURN_FAILURE; + + default: + TRACE(("|%p|%p|UNKNOWN %d\n", ctx->pattern, ctx->ptr, + ctx->pattern[-1])); + RETURN_ERROR(SRE_ERROR_ILLEGAL); + } + } + +exit: + ctx_pos = ctx->last_ctx_pos; + jump = ctx->jump; + DATA_POP_DISCARD(ctx); + if (ctx_pos == -1) + return ret; + DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); + + switch (jump) { + case JUMP_MAX_UNTIL_2: + TRACE(("|%p|%p|JUMP_MAX_UNTIL_2\n", ctx->pattern, ctx->ptr)); + goto jump_max_until_2; + case JUMP_MAX_UNTIL_3: + TRACE(("|%p|%p|JUMP_MAX_UNTIL_3\n", ctx->pattern, ctx->ptr)); + goto jump_max_until_3; + case JUMP_MIN_UNTIL_2: + TRACE(("|%p|%p|JUMP_MIN_UNTIL_2\n", ctx->pattern, ctx->ptr)); + goto jump_min_until_2; + case JUMP_MIN_UNTIL_3: + TRACE(("|%p|%p|JUMP_MIN_UNTIL_3\n", ctx->pattern, ctx->ptr)); + goto jump_min_until_3; + case JUMP_BRANCH: + TRACE(("|%p|%p|JUMP_BRANCH\n", ctx->pattern, ctx->ptr)); + goto jump_branch; + case JUMP_MAX_UNTIL_1: + TRACE(("|%p|%p|JUMP_MAX_UNTIL_1\n", ctx->pattern, ctx->ptr)); + goto jump_max_until_1; + case JUMP_MIN_UNTIL_1: + TRACE(("|%p|%p|JUMP_MIN_UNTIL_1\n", ctx->pattern, ctx->ptr)); + goto jump_min_until_1; + case JUMP_REPEAT: + TRACE(("|%p|%p|JUMP_REPEAT\n", ctx->pattern, ctx->ptr)); + goto jump_repeat; + case JUMP_REPEAT_ONE_1: + TRACE(("|%p|%p|JUMP_REPEAT_ONE_1\n", ctx->pattern, ctx->ptr)); + goto jump_repeat_one_1; + case JUMP_REPEAT_ONE_2: + TRACE(("|%p|%p|JUMP_REPEAT_ONE_2\n", ctx->pattern, ctx->ptr)); + goto jump_repeat_one_2; + case JUMP_MIN_REPEAT_ONE: + TRACE(("|%p|%p|JUMP_MIN_REPEAT_ONE\n", ctx->pattern, ctx->ptr)); + goto jump_min_repeat_one; + case JUMP_ASSERT: + TRACE(("|%p|%p|JUMP_ASSERT\n", ctx->pattern, ctx->ptr)); + goto jump_assert; + case JUMP_ASSERT_NOT: + TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); + goto jump_assert_not; + case JUMP_NONE: + TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + break; + } + + return ret; /* should never get here */ +} + +LOCAL(Py_ssize_t) +SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) +{ + SRE_CHAR* ptr = (SRE_CHAR *)state->start; + SRE_CHAR* end = (SRE_CHAR *)state->end; + Py_ssize_t status = 0; + Py_ssize_t prefix_len = 0; + Py_ssize_t prefix_skip = 0; + SRE_CODE* prefix = NULL; + SRE_CODE* charset = NULL; + SRE_CODE* overlap = NULL; + int flags = 0; + + if (pattern[0] == SRE_OP_INFO) { + /* optimization info block */ + /* <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> */ + + flags = pattern[2]; + + if (pattern[3] > 1) { + /* adjust end point (but make sure we leave at least one + character in there, so literal search will work) */ + end -= pattern[3]-1; + if (end <= ptr) + end = ptr+1; + } + + if (flags & SRE_INFO_PREFIX) { + /* pattern starts with a known prefix */ + /* */ + prefix_len = pattern[5]; + prefix_skip = pattern[6]; + prefix = pattern + 7; + overlap = prefix + prefix_len - 1; + } else if (flags & SRE_INFO_CHARSET) + /* pattern starts with a character from a known set */ + /* */ + charset = pattern + 5; + + pattern += 1 + pattern[1]; + } + + TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("charset = %p\n", charset)); + +#if defined(USE_FAST_SEARCH) + if (prefix_len > 1) { + /* pattern starts with a known prefix. use the overlap + table to skip forward as fast as we possibly can */ + Py_ssize_t i = 0; + end = (SRE_CHAR *)state->end; + while (ptr < end) { + for (;;) { + if ((SRE_CODE) ptr[0] != prefix[i]) { + if (!i) + break; + else + i = overlap[i]; + } else { + if (++i == prefix_len) { + /* found a potential match */ + TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr)); + state->start = ptr + 1 - prefix_len; + state->ptr = ptr + 1 - prefix_len + prefix_skip; + if (flags & SRE_INFO_LITERAL) + return 1; /* we got all of it */ + status = SRE_MATCH(state, pattern + 2*prefix_skip); + if (status != 0) + return status; + /* close but no cigar -- try again */ + i = overlap[i]; + } + break; + } + } + ptr++; + } + return 0; + } +#endif + + if (pattern[0] == SRE_OP_LITERAL) { + /* pattern starts with a literal character. this is used + for short prefixes, and if fast search is disabled */ + SRE_CODE chr = pattern[1]; + end = (SRE_CHAR *)state->end; + for (;;) { + while (ptr < end && (SRE_CODE) ptr[0] != chr) + ptr++; + if (ptr >= end) + return 0; + TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); + state->start = ptr; + state->ptr = ++ptr; + if (flags & SRE_INFO_LITERAL) + return 1; /* we got all of it */ + status = SRE_MATCH(state, pattern + 2); + if (status != 0) + break; + } + } else if (charset) { + /* pattern starts with a character from a known set */ + end = (SRE_CHAR *)state->end; + for (;;) { + while (ptr < end && !SRE_CHARSET(charset, ptr[0])) + ptr++; + if (ptr >= end) + return 0; + TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr)); + state->start = ptr; + state->ptr = ptr; + status = SRE_MATCH(state, pattern); + if (status != 0) + break; + ptr++; + } + } else + /* general case */ + while (ptr <= end) { + TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); + state->start = state->ptr = ptr++; + status = SRE_MATCH(state, pattern); + if (status != 0) + break; + } + + return status; +} + +LOCAL(int) +SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len) +{ + /* check if given string is a literal template (i.e. no escapes) */ + while (len-- > 0) + if (*ptr++ == '\\') + return 0; + return 1; +} + +#if !defined(SRE_RECURSIVE) + +/* -------------------------------------------------------------------- */ +/* factories and destructors */ + +/* see sre.h for object declarations */ +static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int); +static PyObject*pattern_scanner(PatternObject*, PyObject*); + +static PyObject * +sre_codesize(PyObject* self, PyObject *unused) +{ + return Py_BuildValue("l", sizeof(SRE_CODE)); +} + +static PyObject * +sre_getlower(PyObject* self, PyObject* args) +{ + int character, flags; + if (!PyArg_ParseTuple(args, "ii", &character, &flags)) + return NULL; + if (flags & SRE_FLAG_LOCALE) + return Py_BuildValue("i", sre_lower_locale(character)); + if (flags & SRE_FLAG_UNICODE) +#if defined(HAVE_UNICODE) + return Py_BuildValue("i", sre_lower_unicode(character)); +#else + return Py_BuildValue("i", sre_lower_locale(character)); +#endif + return Py_BuildValue("i", sre_lower(character)); +} + +LOCAL(void) +state_reset(SRE_STATE* state) +{ + /* FIXME: dynamic! */ + /*memset(state->mark, 0, sizeof(*state->mark) * SRE_MARK_SIZE);*/ + + state->lastmark = -1; + state->lastindex = -1; + + state->repeat = NULL; + + data_stack_dealloc(state); +} + +static void* +getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) +{ + /* given a python object, return a data pointer, a length (in + characters), and a character size. return NULL if the object + is not a string (or not compatible) */ + + PyBufferProcs *buffer; + Py_ssize_t size, bytes; + int charsize; + void* ptr; + +#if defined(HAVE_UNICODE) + if (PyUnicode_Check(string)) { + /* unicode strings doesn't always support the buffer interface */ + ptr = (void*) PyUnicode_AS_DATA(string); + /* bytes = PyUnicode_GET_DATA_SIZE(string); */ + size = PyUnicode_GET_SIZE(string); + charsize = sizeof(Py_UNICODE); + + } else { +#endif + + /* get pointer to string buffer */ + buffer = Py_TYPE(string)->tp_as_buffer; + if (!buffer || !buffer->bf_getreadbuffer || !buffer->bf_getsegcount || + buffer->bf_getsegcount(string, NULL) != 1) { + PyErr_SetString(PyExc_TypeError, "expected string or buffer"); + return NULL; + } + + /* determine buffer size */ + bytes = buffer->bf_getreadbuffer(string, 0, &ptr); + if (bytes < 0) { + PyErr_SetString(PyExc_TypeError, "buffer has negative size"); + return NULL; + } + + /* determine character size */ +#if PY_VERSION_HEX >= 0x01060000 + size = PyObject_Size(string); +#else + size = PyObject_Length(string); +#endif + + if (PyString_Check(string) || bytes == size) + charsize = 1; +#if defined(HAVE_UNICODE) + else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) + charsize = sizeof(Py_UNICODE); +#endif + else { + PyErr_SetString(PyExc_TypeError, "buffer size mismatch"); + return NULL; + } + +#if defined(HAVE_UNICODE) + } +#endif + + *p_length = size; + *p_charsize = charsize; + + return ptr; +} + +LOCAL(PyObject*) +state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, + Py_ssize_t start, Py_ssize_t end) +{ + /* prepare state object */ + + Py_ssize_t length; + int charsize; + void* ptr; + + memset(state, 0, sizeof(SRE_STATE)); + + state->lastmark = -1; + state->lastindex = -1; + + ptr = getstring(string, &length, &charsize); + if (!ptr) + return NULL; + + /* adjust boundaries */ + if (start < 0) + start = 0; + else if (start > length) + start = length; + + if (end < 0) + end = 0; + else if (end > length) + end = length; + + state->charsize = charsize; + + state->beginning = ptr; + + state->start = (void*) ((char*) ptr + start * state->charsize); + state->end = (void*) ((char*) ptr + end * state->charsize); + + Py_INCREF(string); + state->string = string; + state->pos = start; + state->endpos = end; + + if (pattern->flags & SRE_FLAG_LOCALE) + state->lower = sre_lower_locale; + else if (pattern->flags & SRE_FLAG_UNICODE) +#if defined(HAVE_UNICODE) + state->lower = sre_lower_unicode; +#else + state->lower = sre_lower_locale; +#endif + else + state->lower = sre_lower; + + return string; +} + +LOCAL(void) +state_fini(SRE_STATE* state) +{ + Py_XDECREF(state->string); + data_stack_dealloc(state); +} + +/* calculate offset from start of string */ +#define STATE_OFFSET(state, member)\ + (((char*)(member) - (char*)(state)->beginning) / (state)->charsize) + +LOCAL(PyObject*) +state_getslice(SRE_STATE* state, Py_ssize_t index, PyObject* string, int empty) +{ + Py_ssize_t i, j; + + index = (index - 1) * 2; + + if (string == Py_None || index >= state->lastmark || !state->mark[index] || !state->mark[index+1]) { + if (empty) + /* want empty string */ + i = j = 0; + else { + Py_INCREF(Py_None); + return Py_None; + } + } else { + i = STATE_OFFSET(state, state->mark[index]); + j = STATE_OFFSET(state, state->mark[index+1]); + } + + return PySequence_GetSlice(string, i, j); +} + +static void +pattern_error(int status) +{ + switch (status) { + case SRE_ERROR_RECURSION_LIMIT: + PyErr_SetString( + PyExc_RuntimeError, + "maximum recursion limit exceeded" + ); + break; + case SRE_ERROR_MEMORY: + PyErr_NoMemory(); + break; + case SRE_ERROR_INTERRUPTED: + /* An exception has already been raised, so let it fly */ + break; + default: + /* other error codes indicate compiler/engine bugs */ + PyErr_SetString( + PyExc_RuntimeError, + "internal error in regular expression engine" + ); + } +} + +static void +pattern_dealloc(PatternObject* self) +{ + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + Py_XDECREF(self->pattern); + Py_XDECREF(self->groupindex); + Py_XDECREF(self->indexgroup); + PyObject_DEL(self); +} + +static PyObject* +pattern_match(PatternObject* self, PyObject* args, PyObject* kw) +{ + SRE_STATE state; + int status; + + PyObject* string; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + static char* kwlist[] = { "pattern", "pos", "endpos", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:match", kwlist, + &string, &start, &end)) + return NULL; + + string = state_init(&state, self, string, start, end); + if (!string) + return NULL; + + state.ptr = state.start; + + TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr)); + + if (state.charsize == 1) { + status = sre_match(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_umatch(&state, PatternObject_GetCode(self)); +#endif + } + + TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); + if (PyErr_Occurred()) + return NULL; + + state_fini(&state); + + return pattern_new_match(self, &state, status); +} + +static PyObject* +pattern_search(PatternObject* self, PyObject* args, PyObject* kw) +{ + SRE_STATE state; + int status; + + PyObject* string; + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + static char* kwlist[] = { "pattern", "pos", "endpos", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:search", kwlist, + &string, &start, &end)) + return NULL; + + string = state_init(&state, self, string, start, end); + if (!string) + return NULL; + + TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr)); + + if (state.charsize == 1) { + status = sre_search(&state, PatternObject_GetCode(self)); + } else { +#if defined(HAVE_UNICODE) + status = sre_usearch(&state, PatternObject_GetCode(self)); +#endif + } + + TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); + + state_fini(&state); + + if (PyErr_Occurred()) + return NULL; + + return pattern_new_match(self, &state, status); +} + +static PyObject* +call(char* module, char* function, PyObject* args) +{ + PyObject* name; + PyObject* mod; + PyObject* func; + PyObject* result; + + if (!args) + return NULL; + name = PyString_FromString(module); + if (!name) + return NULL; + mod = PyImport_Import(name); + Py_DECREF(name); + if (!mod) + return NULL; + func = PyObject_GetAttrString(mod, function); + Py_DECREF(mod); + if (!func) + return NULL; + result = PyObject_CallObject(func, args); + Py_DECREF(func); + Py_DECREF(args); + return result; +} + +#ifdef USE_BUILTIN_COPY +static int +deepcopy(PyObject** object, PyObject* memo) +{ + PyObject* copy; + + copy = call( + "copy", "deepcopy", + PyTuple_Pack(2, *object, memo) + ); + if (!copy) + return 0; + + Py_DECREF(*object); + *object = copy; + + return 1; /* success */ +} +#endif + +static PyObject* +join_list(PyObject* list, PyObject* string) +{ + /* join list elements */ + + PyObject* joiner; +#if PY_VERSION_HEX >= 0x01060000 From pypy.commits at gmail.com Tue Mar 27 07:53:17 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 04:53:17 -0700 (PDT) Subject: [pypy-commit] pypy default: raise OSError not IOError (py2.7, linux failing stdlib test) Message-ID: <5aba30ad.93e51c0a.acdef.6e32@mx.google.com> Author: Matti Picus Branch: Changeset: r94146:56c6bd396489 Date: 2018-03-27 13:26 +0300 http://bitbucket.org/pypy/pypy/changeset/56c6bd396489/ Log: raise OSError not IOError (py2.7, linux failing stdlib test) diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -94,7 +94,7 @@ try: posix.fstat(fd) except OSError as e: - raise IOError(e.errno, e.message) + raise OSError(e.errno, e.message) return _fdopen(fd, mode, buffering) else: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except IOError as e: + except OSError as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" From pypy.commits at gmail.com Tue Mar 27 07:53:21 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 04:53:21 -0700 (PDT) Subject: [pypy-commit] pypy default: make test pass, note jitted code seems have many more ops now Message-ID: <5aba30b1.49d71c0a.a262c.6ca7@mx.google.com> Author: Matti Picus Branch: Changeset: r94148:bdbe0dc2c7be Date: 2018-03-27 13:27 +0300 http://bitbucket.org/pypy/pypy/changeset/bdbe0dc2c7be/ Log: make test pass, note jitted code seems have many more ops now diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -375,27 +375,58 @@ log = self.run(main, [300]) loop, = log.loops_by_filename(self.filepath) assert loop.match(""" - i161 = int_lt(i160, i43) + i106 = getfield_gc_i(p20, descr=...) + i161 = int_lt(i106, i43) guard_true(i161, descr=...) - i162 = int_add(i160, 1) - setfield_gc(p22, i162, descr=) + i162 = int_add(i106, 1) + p110 = getfield_gc_r(p16, descr=...) + setfield_gc(p20, i162, descr=...) + guard_value(p110, ConstPtr(ptr111), descr=...) guard_not_invalidated(descr=...) p163 = force_token() p164 = force_token() - p167 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=) + p118 = getfield_gc_r(p16, descr=...) + p120 = getarrayitem_gc_r(p118, 0, descr=...) + guard_value(p120, ConstPtr(ptr121), descr=...) + p122 = getfield_gc_r(p120, descr=...) + guard_value(p122, ConstPtr(ptr123), descr=...) + p125 = getfield_gc_r(p16, descr=...) + guard_nonnull_class(p125, ..., descr=...) + p127 = getfield_gc_r(p125, descr=...) + guard_value(p127, ConstPtr(ptr128), descr=...) + p129 = getfield_gc_r(p127, descr=...) + guard_value(p129, ConstPtr(ptr130), descr=...) + p132 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=...) guard_no_exception(descr=...) - i112 = int_signext(i160, 2) - setfield_gc(p167, ConstPtr(ptr85), descr=) - setfield_gc(p167, -1, descr=) - i114 = int_ne(i160, i112) - guard_false(i114, descr=...) - --TICK-- - i123 = arraylen_gc(p67, descr=) - i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero_mpressure__Signed), 6, descr=) - check_memory_error(i119) - raw_store(i119, 0, i160, descr=) - raw_store(i119, 2, i160, descr=) - raw_store(i119, 4, i160, descr=) - setfield_gc(p167, i119, descr=) + p133 = force_token() + p134 = new_with_vtable(descr=...) + setfield_gc(p134, ..., descr=...) + setfield_gc(p134, ConstPtr(null), descr=...) + setfield_gc(p48, p134, descr=...) + setfield_gc(p132, ..., descr=...) + i138 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=...) + check_memory_error(i138) + setfield_gc(p132, i138, descr=...) + setfield_gc(p132, ConstPtr(ptr139), descr=...) + setfield_gc(p132, -1, descr=...) + setfield_gc(p0, p133, descr=...) + call_may_force_n(ConstClass(_ll_2_gc_add_memory_pressure__Signed_pypy_module__cffi_backend_cdataobj_W_CDataNewStdPtr), 6, p132, descr=...) + guard_not_forced(descr=...) + guard_no_exception(descr=...) + i144 = int_add(i138, 0) + i146 = int_signext(i106, 2) + i147 = int_ne(i106, i146) + guard_false(i147, descr=...) + setarrayitem_raw(i144, 0, i106, descr=...) + i150 = int_add(i138, 2) + setarrayitem_raw(i150, 0, i106, descr=...) + i153 = int_add(i138, 4) + setarrayitem_raw(i153, 0, i106, descr=...) + p156 = getfield_gc_r(p48, descr=...) + i158 = getfield_raw_i(..., descr=...) + setfield_gc(p48, p49, descr=...) + setfield_gc(p134, ConstPtr(null), descr=...) + i160 = int_lt(i158, 0) + guard_false(i160, descr=...) jump(..., descr=...) """) From pypy.commits at gmail.com Tue Mar 27 07:53:23 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 04:53:23 -0700 (PDT) Subject: [pypy-commit] pypy unicode-utf8: merge default into branch Message-ID: <5aba30b3.138fdf0a.34481.aeb4@mx.google.com> Author: Matti Picus Branch: unicode-utf8 Changeset: r94149:a476da4baed5 Date: 2018-03-27 14:50 +0300 http://bitbucket.org/pypy/pypy/changeset/a476da4baed5/ Log: merge default into branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -59,7 +59,18 @@ Refactor in rpython signatures +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + .. branch: unicode-utf8-re .. branch: utf8-io Utf8 handling for unicode + diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1,3 +1,4 @@ +import pytest from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import rffi from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -6,6 +7,7 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import PyTypeObjectPtr + class AppTestTypeObject(AppTestCpythonExtensionBase): def setup_class(cls): @@ -136,8 +138,10 @@ module = self.import_module(name='foo') descr = module.fooType.copy assert type(descr).__name__ == 'method_descriptor' - assert str(descr) == "" - assert repr(descr) == "" + assert str(descr) in ("", + "") + assert repr(descr) in ("", + "") raises(TypeError, descr, None) def test_cython_fake_classmethod(self): @@ -250,7 +254,7 @@ import re assert re.sre_compile._sre is module s = u"Foo " * 1000 + u"Bar" - prog = re.compile(ur"Foo.*Bar") + prog = re.compile(u"Foo.*Bar") assert prog.match(s) m = re.search(u"xyz", u"xyzxyz") assert m @@ -319,7 +323,7 @@ def test_tp_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("read_tp_dict", "METH_O", + ("read_tp_dict", "METH_O", ''' PyObject *method; if (!args->ob_type->tp_dict) @@ -420,7 +424,7 @@ return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject*)type, "a"); + value = PyObject_GetAttrString((PyObject *)type, "a"); Py_DECREF(value); if (PyDict_SetItemString(type->tp_dict, "a", @@ -428,7 +432,7 @@ return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject*)type, "a"); + value = PyObject_GetAttrString((PyObject *)type, "a"); return value; ''' ) @@ -529,7 +533,7 @@ py_type = rffi.cast(PyTypeObjectPtr, ref) w_dict = from_ref(space, py_type.c_tp_dict) - w_name = space.wrap('a') + w_name = space.newtext('a') space.setitem(w_dict, w_name, space.wrap(1)) assert space.int_w(space.getattr(w_class, w_name)) == 1 space.delitem(w_dict, w_name) @@ -611,16 +615,21 @@ module = self.import_extension('foo', [ ("test_tp_getattro", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #define PyIntObject PyLongObject + #define PyInt_AsLong PyLong_AsLong + #endif PyObject *name, *obj = PyTuple_GET_ITEM(args, 0); - PyIntObject *attr, *value = (PyIntObject*) PyTuple_GET_ITEM(args, 1); + PyObject *attr, *value = PyTuple_GET_ITEM(args, 1); if (!obj->ob_type->tp_getattro) { PyErr_SetString(PyExc_ValueError, "missing tp_getattro"); return NULL; } name = PyString_FromString("attr1"); - attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name); - if (attr->ob_ival != value->ob_ival) + attr = obj->ob_type->tp_getattro(obj, name); + if (PyInt_AsLong(attr) != PyInt_AsLong(value)) { PyErr_SetString(PyExc_ValueError, "tp_getattro returned wrong value"); @@ -629,7 +638,7 @@ Py_DECREF(name); Py_DECREF(attr); name = PyString_FromString("attr2"); - attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name); + attr = obj->ob_type->tp_getattro(obj, name); if (attr == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -652,6 +661,9 @@ module = self.import_extension('foo', [ ("get_foo", "METH_O", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #endif char* name = "foo"; PyTypeObject *tp = Py_TYPE(args); PyObject *res; @@ -836,6 +848,10 @@ ''' )], prologue=''' static int + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #define PyInt_Check PyLong_Check + #endif mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value) { if (PyInt_Check(key)) { @@ -898,6 +914,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) { @@ -935,6 +955,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { @@ -983,6 +1007,9 @@ ), ("tp_iternext", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #endif PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0); PyObject *obj = PyTuple_GET_ITEM(args, 1); PyObject *result; @@ -1002,7 +1029,7 @@ it = module.tp_iter(list, l) assert type(it) is type(iter([])) assert module.tp_iternext(type(it), it) == 1 - assert module.tp_iternext(type(it), it) == "stop!" + assert module.tp_iternext(type(it), it) == b"stop!" # class LL(list): def __iter__(self): @@ -1132,7 +1159,11 @@ PyObject_HEAD long ival; } IntLikeObject; - + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #define PyInt_FromLong PyLong_FromLong + #endif static PyObject * intlike_nb_add(PyObject *self, PyObject *other) { @@ -1476,7 +1507,6 @@ )]) # used to segfault after some iterations for i in range(11): - print i class A(object): pass class B: @@ -1488,10 +1518,10 @@ def test_getattr_getattro(self): module = self.import_module(name='foo') - assert module.gettype2.dcba == 'getattro:dcba' + assert module.gettype2.dcba == b'getattro:dcba' assert (type(module.gettype2).__getattribute__(module.gettype2, 'dcBA') - == 'getattro:dcBA') - assert module.gettype1.abcd == 'getattr:abcd' + == b'getattro:dcBA') + assert module.gettype1.abcd == b'getattr:abcd' # GetType1 objects have a __getattribute__ method, but this # doesn't call tp_getattr at all, also on CPython raises(AttributeError, type(module.gettype1).__getattribute__, @@ -1533,6 +1563,9 @@ return PyInt_FromLong(42); ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #endif static PyTypeObject Foo_Type = { PyVarObject_HEAD_INIT(NULL, 0) "foo.foo", @@ -1635,8 +1668,10 @@ (int, Py_TPFLAGS_INT_SUBCLASS), (list, Py_TPFLAGS_LIST_SUBCLASS), (tuple, Py_TPFLAGS_TUPLE_SUBCLASS), + (bytes, Py_TPFLAGS_STRING_SUBCLASS), (str, Py_TPFLAGS_STRING_SUBCLASS), (unicode, Py_TPFLAGS_UNICODE_SUBCLASS), + (dict, Py_TPFLAGS_DICT_SUBCLASS), (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS), (type, Py_TPFLAGS_TYPE_SUBCLASS), ): @@ -1664,7 +1699,7 @@ return PyLong_FromLong(0); '''),]) # copied from object.h - Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1<<0) class MyFloat(float): pass diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py --- a/pypy/module/cpyext/userslot.py +++ b/pypy/module/cpyext/userslot.py @@ -49,6 +49,11 @@ w_stararg=w_args, w_starstararg=w_kwds) return space.call_args(w_impl, args) + at slot_function([PyObject, PyObject, PyObject], PyObject) +def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(w_self, args) + # unary functions @slot_function([PyObject], PyObject) diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -94,7 +94,7 @@ try: posix.fstat(fd) except OSError as e: - raise IOError(e.errno, e.message) + raise OSError(e.errno, e.message) return _fdopen(fd, mode, buffering) else: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except IOError as e: + except OSError as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -375,27 +375,58 @@ log = self.run(main, [300]) loop, = log.loops_by_filename(self.filepath) assert loop.match(""" - i161 = int_lt(i160, i43) + i106 = getfield_gc_i(p20, descr=...) + i161 = int_lt(i106, i43) guard_true(i161, descr=...) - i162 = int_add(i160, 1) - setfield_gc(p22, i162, descr=) + i162 = int_add(i106, 1) + p110 = getfield_gc_r(p16, descr=...) + setfield_gc(p20, i162, descr=...) + guard_value(p110, ConstPtr(ptr111), descr=...) guard_not_invalidated(descr=...) p163 = force_token() p164 = force_token() - p167 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=) + p118 = getfield_gc_r(p16, descr=...) + p120 = getarrayitem_gc_r(p118, 0, descr=...) + guard_value(p120, ConstPtr(ptr121), descr=...) + p122 = getfield_gc_r(p120, descr=...) + guard_value(p122, ConstPtr(ptr123), descr=...) + p125 = getfield_gc_r(p16, descr=...) + guard_nonnull_class(p125, ..., descr=...) + p127 = getfield_gc_r(p125, descr=...) + guard_value(p127, ConstPtr(ptr128), descr=...) + p129 = getfield_gc_r(p127, descr=...) + guard_value(p129, ConstPtr(ptr130), descr=...) + p132 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=...) guard_no_exception(descr=...) - i112 = int_signext(i160, 2) - setfield_gc(p167, ConstPtr(ptr85), descr=) - setfield_gc(p167, -1, descr=) - i114 = int_ne(i160, i112) - guard_false(i114, descr=...) - --TICK-- - i123 = arraylen_gc(p67, descr=) - i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero_mpressure__Signed), 6, descr=) - check_memory_error(i119) - raw_store(i119, 0, i160, descr=) - raw_store(i119, 2, i160, descr=) - raw_store(i119, 4, i160, descr=) - setfield_gc(p167, i119, descr=) + p133 = force_token() + p134 = new_with_vtable(descr=...) + setfield_gc(p134, ..., descr=...) + setfield_gc(p134, ConstPtr(null), descr=...) + setfield_gc(p48, p134, descr=...) + setfield_gc(p132, ..., descr=...) + i138 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=...) + check_memory_error(i138) + setfield_gc(p132, i138, descr=...) + setfield_gc(p132, ConstPtr(ptr139), descr=...) + setfield_gc(p132, -1, descr=...) + setfield_gc(p0, p133, descr=...) + call_may_force_n(ConstClass(_ll_2_gc_add_memory_pressure__Signed_pypy_module__cffi_backend_cdataobj_W_CDataNewStdPtr), 6, p132, descr=...) + guard_not_forced(descr=...) + guard_no_exception(descr=...) + i144 = int_add(i138, 0) + i146 = int_signext(i106, 2) + i147 = int_ne(i106, i146) + guard_false(i147, descr=...) + setarrayitem_raw(i144, 0, i106, descr=...) + i150 = int_add(i138, 2) + setarrayitem_raw(i150, 0, i106, descr=...) + i153 = int_add(i138, 4) + setarrayitem_raw(i153, 0, i106, descr=...) + p156 = getfield_gc_r(p48, descr=...) + i158 = getfield_raw_i(..., descr=...) + setfield_gc(p48, p49, descr=...) + setfield_gc(p134, ConstPtr(null), descr=...) + i160 = int_lt(i158, 0) + guard_false(i160, descr=...) jump(..., descr=...) """) From pypy.commits at gmail.com Tue Mar 27 07:53:19 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 04:53:19 -0700 (PDT) Subject: [pypy-commit] pypy default: document merged branches Message-ID: <5aba30af.94101c0a.24829.8fac@mx.google.com> Author: Matti Picus Branch: Changeset: r94147:25546c935c5e Date: 2018-03-27 13:26 +0300 http://bitbucket.org/pypy/pypy/changeset/25546c935c5e/ Log: document merged branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -58,3 +58,13 @@ .. branch: rpython-sprint Refactor in rpython signatures + +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks From pypy.commits at gmail.com Tue Mar 27 17:24:45 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 14:24:45 -0700 (PDT) Subject: [pypy-commit] pypy default: test, fix for cpython2 optimization where "buffer('') + obj is obj" Message-ID: <5abab69d.6ca0df0a.5314f.70ec@mx.google.com> Author: Matti Picus Branch: Changeset: r94151:d044ce2d7d53 Date: 2018-03-28 00:14 +0300 http://bitbucket.org/pypy/pypy/changeset/d044ce2d7d53/ Log: test, fix for cpython2 optimization where "buffer('') + obj is obj" diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -76,7 +76,9 @@ else: expected = '\x01\0\0\0' '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0' assert str(buf) == expected - assert str(buffer('') + arr) == expected + assert str(buffer('a') + arr) == "a" + expected + # python2 special cases empty-buffer + obj + assert str(buffer('') + arr) == "array('i', [1, 2, 3, 4])" def test_releasebuffer(self): module = self.import_module(name='array') diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py --- a/pypy/objspace/std/bufferobject.py +++ b/pypy/objspace/std/bufferobject.py @@ -89,9 +89,14 @@ def descr_str(self, space): return space.newbytes(self.buf.as_str()) - @unwrap_spec(other='bufferstr') - def descr_add(self, space, other): - return space.newbytes(self.buf.as_str() + other) + def descr_add(self, space, w_other): + try: + other = w_other.readbuf_w(space) + except BufferInterfaceNotFound: + raise oefmt(space.w_TypeError, "bad argument type for built-in operation") + if self.buf.getlength() < 1: + return w_other + return space.newbytes(self.buf.as_str() + other.as_str()) def _make_descr__cmp(name): def descr__cmp(self, space, w_other): diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py --- a/pypy/objspace/std/test/test_bufferobject.py +++ b/pypy/objspace/std/test/test_bufferobject.py @@ -29,9 +29,11 @@ def test_array_buffer(self): import array - b = buffer(array.array("B", [1, 2, 3])) + arr = array.array("B", [1, 2, 3]) + b = buffer(arr) assert len(b) == 3 assert b[0:3] == "\x01\x02\x03" + assert buffer('') + arr is arr def test_nonzero(self): assert buffer('\x00') @@ -51,6 +53,7 @@ assert buffer('abc') + 'def' == 'abcdef' import array assert buffer('abc') + array.array('c', 'def') == 'abcdef' + raises(TypeError, buffer('abc').__add__, 3) def test_cmp(self): assert buffer('ab') != 'ab' @@ -199,6 +202,9 @@ raises(TypeError, "buf[MyInt(0):MyInt(5)]") def test_pypy_raw_address_base(self): + import sys + if '__pypy__' not in sys.builtin_module_names: + skip('PyPy only') a = buffer("foobar")._pypy_raw_address() assert a != 0 b = buffer(u"foobar")._pypy_raw_address() From pypy.commits at gmail.com Tue Mar 27 17:24:42 2018 From: pypy.commits at gmail.com (mattip) Date: Tue, 27 Mar 2018 14:24:42 -0700 (PDT) Subject: [pypy-commit] pypy default: test, fix for PySequence_ITEM accessing sq_item, not mp_subscr. Both are __getitem__, the second takes precedence Message-ID: <5abab69a.04421c0a.2237f.048e@mx.google.com> Author: Matti Picus Branch: Changeset: r94150:abd1818fcde4 Date: 2018-03-28 00:02 +0300 http://bitbucket.org/pypy/pypy/changeset/abd1818fcde4/ Log: test, fix for PySequence_ITEM accessing sq_item, not mp_subscr. Both are __getitem__, the second takes precedence diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -5,7 +5,8 @@ from pypy.objspace.std.listobject import ( ListStrategy, UNROLL_CUTOFF, W_ListObject, ObjectListStrategy) from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP) + cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP, + generic_cpy_call) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import as_pyobj, incref from rpython.rtyper.lltypesystem import rffi, lltype @@ -145,21 +146,26 @@ # XXX we should call Py*_GET_ITEM() instead of Py*_GetItem() # from here, but we cannot because we are also called from # PySequence_GetItem() + py_obj = as_pyobj(space, w_obj) if isinstance(w_obj, tupleobject.W_TupleObject): from pypy.module.cpyext.tupleobject import PyTuple_GetItem - py_obj = as_pyobj(space, w_obj) py_res = PyTuple_GetItem(space, py_obj, i) incref(space, py_res) keepalive_until_here(w_obj) return py_res if isinstance(w_obj, W_ListObject): from pypy.module.cpyext.listobject import PyList_GetItem - py_obj = as_pyobj(space, w_obj) py_res = PyList_GetItem(space, py_obj, i) incref(space, py_res) keepalive_until_here(w_obj) return py_res - return make_ref(space, space.getitem(w_obj, space.newint(i))) + + as_sequence = py_obj.c_ob_type.c_tp_as_sequence + if not as_sequence or not as_sequence.c_sq_item: + raise oefmt(space.w_TypeError, + "'%T' object does not support indexing", w_obj) + ret = generic_cpy_call(space, as_sequence.c_sq_item, w_obj, i) + return make_ref(space, ret) @cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True) def PySequence_GetItem(space, w_obj, i): diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2202,6 +2202,16 @@ Py_RETURN_NONE; }; +static PyObject * +getitem(PyObject* self, PyObject * args) { + PyObject * obj; + int i; + if (!PyArg_ParseTuple(args, "Oi", &obj, &i)) { + return NULL; + } + return PySequence_ITEM(obj, i); +} + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ an array of basic values: characters, integers, floating point\n\ @@ -2491,6 +2501,7 @@ {"get_releasebuffer_cnt", (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL}, {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, + {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -172,3 +172,15 @@ fd = BytesIO() # only test that it works fd.write(a) + + def test_getitem_via_PySequence_GetItem(self): + module = self.import_module(name='array') + a = module.array('i', range(10)) + # call via tp_as_mapping.mp_subscript + assert 5 == a[-5] + # PySequence_ITEM used to call space.getitem() which + # prefers tp_as_mapping.mp_subscript over tp_as_sequence.sq_item + # Now fixed so this test raises (array_item does not add len(a), + # array_subscr does) + raises(IndexError, module.getitem, a, -5) + From pypy.commits at gmail.com Wed Mar 28 03:24:48 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 00:24:48 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: merge default Message-ID: <5abb4340.c7b3df0a.2af3a.6d67@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94152:f034b1d5a9e9 Date: 2018-03-28 09:24 +0200 http://bitbucket.org/pypy/pypy/changeset/f034b1d5a9e9/ Log: merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -59,6 +59,16 @@ Refactor in rpython signatures +.. branch: cpyext-tls-operror2 + +Store error state thread-locally in executioncontext, fixes issue #2764 + +.. branch: cpyext-fast-typecheck + +Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify +`W_PyCWrapperObject` which is used to call slots from the C-API, greatly +improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + .. branch: fix-sre-problems: diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -5,7 +5,8 @@ from pypy.objspace.std.listobject import ( ListStrategy, UNROLL_CUTOFF, W_ListObject, ObjectListStrategy) from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP) + cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP, + generic_cpy_call) from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref from pypy.module.cpyext.pyobject import as_pyobj, incref from rpython.rtyper.lltypesystem import rffi, lltype @@ -145,21 +146,26 @@ # XXX we should call Py*_GET_ITEM() instead of Py*_GetItem() # from here, but we cannot because we are also called from # PySequence_GetItem() + py_obj = as_pyobj(space, w_obj) if isinstance(w_obj, tupleobject.W_TupleObject): from pypy.module.cpyext.tupleobject import PyTuple_GetItem - py_obj = as_pyobj(space, w_obj) py_res = PyTuple_GetItem(space, py_obj, i) incref(space, py_res) keepalive_until_here(w_obj) return py_res if isinstance(w_obj, W_ListObject): from pypy.module.cpyext.listobject import PyList_GetItem - py_obj = as_pyobj(space, w_obj) py_res = PyList_GetItem(space, py_obj, i) incref(space, py_res) keepalive_until_here(w_obj) return py_res - return make_ref(space, space.getitem(w_obj, space.newint(i))) + + as_sequence = py_obj.c_ob_type.c_tp_as_sequence + if not as_sequence or not as_sequence.c_sq_item: + raise oefmt(space.w_TypeError, + "'%T' object does not support indexing", w_obj) + ret = generic_cpy_call(space, as_sequence.c_sq_item, w_obj, i) + return make_ref(space, ret) @cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True) def PySequence_GetItem(space, w_obj, i): diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c --- a/pypy/module/cpyext/test/array.c +++ b/pypy/module/cpyext/test/array.c @@ -2202,6 +2202,16 @@ Py_RETURN_NONE; }; +static PyObject * +getitem(PyObject* self, PyObject * args) { + PyObject * obj; + int i; + if (!PyArg_ParseTuple(args, "Oi", &obj, &i)) { + return NULL; + } + return PySequence_ITEM(obj, i); +} + PyDoc_STRVAR(module_doc, "This module defines an object type which can efficiently represent\n\ an array of basic values: characters, integers, floating point\n\ @@ -2491,6 +2501,7 @@ {"get_releasebuffer_cnt", (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL}, {"create_and_release_buffer", (PyCFunction)create_and_release_buffer, METH_O, NULL}, {"same_dealloc", (PyCFunction)same_dealloc, METH_VARARGS, NULL}, + {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py --- a/pypy/module/cpyext/test/test_arraymodule.py +++ b/pypy/module/cpyext/test/test_arraymodule.py @@ -76,7 +76,9 @@ else: expected = '\x01\0\0\0' '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0' assert str(buf) == expected - assert str(buffer('') + arr) == expected + assert str(buffer('a') + arr) == "a" + expected + # python2 special cases empty-buffer + obj + assert str(buffer('') + arr) == "array('i', [1, 2, 3, 4])" def test_releasebuffer(self): module = self.import_module(name='array') @@ -172,3 +174,15 @@ fd = BytesIO() # only test that it works fd.write(a) + + def test_getitem_via_PySequence_GetItem(self): + module = self.import_module(name='array') + a = module.array('i', range(10)) + # call via tp_as_mapping.mp_subscript + assert 5 == a[-5] + # PySequence_ITEM used to call space.getitem() which + # prefers tp_as_mapping.mp_subscript over tp_as_sequence.sq_item + # Now fixed so this test raises (array_item does not add len(a), + # array_subscr does) + raises(IndexError, module.getitem, a, -5) + diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -1,3 +1,4 @@ +import pytest from pypy.interpreter import gateway from rpython.rtyper.lltypesystem import rffi from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase @@ -6,6 +7,7 @@ from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj from pypy.module.cpyext.typeobject import PyTypeObjectPtr + class AppTestTypeObject(AppTestCpythonExtensionBase): def setup_class(cls): @@ -136,8 +138,10 @@ module = self.import_module(name='foo') descr = module.fooType.copy assert type(descr).__name__ == 'method_descriptor' - assert str(descr) == "" - assert repr(descr) == "" + assert str(descr) in ("", + "") + assert repr(descr) in ("", + "") raises(TypeError, descr, None) def test_cython_fake_classmethod(self): @@ -250,7 +254,7 @@ import re assert re.sre_compile._sre is module s = u"Foo " * 1000 + u"Bar" - prog = re.compile(ur"Foo.*Bar") + prog = re.compile(u"Foo.*Bar") assert prog.match(s) m = re.search(u"xyz", u"xyzxyz") assert m @@ -319,7 +323,7 @@ def test_tp_dict(self): foo = self.import_module("foo") module = self.import_extension('test', [ - ("read_tp_dict", "METH_O", + ("read_tp_dict", "METH_O", ''' PyObject *method; if (!args->ob_type->tp_dict) @@ -420,7 +424,7 @@ return NULL; Py_DECREF(a1); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject*)type, "a"); + value = PyObject_GetAttrString((PyObject *)type, "a"); Py_DECREF(value); if (PyDict_SetItemString(type->tp_dict, "a", @@ -428,7 +432,7 @@ return NULL; Py_DECREF(a2); PyType_Modified(type); - value = PyObject_GetAttrString((PyObject*)type, "a"); + value = PyObject_GetAttrString((PyObject *)type, "a"); return value; ''' ) @@ -529,7 +533,7 @@ py_type = rffi.cast(PyTypeObjectPtr, ref) w_dict = from_ref(space, py_type.c_tp_dict) - w_name = space.wrap('a') + w_name = space.newtext('a') space.setitem(w_dict, w_name, space.wrap(1)) assert space.int_w(space.getattr(w_class, w_name)) == 1 space.delitem(w_dict, w_name) @@ -611,16 +615,21 @@ module = self.import_extension('foo', [ ("test_tp_getattro", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #define PyIntObject PyLongObject + #define PyInt_AsLong PyLong_AsLong + #endif PyObject *name, *obj = PyTuple_GET_ITEM(args, 0); - PyIntObject *attr, *value = (PyIntObject*) PyTuple_GET_ITEM(args, 1); + PyObject *attr, *value = PyTuple_GET_ITEM(args, 1); if (!obj->ob_type->tp_getattro) { PyErr_SetString(PyExc_ValueError, "missing tp_getattro"); return NULL; } name = PyString_FromString("attr1"); - attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name); - if (attr->ob_ival != value->ob_ival) + attr = obj->ob_type->tp_getattro(obj, name); + if (PyInt_AsLong(attr) != PyInt_AsLong(value)) { PyErr_SetString(PyExc_ValueError, "tp_getattro returned wrong value"); @@ -629,7 +638,7 @@ Py_DECREF(name); Py_DECREF(attr); name = PyString_FromString("attr2"); - attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name); + attr = obj->ob_type->tp_getattro(obj, name); if (attr == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -652,6 +661,9 @@ module = self.import_extension('foo', [ ("get_foo", "METH_O", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyUnicode_FromString + #endif char* name = "foo"; PyTypeObject *tp = Py_TYPE(args); PyObject *res; @@ -836,6 +848,10 @@ ''' )], prologue=''' static int + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #define PyInt_Check PyLong_Check + #endif mp_ass_subscript(PyObject *self, PyObject *key, PyObject *value) { if (PyInt_Check(key)) { @@ -898,6 +914,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_slice(PyObject *self, Py_ssize_t a, Py_ssize_t b, PyObject *o) { @@ -935,6 +955,10 @@ return obj; ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #endif static int sq_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { @@ -983,6 +1007,9 @@ ), ("tp_iternext", "METH_VARARGS", ''' + #if PY_MAJOR_VERSION > 2 + #define PyString_FromString PyBytes_FromString + #endif PyTypeObject *type = (PyTypeObject *)PyTuple_GET_ITEM(args, 0); PyObject *obj = PyTuple_GET_ITEM(args, 1); PyObject *result; @@ -1002,7 +1029,7 @@ it = module.tp_iter(list, l) assert type(it) is type(iter([])) assert module.tp_iternext(type(it), it) == 1 - assert module.tp_iternext(type(it), it) == "stop!" + assert module.tp_iternext(type(it), it) == b"stop!" # class LL(list): def __iter__(self): @@ -1132,7 +1159,11 @@ PyObject_HEAD long ival; } IntLikeObject; - + #if PY_MAJOR_VERSION > 2 + #define PyInt_Check PyLong_Check + #define PyInt_AsLong PyLong_AsLong + #define PyInt_FromLong PyLong_FromLong + #endif static PyObject * intlike_nb_add(PyObject *self, PyObject *other) { @@ -1476,7 +1507,6 @@ )]) # used to segfault after some iterations for i in range(11): - print i class A(object): pass class B: @@ -1488,10 +1518,10 @@ def test_getattr_getattro(self): module = self.import_module(name='foo') - assert module.gettype2.dcba == 'getattro:dcba' + assert module.gettype2.dcba == b'getattro:dcba' assert (type(module.gettype2).__getattribute__(module.gettype2, 'dcBA') - == 'getattro:dcBA') - assert module.gettype1.abcd == 'getattr:abcd' + == b'getattro:dcBA') + assert module.gettype1.abcd == b'getattr:abcd' # GetType1 objects have a __getattribute__ method, but this # doesn't call tp_getattr at all, also on CPython raises(AttributeError, type(module.gettype1).__getattribute__, @@ -1533,6 +1563,9 @@ return PyInt_FromLong(42); ''' )], prologue=''' + #if PY_MAJOR_VERSION > 2 + #define PyInt_FromLong PyLong_FromLong + #endif static PyTypeObject Foo_Type = { PyVarObject_HEAD_INIT(NULL, 0) "foo.foo", @@ -1635,8 +1668,10 @@ (int, Py_TPFLAGS_INT_SUBCLASS), (list, Py_TPFLAGS_LIST_SUBCLASS), (tuple, Py_TPFLAGS_TUPLE_SUBCLASS), + (bytes, Py_TPFLAGS_STRING_SUBCLASS), (str, Py_TPFLAGS_STRING_SUBCLASS), (unicode, Py_TPFLAGS_UNICODE_SUBCLASS), + (dict, Py_TPFLAGS_DICT_SUBCLASS), (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS), (type, Py_TPFLAGS_TYPE_SUBCLASS), ): @@ -1664,7 +1699,7 @@ return PyLong_FromLong(0); '''),]) # copied from object.h - Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1L<<0) + Py_TPPYPYFLAGS_FLOAT_SUBCLASS = (1<<0) class MyFloat(float): pass diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py --- a/pypy/module/cpyext/userslot.py +++ b/pypy/module/cpyext/userslot.py @@ -49,6 +49,11 @@ w_stararg=w_args, w_starstararg=w_kwds) return space.call_args(w_impl, args) + at slot_function([PyObject, PyObject, PyObject], PyObject) +def slot_tp_call(space, w_self, w_args, w_kwds): + args = Arguments(space, [], w_stararg=w_args, w_starstararg=w_kwds) + return space.call_args(w_self, args) + # unary functions @slot_function([PyObject], PyObject) diff --git a/pypy/module/posix/app_posix.py b/pypy/module/posix/app_posix.py --- a/pypy/module/posix/app_posix.py +++ b/pypy/module/posix/app_posix.py @@ -94,7 +94,7 @@ try: posix.fstat(fd) except OSError as e: - raise IOError(e.errno, e.message) + raise OSError(e.errno, e.message) return _fdopen(fd, mode, buffering) else: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -303,7 +303,7 @@ try: fid = posix.fdopen(fd) fid.read(10) - except IOError as e: + except OSError as e: assert e.errno == errno.EBADF else: assert False, "using result of fdopen(fd) on closed file must raise" diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py --- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py +++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py @@ -375,27 +375,58 @@ log = self.run(main, [300]) loop, = log.loops_by_filename(self.filepath) assert loop.match(""" - i161 = int_lt(i160, i43) + i106 = getfield_gc_i(p20, descr=...) + i161 = int_lt(i106, i43) guard_true(i161, descr=...) - i162 = int_add(i160, 1) - setfield_gc(p22, i162, descr=) + i162 = int_add(i106, 1) + p110 = getfield_gc_r(p16, descr=...) + setfield_gc(p20, i162, descr=...) + guard_value(p110, ConstPtr(ptr111), descr=...) guard_not_invalidated(descr=...) p163 = force_token() p164 = force_token() - p167 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=) + p118 = getfield_gc_r(p16, descr=...) + p120 = getarrayitem_gc_r(p118, 0, descr=...) + guard_value(p120, ConstPtr(ptr121), descr=...) + p122 = getfield_gc_r(p120, descr=...) + guard_value(p122, ConstPtr(ptr123), descr=...) + p125 = getfield_gc_r(p16, descr=...) + guard_nonnull_class(p125, ..., descr=...) + p127 = getfield_gc_r(p125, descr=...) + guard_value(p127, ConstPtr(ptr128), descr=...) + p129 = getfield_gc_r(p127, descr=...) + guard_value(p129, ConstPtr(ptr130), descr=...) + p132 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=...) guard_no_exception(descr=...) - i112 = int_signext(i160, 2) - setfield_gc(p167, ConstPtr(ptr85), descr=) - setfield_gc(p167, -1, descr=) - i114 = int_ne(i160, i112) - guard_false(i114, descr=...) - --TICK-- - i123 = arraylen_gc(p67, descr=) - i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero_mpressure__Signed), 6, descr=) - check_memory_error(i119) - raw_store(i119, 0, i160, descr=) - raw_store(i119, 2, i160, descr=) - raw_store(i119, 4, i160, descr=) - setfield_gc(p167, i119, descr=) + p133 = force_token() + p134 = new_with_vtable(descr=...) + setfield_gc(p134, ..., descr=...) + setfield_gc(p134, ConstPtr(null), descr=...) + setfield_gc(p48, p134, descr=...) + setfield_gc(p132, ..., descr=...) + i138 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=...) + check_memory_error(i138) + setfield_gc(p132, i138, descr=...) + setfield_gc(p132, ConstPtr(ptr139), descr=...) + setfield_gc(p132, -1, descr=...) + setfield_gc(p0, p133, descr=...) + call_may_force_n(ConstClass(_ll_2_gc_add_memory_pressure__Signed_pypy_module__cffi_backend_cdataobj_W_CDataNewStdPtr), 6, p132, descr=...) + guard_not_forced(descr=...) + guard_no_exception(descr=...) + i144 = int_add(i138, 0) + i146 = int_signext(i106, 2) + i147 = int_ne(i106, i146) + guard_false(i147, descr=...) + setarrayitem_raw(i144, 0, i106, descr=...) + i150 = int_add(i138, 2) + setarrayitem_raw(i150, 0, i106, descr=...) + i153 = int_add(i138, 4) + setarrayitem_raw(i153, 0, i106, descr=...) + p156 = getfield_gc_r(p48, descr=...) + i158 = getfield_raw_i(..., descr=...) + setfield_gc(p48, p49, descr=...) + setfield_gc(p134, ConstPtr(null), descr=...) + i160 = int_lt(i158, 0) + guard_false(i160, descr=...) jump(..., descr=...) """) diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py --- a/pypy/objspace/std/bufferobject.py +++ b/pypy/objspace/std/bufferobject.py @@ -89,9 +89,14 @@ def descr_str(self, space): return space.newbytes(self.buf.as_str()) - @unwrap_spec(other='bufferstr') - def descr_add(self, space, other): - return space.newbytes(self.buf.as_str() + other) + def descr_add(self, space, w_other): + try: + other = w_other.readbuf_w(space) + except BufferInterfaceNotFound: + raise oefmt(space.w_TypeError, "bad argument type for built-in operation") + if self.buf.getlength() < 1: + return w_other + return space.newbytes(self.buf.as_str() + other.as_str()) def _make_descr__cmp(name): def descr__cmp(self, space, w_other): diff --git a/pypy/objspace/std/test/test_bufferobject.py b/pypy/objspace/std/test/test_bufferobject.py --- a/pypy/objspace/std/test/test_bufferobject.py +++ b/pypy/objspace/std/test/test_bufferobject.py @@ -29,9 +29,11 @@ def test_array_buffer(self): import array - b = buffer(array.array("B", [1, 2, 3])) + arr = array.array("B", [1, 2, 3]) + b = buffer(arr) assert len(b) == 3 assert b[0:3] == "\x01\x02\x03" + assert buffer('') + arr is arr def test_nonzero(self): assert buffer('\x00') @@ -51,6 +53,7 @@ assert buffer('abc') + 'def' == 'abcdef' import array assert buffer('abc') + array.array('c', 'def') == 'abcdef' + raises(TypeError, buffer('abc').__add__, 3) def test_cmp(self): assert buffer('ab') != 'ab' @@ -199,6 +202,9 @@ raises(TypeError, "buf[MyInt(0):MyInt(5)]") def test_pypy_raw_address_base(self): + import sys + if '__pypy__' not in sys.builtin_module_names: + skip('PyPy only') a = buffer("foobar")._pypy_raw_address() assert a != 0 b = buffer(u"foobar")._pypy_raw_address() From pypy.commits at gmail.com Wed Mar 28 04:45:53 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 28 Mar 2018 01:45:53 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: Add comment Message-ID: <5abb5641.8f8d1c0a.874ae.5148@mx.google.com> Author: Armin Rigo Branch: fix-sre-problems Changeset: r94153:e0f94dd72f35 Date: 2018-03-28 10:45 +0200 http://bitbucket.org/pypy/pypy/changeset/e0f94dd72f35/ Log: Add comment diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -388,6 +388,10 @@ srepat.space = space srepat.w_pattern = w_pattern # the original uncompiled pattern srepat.flags = flags + # note: we assume that the app-level is caching SRE_Pattern objects, + # so that we don't need to do it here. Creating new SRE_Pattern + # objects all the time would be bad for the JIT, which relies on the + # identity of the CompiledPattern() object. srepat.code = rsre_core.CompiledPattern(code) srepat.num_groups = groups srepat.w_groupindex = w_groupindex From pypy.commits at gmail.com Wed Mar 28 04:50:31 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 01:50:31 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: - remove frontend_tag_overflow Message-ID: <5abb5757.96e81c0a.407e5.4956@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94154:c7964f0be55f Date: 2018-03-28 10:49 +0200 http://bitbucket.org/pypy/pypy/changeset/c7964f0be55f/ Log: - remove frontend_tag_overflow - rename done to not clash with TraceIterator.done - add a defensive assert there diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -49,13 +49,6 @@ way up to lltype.Signed for indexes everywhere """ -def frontend_tag_overflow(): - # Minor abstraction leak: raise directly the right exception - # expected by the rest of the machinery - from rpython.jit.metainterp import history - from rpython.rlib.jit import Counters - raise history.SwitchToBlackhole(Counters.ABORT_TOO_LONG) - class BaseTrace(object): pass @@ -306,8 +299,10 @@ self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v) self._pos += 1 - def done(self): + def tracing_done(self, abandoned_trace=False): from rpython.rlib.debug import debug_start, debug_stop, debug_print + if not abandoned_trace: + assert not self.tag_overflow self._bigints_dict = {} self._refs_dict = llhelper.new_ref_dict_3() @@ -319,8 +314,6 @@ debug_print(" ref consts: " + str(self._consts_ptr) + " " + str(len(self._refs))) debug_print(" descrs: " + str(len(self._descrs))) debug_stop("jit-trace-done") - return 0 # completely different than TraceIter.done, but we have to - # share the base class def length(self): return self._pos diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2387,7 +2387,7 @@ if (self.history.length() > warmrunnerstate.trace_limit or self.history.trace_tag_overflow()): jd_sd, greenkey_of_huge_function = self.find_biggest_function() - self.history.trace.done() + self.history.trace.tracing_done(abandoned_trace=True) self.staticdata.stats.record_aborted(greenkey_of_huge_function) self.portal_trace_positions = None if greenkey_of_huge_function is not None: @@ -2690,7 +2690,7 @@ try_disabling_unroll=False, exported_state=None): num_green_args = self.jitdriver_sd.num_green_args greenkey = original_boxes[:num_green_args] - self.history.trace.done() + self.history.trace.tracing_done() if not self.partial_trace: ptoken = self.get_procedure_token(greenkey) if ptoken is not None and ptoken.target_tokens is not None: @@ -2743,7 +2743,7 @@ self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None, descr=target_jitcell_token) self.history.ends_with_jump = True - self.history.trace.done() + self.history.trace.tracing_done() try: target_token = compile.compile_trace(self, self.resumekey, live_arg_boxes[num_green_args:]) @@ -2777,7 +2777,7 @@ assert False # FIXME: can we call compile_trace? self.history.record(rop.FINISH, exits, None, descr=token) - self.history.trace.done() + self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, exits) if target_token is not token: compile.giveup() @@ -2803,7 +2803,7 @@ sd = self.staticdata token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) - self.history.trace.done() + self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) if target_token is not token: compile.giveup() From pypy.commits at gmail.com Wed Mar 28 04:51:25 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 01:51:25 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: typo Message-ID: <5abb578d.22b7df0a.8b10c.32ba@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94155:388756e1820b Date: 2018-03-28 10:50 +0200 http://bitbucket.org/pypy/pypy/changeset/388756e1820b/ Log: typo diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -70,7 +70,7 @@ improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks -.. branch: fix-sre-problems: +.. branch: fix-sre-problems Fix two (unrelated) JIT bugs manifesting in the re module: From pypy.commits at gmail.com Wed Mar 28 05:35:02 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 02:35:02 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: fix test for green fields Message-ID: <5abb61c6.0f8d1c0a.31cb0.5c9a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94158:30487ab1f380 Date: 2018-03-28 11:34 +0200 http://bitbucket.org/pypy/pypy/changeset/30487ab1f380/ Log: fix test for green fields diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -225,8 +225,10 @@ def test_green_field(self): def get_printable_location(xfoo): return str(ord(xfoo)) # xfoo must be annotated as a character - myjitdriver = JitDriver(greens=['x.foo'], reds=['n', 'x'], + # green fields are disabled! + pytest.raises(ValueError, JitDriver, greens=['x.foo'], reds=['n', 'x'], get_printable_location=get_printable_location) + return class A(object): _immutable_fields_ = ['foo'] def fn(n): From pypy.commits at gmail.com Wed Mar 28 05:34:57 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 02:34:57 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: check for tag overflow before every call to tracing_done Message-ID: <5abb61c1.14a1df0a.ef506.5511@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94156:bee54814e222 Date: 2018-03-28 11:07 +0200 http://bitbucket.org/pypy/pypy/changeset/bee54814e222/ Log: check for tag overflow before every call to tracing_done diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2690,6 +2690,8 @@ try_disabling_unroll=False, exported_state=None): num_green_args = self.jitdriver_sd.num_green_args greenkey = original_boxes[:num_green_args] + if self.trace_tag_overflow(): + raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) self.history.trace.tracing_done() if not self.partial_trace: ptoken = self.get_procedure_token(greenkey) @@ -2743,6 +2745,8 @@ self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None, descr=target_jitcell_token) self.history.ends_with_jump = True + if self.trace_tag_overflow(): + raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) self.history.trace.tracing_done() try: target_token = compile.compile_trace(self, self.resumekey, @@ -2777,6 +2781,8 @@ assert False # FIXME: can we call compile_trace? self.history.record(rop.FINISH, exits, None, descr=token) + if self.trace_tag_overflow(): + raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, exits) if target_token is not token: @@ -2803,6 +2809,8 @@ sd = self.staticdata token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) + if self.trace_tag_overflow(): + raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) if target_token is not token: From pypy.commits at gmail.com Wed Mar 28 05:35:00 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 02:35:00 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: check for trace tag overflow more systematically (thanks Armin) Message-ID: <5abb61c4.516d1c0a.7267b.5b8f@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94157:0d7163be06ae Date: 2018-03-28 11:32 +0200 http://bitbucket.org/pypy/pypy/changeset/0d7163be06ae/ Log: check for trace tag overflow more systematically (thanks Armin) diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -299,10 +299,9 @@ self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v) self._pos += 1 - def tracing_done(self, abandoned_trace=False): + def tracing_done(self): from rpython.rlib.debug import debug_start, debug_stop, debug_print - if not abandoned_trace: - assert not self.tag_overflow + assert not self.tag_overflow self._bigints_dict = {} self._refs_dict = llhelper.new_ref_dict_3() diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2387,7 +2387,6 @@ if (self.history.length() > warmrunnerstate.trace_limit or self.history.trace_tag_overflow()): jd_sd, greenkey_of_huge_function = self.find_biggest_function() - self.history.trace.tracing_done(abandoned_trace=True) self.staticdata.stats.record_aborted(greenkey_of_huge_function) self.portal_trace_positions = None if greenkey_of_huge_function is not None: @@ -2690,8 +2689,8 @@ try_disabling_unroll=False, exported_state=None): num_green_args = self.jitdriver_sd.num_green_args greenkey = original_boxes[:num_green_args] - if self.trace_tag_overflow(): - raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) self.history.trace.tracing_done() if not self.partial_trace: ptoken = self.get_procedure_token(greenkey) @@ -2745,8 +2744,8 @@ self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None, descr=target_jitcell_token) self.history.ends_with_jump = True - if self.trace_tag_overflow(): - raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) self.history.trace.tracing_done() try: target_token = compile.compile_trace(self, self.resumekey, @@ -2781,8 +2780,8 @@ assert False # FIXME: can we call compile_trace? self.history.record(rop.FINISH, exits, None, descr=token) - if self.trace_tag_overflow(): - raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, exits) if target_token is not token: @@ -2809,8 +2808,8 @@ sd = self.staticdata token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) - if self.trace_tag_overflow(): - raise SwitchToBlackhole(Counter.ABORT_TOO_LONG) + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) if target_token is not token: From pypy.commits at gmail.com Wed Mar 28 06:39:04 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 03:39:04 -0700 (PDT) Subject: [pypy-commit] pypy default: silence test_llinterp_complete Message-ID: <5abb70c8.6685df0a.27b10.52d6@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r94159:b437cad15ce6 Date: 2018-03-28 12:38 +0200 http://bitbucket.org/pypy/pypy/changeset/b437cad15ce6/ Log: silence test_llinterp_complete (bad fijal, no cookie) diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -742,6 +742,9 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") + def op_gc_get_stats(self, obj): + raise NotImplementedError("gc_get_stats") + def op_gc_writebarrier_before_copy(self, source, dest, source_start, dest_start, length): if hasattr(self.heap, 'writebarrier_before_copy'): From pypy.commits at gmail.com Wed Mar 28 06:47:04 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 03:47:04 -0700 (PDT) Subject: [pypy-commit] pypy fix-sre-problems: merge default Message-ID: <5abb72a8.0eaddf0a.355dd.6029@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: fix-sre-problems Changeset: r94160:f44921d7b728 Date: 2018-03-28 12:46 +0200 http://bitbucket.org/pypy/pypy/changeset/f44921d7b728/ Log: merge default diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -742,6 +742,9 @@ def op_zero_gc_pointers_inside(self, obj): raise NotImplementedError("zero_gc_pointers_inside") + def op_gc_get_stats(self, obj): + raise NotImplementedError("gc_get_stats") + def op_gc_writebarrier_before_copy(self, source, dest, source_start, dest_start, length): if hasattr(self.heap, 'writebarrier_before_copy'): From pypy.commits at gmail.com Wed Mar 28 08:03:46 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 05:03:46 -0700 (PDT) Subject: [pypy-commit] pypy jit-hooks-can-be-disabled: use new hook to check whether any hooks are set at all Message-ID: <5abb84a2.01b9df0a.78523.9f24@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: jit-hooks-can-be-disabled Changeset: r94162:d27c76b67b51 Date: 2018-03-28 14:03 +0200 http://bitbucket.org/pypy/pypy/changeset/d27c76b67b51/ Log: use new hook to check whether any hooks are set at all diff --git a/pypy/module/pypyjit/hooks.py b/pypy/module/pypyjit/hooks.py --- a/pypy/module/pypyjit/hooks.py +++ b/pypy/module/pypyjit/hooks.py @@ -7,12 +7,20 @@ WrappedOp, W_JitLoopInfo, wrap_oplist) class PyPyJitIface(JitHookInterface): + def are_hooks_enabled(self): + space = self.space + cache = space.fromcache(Cache) + return (cache.w_compile_hook is not None or + cache.w_abort_hook is not None or + cache.w_trace_too_long_hook is not None) + + def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): space = self.space cache = space.fromcache(Cache) if cache.in_recursion: return - if space.is_true(cache.w_abort_hook): + if cache.w_abort_hook is not None: cache.in_recursion = True oplist_w = wrap_oplist(space, logops, operations) try: @@ -33,7 +41,7 @@ cache = space.fromcache(Cache) if cache.in_recursion: return - if space.is_true(cache.w_trace_too_long_hook): + if cache.w_trace_too_long_hook is not None: cache.in_recursion = True try: try: @@ -62,7 +70,7 @@ cache = space.fromcache(Cache) if cache.in_recursion: return - if space.is_true(cache.w_compile_hook): + if cache.w_compile_hook is not None: w_debug_info = W_JitLoopInfo(space, debug_info, is_bridge, cache.compile_hook_with_ops) cache.in_recursion = True diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -21,9 +21,10 @@ no = 0 def __init__(self, space): - self.w_compile_hook = space.w_None - self.w_abort_hook = space.w_None - self.w_trace_too_long_hook = space.w_None + self.w_compile_hook = None + self.w_abort_hook = None + self.w_trace_too_long_hook = None + self.compile_hook_with_ops = False def getno(self): self.no += 1 @@ -58,7 +59,8 @@ jit hook won't be called for that. """ cache = space.fromcache(Cache) - assert w_hook is not None + if space.is_w(w_hook, space.w_None): + w_hook = None cache.w_compile_hook = w_hook cache.compile_hook_with_ops = operations cache.in_recursion = NonConstant(False) @@ -77,7 +79,8 @@ as attributes on JitLoopInfo object. """ cache = space.fromcache(Cache) - assert w_hook is not None + if space.is_w(w_hook, space.w_None): + w_hook = None cache.w_abort_hook = w_hook cache.in_recursion = NonConstant(False) @@ -92,14 +95,15 @@ hook(jitdriver_name, greenkey) """ cache = space.fromcache(Cache) - assert w_hook is not None + if space.is_w(w_hook, space.w_None): + w_hook = None cache.w_trace_too_long_hook = w_hook cache.in_recursion = NonConstant(False) def wrap_oplist(space, logops, operations, ops_offset=None): # this function is called from the JIT from rpython.jit.metainterp.resoperation import rop - + l_w = [] jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd for op in operations: diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -86,18 +86,22 @@ def interp_on_compile(): di_loop.oplist = cls.oplist - pypy_hooks.after_compile(di_loop) + if pypy_hooks.are_hooks_enabled(): + pypy_hooks.after_compile(di_loop) def interp_on_compile_bridge(): - pypy_hooks.after_compile_bridge(di_bridge) + if pypy_hooks.are_hooks_enabled(): + pypy_hooks.after_compile_bridge(di_bridge) def interp_on_optimize(): - di_loop_optimize.oplist = cls.oplist - pypy_hooks.before_compile(di_loop_optimize) + if pypy_hooks.are_hooks_enabled(): + di_loop_optimize.oplist = cls.oplist + pypy_hooks.before_compile(di_loop_optimize) def interp_on_abort(): - pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver, - greenkey, 'blah', Logger(MockSD), []) + if pypy_hooks.are_hooks_enabled(): + pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver, + greenkey, 'blah', Logger(MockSD), []) space = cls.space cls.w_on_compile = space.wrap(interp2app(interp_on_compile)) From pypy.commits at gmail.com Wed Mar 28 08:03:44 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 05:03:44 -0700 (PDT) Subject: [pypy-commit] pypy jit-hooks-can-be-disabled: add a hook that makes it possible to disable all other hooks. Message-ID: <5abb84a0.f4a0df0a.b4d43.d288@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: jit-hooks-can-be-disabled Changeset: r94161:df360c36c909 Date: 2018-03-28 13:28 +0200 http://bitbucket.org/pypy/pypy/changeset/df360c36c909/ Log: add a hook that makes it possible to disable all other hooks. diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -545,15 +545,15 @@ show_procedures(metainterp_sd, loop) loop.check_consistency() + debug_info = None + hooks = None if metainterp_sd.warmrunnerdesc is not None: hooks = metainterp_sd.warmrunnerdesc.hooks - debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, - original_jitcell_token, loop.operations, - type, greenkey) - hooks.before_compile(debug_info) - else: - debug_info = None - hooks = None + if hooks.are_hooks_enabled(): + debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, + original_jitcell_token, loop.operations, + type, greenkey) + hooks.before_compile(debug_info) operations = get_deep_immutable_oplist(loop.operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") @@ -597,15 +597,15 @@ show_procedures(metainterp_sd) seen = dict.fromkeys(inputargs) TreeLoop.check_consistency_of_branch(operations, seen) + debug_info = None + hooks = None if metainterp_sd.warmrunnerdesc is not None: hooks = metainterp_sd.warmrunnerdesc.hooks - debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, - original_loop_token, operations, 'bridge', - fail_descr=faildescr) - hooks.before_compile_bridge(debug_info) - else: - hooks = None - debug_info = None + if hooks.are_hooks_enabled(): + debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, + original_loop_token, operations, 'bridge', + fail_descr=faildescr) + hooks.before_compile_bridge(debug_info) operations = get_deep_immutable_oplist(operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2365,7 +2365,9 @@ greenkey = None # we're in the bridge else: greenkey = self.current_merge_points[0][0][:jd_sd.num_green_args] - self.staticdata.warmrunnerdesc.hooks.on_abort(reason, + hooks = self.staticdata.warmrunnerdesc.hooks + if hooks.are_hooks_enabled(): + hooks.on_abort(reason, jd_sd.jitdriver, greenkey, jd_sd.warmstate.get_location_str(greenkey), self.staticdata.logger_ops._make_log_operations( @@ -2374,9 +2376,10 @@ if self.aborted_tracing_jitdriver is not None: jd_sd = self.aborted_tracing_jitdriver greenkey = self.aborted_tracing_greenkey - self.staticdata.warmrunnerdesc.hooks.on_trace_too_long( - jd_sd.jitdriver, greenkey, - jd_sd.warmstate.get_location_str(greenkey)) + if hooks.are_hooks_enabled(): + hooks.on_trace_too_long( + jd_sd.jitdriver, greenkey, + jd_sd.warmstate.get_location_str(greenkey)) # no ops for now self.aborted_tracing_jitdriver = None self.aborted_tracing_greenkey = None diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py --- a/rpython/jit/metainterp/test/test_jitiface.py +++ b/rpython/jit/metainterp/test/test_jitiface.py @@ -238,7 +238,7 @@ hashes = Hashes() - class Hooks(object): + class Hooks(JitHookInterface): def before_compile(self, debug_info): pass @@ -279,6 +279,44 @@ self.meta_interp(main, [1, 1], policy=JitPolicy(hooks)) assert len(hashes.t) == 1 + + def test_are_hooks_enabled(self): + reasons = [] + + class MyJitIface(JitHookInterface): + def are_hooks_enabled(self): + return False + + def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, ops): + reasons.append(reason) + + iface = MyJitIface() + + myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'], + get_printable_location=lambda *args: 'blah') + + class Foo: + _immutable_fields_ = ['a?'] + + def __init__(self, a): + self.a = a + + def f(a, x): + foo = Foo(a) + total = 0 + while x > 0: + myjitdriver.jit_merge_point(foo=foo, x=x, total=total) + total += foo.a + foo.a += 1 + x -= 1 + return total + # + assert f(100, 7) == 721 + res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface)) + assert res == 721 + assert reasons == [] + + class LLJitHookInterfaceTests(JitHookInterfaceTests): # use this for any backend, instead of the super class @@ -320,7 +358,6 @@ # this so far does not work because of the way setup_once is done, # but fine, it's only about untranslated version anyway #self.meta_interp(main, [False], ProfilerClass=Profiler) - class TestJitHookInterface(JitHookInterfaceTests, LLJitMixin): pass diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1094,6 +1094,13 @@ # of the program! A line like ``pypy_hooks.foo = ...`` must not # appear inside your interpreter's RPython code. + def are_hooks_enabled(self): + """ A hook that is called to check whether the interpreter's hooks are + enabled at all. Only if this function returns True, are the other hooks + called. Otherwise, nothing happens. This is done because constructing + some of the hooks' arguments is expensive, so we'd rather not do it.""" + return True + def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): """ A hook called each time a loop is aborted with jitdriver and greenkey where it started, reason is a string why it got aborted From pypy.commits at gmail.com Wed Mar 28 08:35:23 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 05:35:23 -0700 (PDT) Subject: [pypy-commit] pypy jit-hooks-can-be-disabled: if no JitHookInterface is given, disable hooks completely Message-ID: <5abb8c0b.0f8d1c0a.31cb0.8b9a@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: jit-hooks-can-be-disabled Changeset: r94163:f85a77753cf9 Date: 2018-03-28 14:20 +0200 http://bitbucket.org/pypy/pypy/changeset/f85a77753cf9/ Log: if no JitHookInterface is given, disable hooks completely diff --git a/rpython/jit/codewriter/policy.py b/rpython/jit/codewriter/policy.py --- a/rpython/jit/codewriter/policy.py +++ b/rpython/jit/codewriter/policy.py @@ -11,9 +11,6 @@ self.supports_floats = False self.supports_longlong = False self.supports_singlefloats = False - if jithookiface is None: - from rpython.rlib.jit import JitHookInterface - jithookiface = JitHookInterface() self.jithookiface = jithookiface def set_supports_floats(self, flag): diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -554,6 +554,8 @@ original_jitcell_token, loop.operations, type, greenkey) hooks.before_compile(debug_info) + else: + hooks = None operations = get_deep_immutable_oplist(loop.operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") @@ -606,6 +608,8 @@ original_loop_token, operations, 'bridge', fail_descr=faildescr) hooks.before_compile_bridge(debug_info) + else: + hooks = None operations = get_deep_immutable_oplist(operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -220,6 +220,15 @@ stats.check_consistency() # ____________________________________________________________ +# always disabled hooks interface + +from rpython.rlib.jit import JitHookInterface + +class NoHooksInterface(JitHookInterface): + def are_hooks_enabled(self): + return False + +# ____________________________________________________________ class WarmRunnerDesc(object): @@ -259,7 +268,7 @@ else: self.jitcounter = counter.DeterministicJitCounter() # - self.hooks = policy.jithookiface + self.make_hooks(policy.jithookiface) self.make_virtualizable_infos() self.make_driverhook_graphs() self.make_enter_functions() @@ -498,6 +507,12 @@ self.metainterp_sd.opencoder_model = Model self.stats.metainterp_sd = self.metainterp_sd + def make_hooks(self, hooks): + if hooks is None: + # interface not overridden, use a special one that is never enabled + hooks = NoHooksInterface() + self.hooks = hooks + def make_virtualizable_infos(self): vinfos = {} for jd in self.jitdrivers_sd: diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1084,7 +1084,8 @@ """ This is the main connector between the JIT and the interpreter. Several methods on this class will be invoked at various stages of JIT running like JIT loops compiled, aborts etc. - An instance of this class will be available as policy.jithookiface. + An instance of this class has to be passed into the JitPolicy constructor + (and will then be available as policy.jithookiface). """ # WARNING: You should make a single prebuilt instance of a subclass # of this class. You can, before translation, initialize some From pypy.commits at gmail.com Wed Mar 28 08:54:52 2018 From: pypy.commits at gmail.com (cfbolz) Date: Wed, 28 Mar 2018 05:54:52 -0700 (PDT) Subject: [pypy-commit] pypy default: merge fix-sre-problems: Message-ID: <5abb909c.942d1c0a.313f8.4743@mx.google.com> Author: Carl Friedrich Bolz-Tereick Branch: Changeset: r94164:ac140c11bea3 Date: 2018-03-28 14:54 +0200 http://bitbucket.org/pypy/pypy/changeset/ac140c11bea3/ Log: merge fix-sre-problems: - stop switching to the blackhole interpreter in random places, which leads to arbitrary misbehaviour - disable greenfields, because their interaction with virtualizables is broken diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -68,3 +68,14 @@ Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify `W_PyCWrapperObject` which is used to call slots from the C-API, greatly improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks + + +.. branch: fix-sre-problems + +Fix two (unrelated) JIT bugs manifesting in the re module: + +- green fields are broken and were thus disabled, plus their usage removed from + the _sre implementation + +- in rare "trace is too long" situations, the JIT could break behaviour + arbitrarily. diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -232,7 +232,9 @@ "different from the 'ffi.h' file seen at compile-time)") def py_invoke(self, ll_res, ll_args): + key_pycode = self.key_pycode jitdriver1.jit_merge_point(callback=self, + key_pycode=key_pycode, ll_res=ll_res, ll_args=ll_args) self.do_invoke(ll_res, ll_args) @@ -294,7 +296,7 @@ return 'cffi_callback ' + key_pycode.get_repr() jitdriver1 = jit.JitDriver(name='cffi_callback', - greens=['callback.key_pycode'], + greens=['key_pycode'], reds=['ll_res', 'll_args', 'callback'], get_printable_location=get_printable_location1) diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -77,15 +77,15 @@ w_import = space.getattr(w_builtin, space.newtext("__import__")) return space.call_function(w_import, space.newtext("re")) -def matchcontext(space, ctx): +def matchcontext(space, ctx, pattern): try: - return rsre_core.match_context(ctx) + return rsre_core.match_context(ctx, pattern) except rsre_core.Error as e: raise OperationError(space.w_RuntimeError, space.newtext(e.msg)) -def searchcontext(space, ctx): +def searchcontext(space, ctx, pattern): try: - return rsre_core.search_context(ctx) + return rsre_core.search_context(ctx, pattern) except rsre_core.Error as e: raise OperationError(space.w_RuntimeError, space.newtext(e.msg)) @@ -114,7 +114,7 @@ pos = len(unicodestr) if endpos > len(unicodestr): endpos = len(unicodestr) - return rsre_core.UnicodeMatchContext(self.code, unicodestr, + return rsre_core.UnicodeMatchContext(unicodestr, pos, endpos, self.flags) elif space.isinstance_w(w_string, space.w_bytes): str = space.bytes_w(w_string) @@ -122,7 +122,7 @@ pos = len(str) if endpos > len(str): endpos = len(str) - return rsre_core.StrMatchContext(self.code, str, + return rsre_core.StrMatchContext(str, pos, endpos, self.flags) else: buf = space.readbuf_w(w_string) @@ -132,7 +132,7 @@ pos = size if endpos > size: endpos = size - return rsre_core.BufMatchContext(self.code, buf, + return rsre_core.BufMatchContext(buf, pos, endpos, self.flags) def getmatch(self, ctx, found): @@ -144,12 +144,12 @@ @unwrap_spec(pos=int, endpos=int) def match_w(self, w_string, pos=0, endpos=sys.maxint): ctx = self.make_ctx(w_string, pos, endpos) - return self.getmatch(ctx, matchcontext(self.space, ctx)) + return self.getmatch(ctx, matchcontext(self.space, ctx, self.code)) @unwrap_spec(pos=int, endpos=int) def search_w(self, w_string, pos=0, endpos=sys.maxint): ctx = self.make_ctx(w_string, pos, endpos) - return self.getmatch(ctx, searchcontext(self.space, ctx)) + return self.getmatch(ctx, searchcontext(self.space, ctx, self.code)) @unwrap_spec(pos=int, endpos=int) def findall_w(self, w_string, pos=0, endpos=sys.maxint): @@ -157,7 +157,7 @@ matchlist_w = [] ctx = self.make_ctx(w_string, pos, endpos) while ctx.match_start <= ctx.end: - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, self.code): break num_groups = self.num_groups w_emptystr = space.newtext("") @@ -182,7 +182,7 @@ # this also works as the implementation of the undocumented # scanner() method. ctx = self.make_ctx(w_string, pos, endpos) - scanner = W_SRE_Scanner(self, ctx) + scanner = W_SRE_Scanner(self, ctx, self.code) return scanner @unwrap_spec(maxsplit=int) @@ -193,7 +193,7 @@ last = 0 ctx = self.make_ctx(w_string) while not maxsplit or n < maxsplit: - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, self.code): break if ctx.match_start == ctx.match_end: # zero-width match if ctx.match_start == ctx.end: # or end of string @@ -274,8 +274,8 @@ else: sublist_w = [] n = last_pos = 0 + pattern = self.code while not count or n < count: - pattern = ctx.pattern sub_jitdriver.jit_merge_point( self=self, use_builder=use_builder, @@ -292,7 +292,7 @@ n=n, last_pos=last_pos, sublist_w=sublist_w ) space = self.space - if not searchcontext(space, ctx): + if not searchcontext(space, ctx, pattern): break if last_pos < ctx.match_start: _sub_append_slice( @@ -388,7 +388,11 @@ srepat.space = space srepat.w_pattern = w_pattern # the original uncompiled pattern srepat.flags = flags - srepat.code = code + # note: we assume that the app-level is caching SRE_Pattern objects, + # so that we don't need to do it here. Creating new SRE_Pattern + # objects all the time would be bad for the JIT, which relies on the + # identity of the CompiledPattern() object. + srepat.code = rsre_core.CompiledPattern(code) srepat.num_groups = groups srepat.w_groupindex = w_groupindex srepat.w_indexgroup = w_indexgroup @@ -611,10 +615,11 @@ # Our version is also directly iterable, to make finditer() easier. class W_SRE_Scanner(W_Root): - def __init__(self, pattern, ctx): + def __init__(self, pattern, ctx, code): self.space = pattern.space self.srepat = pattern self.ctx = ctx + self.code = code # 'self.ctx' is always a fresh context in which no searching # or matching succeeded so far. @@ -624,19 +629,19 @@ def next_w(self): if self.ctx.match_start > self.ctx.end: raise OperationError(self.space.w_StopIteration, self.space.w_None) - if not searchcontext(self.space, self.ctx): + if not searchcontext(self.space, self.ctx, self.code): raise OperationError(self.space.w_StopIteration, self.space.w_None) return self.getmatch(True) def match_w(self): if self.ctx.match_start > self.ctx.end: return self.space.w_None - return self.getmatch(matchcontext(self.space, self.ctx)) + return self.getmatch(matchcontext(self.space, self.ctx, self.code)) def search_w(self): if self.ctx.match_start > self.ctx.end: return self.space.w_None - return self.getmatch(searchcontext(self.space, self.ctx)) + return self.getmatch(searchcontext(self.space, self.ctx, self.code)) def getmatch(self, found): if found: diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -701,6 +701,9 @@ def length(self): return self.trace._count - len(self.trace.inputargs) + def trace_tag_overflow(self): + return self.trace.tag_overflow + def get_trace_position(self): return self.trace.cut_point() diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py --- a/rpython/jit/metainterp/opencoder.py +++ b/rpython/jit/metainterp/opencoder.py @@ -49,13 +49,6 @@ way up to lltype.Signed for indexes everywhere """ -def frontend_tag_overflow(): - # Minor abstraction leak: raise directly the right exception - # expected by the rest of the machinery - from rpython.jit.metainterp import history - from rpython.rlib.jit import Counters - raise history.SwitchToBlackhole(Counters.ABORT_TOO_LONG) - class BaseTrace(object): pass @@ -293,6 +286,7 @@ self._start = len(inputargs) self._pos = self._start self.inputargs = inputargs + self.tag_overflow = False def append(self, v): model = get_model(self) @@ -300,12 +294,14 @@ # grow by 2X self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops) if not model.MIN_VALUE <= v <= model.MAX_VALUE: - raise frontend_tag_overflow() + v = 0 # broken value, but that's fine, tracing will stop soon + self.tag_overflow = True self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v) self._pos += 1 - def done(self): + def tracing_done(self): from rpython.rlib.debug import debug_start, debug_stop, debug_print + assert not self.tag_overflow self._bigints_dict = {} self._refs_dict = llhelper.new_ref_dict_3() @@ -317,8 +313,6 @@ debug_print(" ref consts: " + str(self._consts_ptr) + " " + str(len(self._refs))) debug_print(" descrs: " + str(len(self._descrs))) debug_stop("jit-trace-done") - return 0 # completely different than TraceIter.done, but we have to - # share the base class def length(self): return self._pos @@ -379,6 +373,7 @@ def record_op(self, opnum, argboxes, descr=None): pos = self._index + old_pos = self._pos self.append(opnum) expected_arity = oparity[opnum] if expected_arity == -1: @@ -397,6 +392,10 @@ self._count += 1 if opclasses[opnum].type != 'v': self._index += 1 + if self.tag_overflow: + # potentially a broken op is left behind + # clean it up + self._pos = old_pos return pos def _encode_descr(self, descr): @@ -424,10 +423,11 @@ vref_array = self._list_of_boxes(vref_boxes) s = TopSnapshot(combine_uint(jitcode.index, pc), array, vable_array, vref_array) - assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 # guards have no descr self._snapshots.append(s) - self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) + if not self.tag_overflow: # otherwise we're broken anyway + assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 + self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) return s def create_empty_top_snapshot(self, vable_boxes, vref_boxes): @@ -436,10 +436,11 @@ vref_array = self._list_of_boxes(vref_boxes) s = TopSnapshot(combine_uint(2**16 - 1, 0), [], vable_array, vref_array) - assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 # guards have no descr self._snapshots.append(s) - self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) + if not self.tag_overflow: # otherwise we're broken anyway + assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0 + self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1) return s def create_snapshot(self, jitcode, pc, frame, flag): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2384,9 +2384,9 @@ def blackhole_if_trace_too_long(self): warmrunnerstate = self.jitdriver_sd.warmstate - if self.history.length() > warmrunnerstate.trace_limit: + if (self.history.length() > warmrunnerstate.trace_limit or + self.history.trace_tag_overflow()): jd_sd, greenkey_of_huge_function = self.find_biggest_function() - self.history.trace.done() self.staticdata.stats.record_aborted(greenkey_of_huge_function) self.portal_trace_positions = None if greenkey_of_huge_function is not None: @@ -2689,7 +2689,9 @@ try_disabling_unroll=False, exported_state=None): num_green_args = self.jitdriver_sd.num_green_args greenkey = original_boxes[:num_green_args] - self.history.trace.done() + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) + self.history.trace.tracing_done() if not self.partial_trace: ptoken = self.get_procedure_token(greenkey) if ptoken is not None and ptoken.target_tokens is not None: @@ -2742,7 +2744,9 @@ self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None, descr=target_jitcell_token) self.history.ends_with_jump = True - self.history.trace.done() + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) + self.history.trace.tracing_done() try: target_token = compile.compile_trace(self, self.resumekey, live_arg_boxes[num_green_args:]) @@ -2776,7 +2780,9 @@ assert False # FIXME: can we call compile_trace? self.history.record(rop.FINISH, exits, None, descr=token) - self.history.trace.done() + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) + self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, exits) if target_token is not token: compile.giveup() @@ -2802,7 +2808,9 @@ sd = self.staticdata token = sd.exit_frame_with_exception_descr_ref self.history.record(rop.FINISH, [valuebox], None, descr=token) - self.history.trace.done() + if self.history.trace_tag_overflow(): + raise SwitchToBlackhole(Counters.ABORT_TOO_LONG) + self.history.trace.tracing_done() target_token = compile.compile_trace(self, self.resumekey, [valuebox]) if target_token is not token: compile.giveup() diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -4661,3 +4661,36 @@ f() # finishes self.meta_interp(f, []) + + def test_trace_too_long_bug(self): + driver = JitDriver(greens=[], reds=['i']) + @unroll_safe + def match(s): + l = len(s) + p = 0 + for i in range(2500): # produces too long trace + c = s[p] + if c != 'a': + return False + p += 1 + if p >= l: + return True + c = s[p] + if c != '\n': + p += 1 + if p >= l: + return True + else: + return False + return True + + def f(i): + while i > 0: + driver.jit_merge_point(i=i) + match('a' * (500 * i)) + i -= 1 + return i + + res = self.meta_interp(f, [10]) + assert res == f(10) + diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py --- a/rpython/jit/metainterp/test/test_greenfield.py +++ b/rpython/jit/metainterp/test/test_greenfield.py @@ -1,6 +1,17 @@ +import pytest from rpython.jit.metainterp.test.support import LLJitMixin from rpython.rlib.jit import JitDriver, assert_green +pytest.skip("this feature is disabled at the moment!") + +# note why it is disabled: before d721da4573ad +# there was a failing assert when inlining python -> sre -> python: +# https://bitbucket.org/pypy/pypy/issues/2775/ +# this shows, that the interaction of greenfields and virtualizables is broken, +# because greenfields use MetaInterp.virtualizable_boxes, which confuses +# MetaInterp._nonstandard_virtualizable somehow (and makes no sense +# conceptually anyway). to fix greenfields, the two mechanisms would have to be +# disentangled. class GreenFieldsTests: diff --git a/rpython/jit/metainterp/test/test_opencoder.py b/rpython/jit/metainterp/test/test_opencoder.py --- a/rpython/jit/metainterp/test/test_opencoder.py +++ b/rpython/jit/metainterp/test/test_opencoder.py @@ -209,5 +209,8 @@ def test_tag_overflow(self): t = Trace([], metainterp_sd) i0 = FakeOp(100000) - py.test.raises(SwitchToBlackhole, t.record_op, rop.FINISH, [i0]) - assert t.unpack() == ([], []) + # if we overflow, we can keep recording + for i in range(10): + t.record_op(rop.FINISH, [i0]) + assert t.unpack() == ([], []) + assert t.tag_overflow diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -653,6 +653,9 @@ self._make_extregistryentries() assert get_jitcell_at is None, "get_jitcell_at no longer used" assert set_jitcell_at is None, "set_jitcell_at no longer used" + for green in self.greens: + if "." in green: + raise ValueError("green fields are buggy! if you need them fixed, please talk to us") self.get_printable_location = get_printable_location self.get_location = get_location self.has_unique_id = (get_unique_id is not None) diff --git a/rpython/rlib/rsre/rpy/_sre.py b/rpython/rlib/rsre/rpy/_sre.py --- a/rpython/rlib/rsre/rpy/_sre.py +++ b/rpython/rlib/rsre/rpy/_sre.py @@ -1,4 +1,4 @@ -from rpython.rlib.rsre import rsre_char +from rpython.rlib.rsre import rsre_char, rsre_core from rpython.rlib.rarithmetic import intmask VERSION = "2.7.6" @@ -12,7 +12,7 @@ pass def compile(pattern, flags, code, *args): - raise GotIt([intmask(i) for i in code], flags, args) + raise GotIt(rsre_core.CompiledPattern([intmask(i) for i in code]), flags, args) def get_code(regexp, flags=0, allargs=False): diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py --- a/rpython/rlib/rsre/rsre_char.py +++ b/rpython/rlib/rsre/rsre_char.py @@ -152,17 +152,16 @@ ##### Charset evaluation @jit.unroll_safe -def check_charset(ctx, ppos, char_code): +def check_charset(ctx, pattern, ppos, char_code): """Checks whether a character matches set of arbitrary length. The set starts at pattern[ppos].""" negated = False result = False - pattern = ctx.pattern while True: - opcode = pattern[ppos] + opcode = pattern.pattern[ppos] for i, function in set_dispatch_unroll: if opcode == i: - newresult, ppos = function(ctx, ppos, char_code) + newresult, ppos = function(ctx, pattern, ppos, char_code) result |= newresult break else: @@ -177,50 +176,44 @@ return not result return result -def set_literal(ctx, index, char_code): +def set_literal(ctx, pattern, index, char_code): # - pat = ctx.pattern - match = pat[index+1] == char_code + match = pattern.pattern[index+1] == char_code return match, index + 2 -def set_category(ctx, index, char_code): +def set_category(ctx, pattern, index, char_code): # - pat = ctx.pattern - match = category_dispatch(pat[index+1], char_code) + match = category_dispatch(pattern.pattern[index+1], char_code) return match, index + 2 -def set_charset(ctx, index, char_code): +def set_charset(ctx, pattern, index, char_code): # (16 bits per code word) - pat = ctx.pattern if CODESIZE == 2: match = char_code < 256 and \ - (pat[index+1+(char_code >> 4)] & (1 << (char_code & 15))) + (pattern.pattern[index+1+(char_code >> 4)] & (1 << (char_code & 15))) return match, index + 17 # skip bitmap else: match = char_code < 256 and \ - (pat[index+1+(char_code >> 5)] & (1 << (char_code & 31))) + (pattern.pattern[index+1+(char_code >> 5)] & (1 << (char_code & 31))) return match, index + 9 # skip bitmap -def set_range(ctx, index, char_code): +def set_range(ctx, pattern, index, char_code): # - pat = ctx.pattern - match = int_between(pat[index+1], char_code, pat[index+2] + 1) + match = int_between(pattern.pattern[index+1], char_code, pattern.pattern[index+2] + 1) return match, index + 3 -def set_range_ignore(ctx, index, char_code): +def set_range_ignore(ctx, pattern, index, char_code): # # the char_code is already lower cased - pat = ctx.pattern - lower = pat[index + 1] - upper = pat[index + 2] + lower = pattern.pattern[index + 1] + upper = pattern.pattern[index + 2] match1 = int_between(lower, char_code, upper + 1) match2 = int_between(lower, getupper(char_code, ctx.flags), upper + 1) return match1 | match2, index + 3 -def set_bigcharset(ctx, index, char_code): +def set_bigcharset(ctx, pattern, index, char_code): # <256 blockindices> - pat = ctx.pattern - count = pat[index+1] + count = pattern.pattern[index+1] index += 2 if CODESIZE == 2: @@ -238,7 +231,7 @@ return False, index shift = 5 - block = pat[index + (char_code >> (shift + 5))] + block = pattern.pattern[index + (char_code >> (shift + 5))] block_shift = char_code >> 5 if BIG_ENDIAN: @@ -247,23 +240,22 @@ block = (block >> block_shift) & 0xFF index += 256 / CODESIZE - block_value = pat[index+(block * (32 / CODESIZE) + block_value = pattern.pattern[index+(block * (32 / CODESIZE) + ((char_code & 255) >> shift))] match = (block_value & (1 << (char_code & ((8 * CODESIZE) - 1)))) index += count * (32 / CODESIZE) # skip blocks return match, index -def set_unicode_general_category(ctx, index, char_code): +def set_unicode_general_category(ctx, pattern, index, char_code): # Unicode "General category property code" (not used by Python). - # A general category is two letters. 'pat[index+1]' contains both + # A general category is two letters. 'pattern.pattern[index+1]' contains both # the first character, and the second character shifted by 8. # http://en.wikipedia.org/wiki/Unicode_character_property#General_Category # Also supports single-character categories, if the second character is 0. # Negative matches are triggered by bit number 7. assert unicodedb is not None cat = unicodedb.category(char_code) - pat = ctx.pattern - category_code = pat[index + 1] + category_code = pattern.pattern[index + 1] first_character = category_code & 0x7F second_character = (category_code >> 8) & 0x7F negative_match = category_code & 0x80 diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -83,35 +83,19 @@ def __init__(self, msg): self.msg = msg -class AbstractMatchContext(object): - """Abstract base class""" - _immutable_fields_ = ['pattern[*]', 'flags', 'end'] - match_start = 0 - match_end = 0 - match_marks = None - match_marks_flat = None - fullmatch_only = False - def __init__(self, pattern, match_start, end, flags): - # 'match_start' and 'end' must be known to be non-negative - # and they must not be more than len(string). - check_nonneg(match_start) - check_nonneg(end) +class CompiledPattern(object): + _immutable_fields_ = ['pattern[*]'] + + def __init__(self, pattern): self.pattern = pattern - self.match_start = match_start - self.end = end - self.flags = flags # check we don't get the old value of MAXREPEAT # during the untranslated tests if not we_are_translated(): assert 65535 not in pattern - def reset(self, start): - self.match_start = start - self.match_marks = None - self.match_marks_flat = None - def pat(self, index): + jit.promote(self) check_nonneg(index) result = self.pattern[index] # Check that we only return non-negative integers from this helper. @@ -121,6 +105,29 @@ assert result >= 0 return result +class AbstractMatchContext(object): + """Abstract base class""" + _immutable_fields_ = ['flags', 'end'] + match_start = 0 + match_end = 0 + match_marks = None + match_marks_flat = None + fullmatch_only = False + + def __init__(self, match_start, end, flags): + # 'match_start' and 'end' must be known to be non-negative + # and they must not be more than len(string). + check_nonneg(match_start) + check_nonneg(end) + self.match_start = match_start + self.end = end + self.flags = flags + + def reset(self, start): + self.match_start = start + self.match_marks = None + self.match_marks_flat = None + @not_rpython def str(self, index): """Must be overridden in a concrete subclass. @@ -183,8 +190,8 @@ _immutable_fields_ = ["_buffer"] - def __init__(self, pattern, buf, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, buf, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._buffer = buf def str(self, index): @@ -196,7 +203,7 @@ return rsre_char.getlower(c, self.flags) def fresh_copy(self, start): - return BufMatchContext(self.pattern, self._buffer, start, + return BufMatchContext(self._buffer, start, self.end, self.flags) class StrMatchContext(AbstractMatchContext): @@ -204,8 +211,8 @@ _immutable_fields_ = ["_string"] - def __init__(self, pattern, string, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, string, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._string = string if not we_are_translated() and isinstance(string, unicode): self.flags |= rsre_char.SRE_FLAG_UNICODE # for rsre_re.py @@ -219,7 +226,7 @@ return rsre_char.getlower(c, self.flags) def fresh_copy(self, start): - return StrMatchContext(self.pattern, self._string, start, + return StrMatchContext(self._string, start, self.end, self.flags) class UnicodeMatchContext(AbstractMatchContext): @@ -227,8 +234,8 @@ _immutable_fields_ = ["_unicodestr"] - def __init__(self, pattern, unicodestr, match_start, end, flags): - AbstractMatchContext.__init__(self, pattern, match_start, end, flags) + def __init__(self, unicodestr, match_start, end, flags): + AbstractMatchContext.__init__(self, match_start, end, flags) self._unicodestr = unicodestr def str(self, index): @@ -240,7 +247,7 @@ return rsre_char.getlower(c, self.flags) def fresh_copy(self, start): - return UnicodeMatchContext(self.pattern, self._unicodestr, start, + return UnicodeMatchContext(self._unicodestr, start, self.end, self.flags) # ____________________________________________________________ @@ -265,16 +272,16 @@ class MatchResult(object): subresult = None - def move_to_next_result(self, ctx): + def move_to_next_result(self, ctx, pattern): # returns either 'self' or None result = self.subresult if result is None: return - if result.move_to_next_result(ctx): + if result.move_to_next_result(ctx, pattern): return self - return self.find_next_result(ctx) + return self.find_next_result(ctx, pattern) - def find_next_result(self, ctx): + def find_next_result(self, ctx, pattern): raise NotImplementedError MATCHED_OK = MatchResult() @@ -287,11 +294,11 @@ self.start_marks = marks @jit.unroll_safe - def find_first_result(self, ctx): + def find_first_result(self, ctx, pattern): ppos = jit.hint(self.ppos, promote=True) - while ctx.pat(ppos): - result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks) - ppos += ctx.pat(ppos) + while pattern.pat(ppos): + result = sre_match(ctx, pattern, ppos + 1, self.start_ptr, self.start_marks) + ppos += pattern.pat(ppos) if result is not None: self.subresult = result self.ppos = ppos @@ -300,7 +307,7 @@ class RepeatOneMatchResult(MatchResult): install_jitdriver('RepeatOne', - greens=['nextppos', 'ctx.pattern'], + greens=['nextppos', 'pattern'], reds=['ptr', 'self', 'ctx'], debugprint=(1, 0)) # indices in 'greens' @@ -310,13 +317,14 @@ self.start_ptr = ptr self.start_marks = marks - def find_first_result(self, ctx): + def find_first_result(self, ctx, pattern): ptr = self.start_ptr nextppos = self.nextppos while ptr >= self.minptr: ctx.jitdriver_RepeatOne.jit_merge_point( - self=self, ptr=ptr, ctx=ctx, nextppos=nextppos) - result = sre_match(ctx, nextppos, ptr, self.start_marks) + self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, + pattern=pattern) + result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks) ptr -= 1 if result is not None: self.subresult = result @@ -327,7 +335,7 @@ class MinRepeatOneMatchResult(MatchResult): install_jitdriver('MinRepeatOne', - greens=['nextppos', 'ppos3', 'ctx.pattern'], + greens=['nextppos', 'ppos3', 'pattern'], reds=['ptr', 'self', 'ctx'], debugprint=(2, 0)) # indices in 'greens' @@ -338,39 +346,40 @@ self.start_ptr = ptr self.start_marks = marks - def find_first_result(self, ctx): + def find_first_result(self, ctx, pattern): ptr = self.start_ptr nextppos = self.nextppos ppos3 = self.ppos3 while ptr <= self.maxptr: ctx.jitdriver_MinRepeatOne.jit_merge_point( - self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3) - result = sre_match(ctx, nextppos, ptr, self.start_marks) + self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3, + pattern=pattern) + result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks) if result is not None: self.subresult = result self.start_ptr = ptr return self - if not self.next_char_ok(ctx, ptr, ppos3): + if not self.next_char_ok(ctx, pattern, ptr, ppos3): break ptr += 1 - def find_next_result(self, ctx): + def find_next_result(self, ctx, pattern): ptr = self.start_ptr - if not self.next_char_ok(ctx, ptr, self.ppos3): + if not self.next_char_ok(ctx, pattern, ptr, self.ppos3): return self.start_ptr = ptr + 1 - return self.find_first_result(ctx) + return self.find_first_result(ctx, pattern) - def next_char_ok(self, ctx, ptr, ppos): + def next_char_ok(self, ctx, pattern, ptr, ppos): if ptr == ctx.end: return False - op = ctx.pat(ppos) + op = pattern.pat(ppos) for op1, checkerfn in unroll_char_checker: if op1 == op: - return checkerfn(ctx, ptr, ppos) + return checkerfn(ctx, pattern, ptr, ppos) # obscure case: it should be a single char pattern, but isn't # one of the opcodes in unroll_char_checker (see test_ext_opcode) - return sre_match(ctx, ppos, ptr, self.start_marks) is not None + return sre_match(ctx, pattern, ppos, ptr, self.start_marks) is not None class AbstractUntilMatchResult(MatchResult): @@ -391,17 +400,17 @@ class MaxUntilMatchResult(AbstractUntilMatchResult): install_jitdriver('MaxUntil', - greens=['ppos', 'tailppos', 'match_more', 'ctx.pattern'], + greens=['ppos', 'tailppos', 'match_more', 'pattern'], reds=['ptr', 'marks', 'self', 'ctx'], debugprint=(3, 0, 2)) - def find_first_result(self, ctx): - return self.search_next(ctx, match_more=True) + def find_first_result(self, ctx, pattern): + return self.search_next(ctx, pattern, match_more=True) - def find_next_result(self, ctx): - return self.search_next(ctx, match_more=False) + def find_next_result(self, ctx, pattern): + return self.search_next(ctx, pattern, match_more=False) - def search_next(self, ctx, match_more): + def search_next(self, ctx, pattern, match_more): ppos = self.ppos tailppos = self.tailppos ptr = self.cur_ptr @@ -409,12 +418,13 @@ while True: ctx.jitdriver_MaxUntil.jit_merge_point( ppos=ppos, tailppos=tailppos, match_more=match_more, - ptr=ptr, marks=marks, self=self, ctx=ctx) + ptr=ptr, marks=marks, self=self, ctx=ctx, + pattern=pattern) if match_more: - max = ctx.pat(ppos+2) + max = pattern.pat(ppos+2) if max == rsre_char.MAXREPEAT or self.num_pending < max: # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + enum = sre_match(ctx, pattern, ppos + 3, ptr, marks) else: enum = None # 'max' reached, no more matches else: @@ -425,9 +435,9 @@ self.num_pending -= 1 ptr = p.ptr marks = p.marks - enum = p.enum.move_to_next_result(ctx) + enum = p.enum.move_to_next_result(ctx, pattern) # - min = ctx.pat(ppos+1) + min = pattern.pat(ppos+1) if enum is not None: # matched one more 'item'. record it and continue. last_match_length = ctx.match_end - ptr @@ -447,7 +457,7 @@ # 'item' no longer matches. if self.num_pending >= min: # try to match 'tail' if we have enough 'item' - result = sre_match(ctx, tailppos, ptr, marks) + result = sre_match(ctx, pattern, tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -457,23 +467,23 @@ class MinUntilMatchResult(AbstractUntilMatchResult): - def find_first_result(self, ctx): - return self.search_next(ctx, resume=False) + def find_first_result(self, ctx, pattern): + return self.search_next(ctx, pattern, resume=False) - def find_next_result(self, ctx): - return self.search_next(ctx, resume=True) + def find_next_result(self, ctx, pattern): + return self.search_next(ctx, pattern, resume=True) - def search_next(self, ctx, resume): + def search_next(self, ctx, pattern, resume): # XXX missing jit support here ppos = self.ppos - min = ctx.pat(ppos+1) - max = ctx.pat(ppos+2) + min = pattern.pat(ppos+1) + max = pattern.pat(ppos+2) ptr = self.cur_ptr marks = self.cur_marks while True: # try to match 'tail' if we have enough 'item' if not resume and self.num_pending >= min: - result = sre_match(ctx, self.tailppos, ptr, marks) + result = sre_match(ctx, pattern, self.tailppos, ptr, marks) if result is not None: self.subresult = result self.cur_ptr = ptr @@ -483,12 +493,12 @@ if max == rsre_char.MAXREPEAT or self.num_pending < max: # try to match one more 'item' - enum = sre_match(ctx, ppos + 3, ptr, marks) + enum = sre_match(ctx, pattern, ppos + 3, ptr, marks) # # zero-width match protection if self.num_pending >= min: while enum is not None and ptr == ctx.match_end: - enum = enum.move_to_next_result(ctx) + enum = enum.move_to_next_result(ctx, pattern) else: enum = None # 'max' reached, no more matches @@ -502,7 +512,7 @@ self.num_pending -= 1 ptr = p.ptr marks = p.marks - enum = p.enum.move_to_next_result(ctx) + enum = p.enum.move_to_next_result(ctx, pattern) # matched one more 'item'. record it and continue self.pending = Pending(ptr, marks, enum, self.pending) @@ -514,13 +524,13 @@ @specializectx @jit.unroll_safe -def sre_match(ctx, ppos, ptr, marks): +def sre_match(ctx, pattern, ppos, ptr, marks): """Returns either None or a MatchResult object. Usually we only need the first result, but there is the case of REPEAT...UNTIL where we need all results; in that case we use the method move_to_next_result() of the MatchResult.""" while True: - op = ctx.pat(ppos) + op = pattern.pat(ppos) ppos += 1 #jit.jit_debug("sre_match", op, ppos, ptr) @@ -563,33 +573,33 @@ elif op == OPCODE_ASSERT: # assert subpattern # <0=skip> <1=back> - ptr1 = ptr - ctx.pat(ppos+1) + ptr1 = ptr - pattern.pat(ppos+1) saved = ctx.fullmatch_only ctx.fullmatch_only = False - stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None + stop = ptr1 < 0 or sre_match(ctx, pattern, ppos + 2, ptr1, marks) is None ctx.fullmatch_only = saved if stop: return marks = ctx.match_marks - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_ASSERT_NOT: # assert not subpattern # <0=skip> <1=back> - ptr1 = ptr - ctx.pat(ppos+1) + ptr1 = ptr - pattern.pat(ppos+1) saved = ctx.fullmatch_only ctx.fullmatch_only = False - stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks) + stop = (ptr1 >= 0 and sre_match(ctx, pattern, ppos + 2, ptr1, marks) is not None) ctx.fullmatch_only = saved if stop: return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_AT: # match at given position (e.g. at beginning, at boundary, etc.) # - if not sre_at(ctx, ctx.pat(ppos), ptr): + if not sre_at(ctx, pattern.pat(ppos), ptr): return ppos += 1 @@ -597,14 +607,14 @@ # alternation # <0=skip> code ... result = BranchMatchResult(ppos, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) elif op == OPCODE_CATEGORY: # seems to be never produced, but used by some tests from # pypy/module/_sre/test # if (ptr == ctx.end or - not rsre_char.category_dispatch(ctx.pat(ppos), ctx.str(ptr))): + not rsre_char.category_dispatch(pattern.pat(ppos), ctx.str(ptr))): return ptr += 1 ppos += 1 @@ -612,7 +622,7 @@ elif op == OPCODE_GROUPREF: # match backreference # - startptr, length = get_group_ref(marks, ctx.pat(ppos)) + startptr, length = get_group_ref(marks, pattern.pat(ppos)) if length < 0: return # group was not previously defined if not match_repeated(ctx, ptr, startptr, length): @@ -623,7 +633,7 @@ elif op == OPCODE_GROUPREF_IGNORE: # match backreference # - startptr, length = get_group_ref(marks, ctx.pat(ppos)) + startptr, length = get_group_ref(marks, pattern.pat(ppos)) if length < 0: return # group was not previously defined if not match_repeated_ignore(ctx, ptr, startptr, length): @@ -634,44 +644,44 @@ elif op == OPCODE_GROUPREF_EXISTS: # conditional match depending on the existence of a group # codeyes codeno ... - _, length = get_group_ref(marks, ctx.pat(ppos)) + _, length = get_group_ref(marks, pattern.pat(ppos)) if length >= 0: ppos += 2 # jump to 'codeyes' else: - ppos += ctx.pat(ppos+1) # jump to 'codeno' + ppos += pattern.pat(ppos+1) # jump to 'codeno' elif op == OPCODE_IN: # match set member (or non_member) # - if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1, + if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1, ctx.str(ptr)): return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) ptr += 1 elif op == OPCODE_IN_IGNORE: # match set member (or non_member), ignoring case # - if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1, + if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1, ctx.lowstr(ptr)): return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) ptr += 1 elif op == OPCODE_INFO: # optimization info block # <0=skip> <1=flags> <2=min> ... - if (ctx.end - ptr) < ctx.pat(ppos+2): + if (ctx.end - ptr) < pattern.pat(ppos+2): return - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_JUMP: - ppos += ctx.pat(ppos) + ppos += pattern.pat(ppos) elif op == OPCODE_LITERAL: # match literal string # - if ptr >= ctx.end or ctx.str(ptr) != ctx.pat(ppos): + if ptr >= ctx.end or ctx.str(ptr) != pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -679,7 +689,7 @@ elif op == OPCODE_LITERAL_IGNORE: # match literal string, ignoring case # - if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.pat(ppos): + if ptr >= ctx.end or ctx.lowstr(ptr) != pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -687,14 +697,14 @@ elif op == OPCODE_MARK: # set mark # - gid = ctx.pat(ppos) + gid = pattern.pat(ppos) marks = Mark(gid, ptr, marks) ppos += 1 elif op == OPCODE_NOT_LITERAL: # match if it's not a literal string # - if ptr >= ctx.end or ctx.str(ptr) == ctx.pat(ppos): + if ptr >= ctx.end or ctx.str(ptr) == pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -702,7 +712,7 @@ elif op == OPCODE_NOT_LITERAL_IGNORE: # match if it's not a literal string, ignoring case # - if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos): + if ptr >= ctx.end or ctx.lowstr(ptr) == pattern.pat(ppos): return ppos += 1 ptr += 1 @@ -715,22 +725,22 @@ # decode the later UNTIL operator to see if it is actually # a MAX_UNTIL or MIN_UNTIL - untilppos = ppos + ctx.pat(ppos) + untilppos = ppos + pattern.pat(ppos) tailppos = untilppos + 1 - op = ctx.pat(untilppos) + op = pattern.pat(untilppos) if op == OPCODE_MAX_UNTIL: # the hard case: we have to match as many repetitions as # possible, followed by the 'tail'. we do this by # remembering each state for each possible number of # 'item' matching. result = MaxUntilMatchResult(ppos, tailppos, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) elif op == OPCODE_MIN_UNTIL: # first try to match the 'tail', and if it fails, try # to match one more 'item' and try again result = MinUntilMatchResult(ppos, tailppos, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) else: raise Error("missing UNTIL after REPEAT") @@ -743,17 +753,18 @@ # use the MAX_REPEAT operator. # <1=min> <2=max> item tail start = ptr - minptr = start + ctx.pat(ppos+1) + minptr = start + pattern.pat(ppos+1) if minptr > ctx.end: return # cannot match - ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2), + ptr = find_repetition_end(ctx, pattern, ppos+3, start, + pattern.pat(ppos+2), marks) # when we arrive here, ptr points to the tail of the target # string. check if the rest of the pattern matches, # and backtrack if not. - nextppos = ppos + ctx.pat(ppos) + nextppos = ppos + pattern.pat(ppos) result = RepeatOneMatchResult(nextppos, minptr, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) elif op == OPCODE_MIN_REPEAT_ONE: # match repeated sequence (minimizing regexp). @@ -763,26 +774,26 @@ # use the MIN_REPEAT operator. # <1=min> <2=max> item tail start = ptr - min = ctx.pat(ppos+1) + min = pattern.pat(ppos+1) if min > 0: minptr = ptr + min if minptr > ctx.end: return # cannot match # count using pattern min as the maximum - ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks) + ptr = find_repetition_end(ctx, pattern, ppos+3, ptr, min, marks) if ptr < minptr: return # did not match minimum number of times maxptr = ctx.end - max = ctx.pat(ppos+2) + max = pattern.pat(ppos+2) if max != rsre_char.MAXREPEAT: maxptr1 = start + max if maxptr1 <= maxptr: maxptr = maxptr1 - nextppos = ppos + ctx.pat(ppos) + nextppos = ppos + pattern.pat(ppos) result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr, ptr, marks) - return result.find_first_result(ctx) + return result.find_first_result(ctx, pattern) else: raise Error("bad pattern code %d" % op) @@ -816,7 +827,7 @@ return True @specializectx -def find_repetition_end(ctx, ppos, ptr, maxcount, marks): +def find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks): end = ctx.end ptrp1 = ptr + 1 # First get rid of the cases where we don't have room for any match. @@ -826,16 +837,16 @@ # The idea is to be fast for cases like re.search("b+"), where we expect # the common case to be a non-match. It's much faster with the JIT to # have the non-match inlined here rather than detect it in the fre() call. - op = ctx.pat(ppos) + op = pattern.pat(ppos) for op1, checkerfn in unroll_char_checker: if op1 == op: - if checkerfn(ctx, ptr, ppos): + if checkerfn(ctx, pattern, ptr, ppos): break return ptr else: # obscure case: it should be a single char pattern, but isn't # one of the opcodes in unroll_char_checker (see test_ext_opcode) - return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks) + return general_find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks) # It matches at least once. If maxcount == 1 (relatively common), # then we are done. if maxcount == 1: @@ -846,14 +857,14 @@ end1 = ptr + maxcount if end1 <= end: end = end1 - op = ctx.pat(ppos) + op = pattern.pat(ppos) for op1, fre in unroll_fre_checker: if op1 == op: - return fre(ctx, ptrp1, end, ppos) + return fre(ctx, pattern, ptrp1, end, ppos) raise Error("rsre.find_repetition_end[%d]" % op) @specializectx -def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks): +def general_find_repetition_end(ctx, patern, ppos, ptr, maxcount, marks): # moved into its own JIT-opaque function end = ctx.end if maxcount != rsre_char.MAXREPEAT: @@ -861,63 +872,65 @@ end1 = ptr + maxcount if end1 <= end: end = end1 - while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None: + while ptr < end and sre_match(ctx, patern, ppos, ptr, marks) is not None: ptr += 1 return ptr @specializectx -def match_ANY(ctx, ptr, ppos): # dot wildcard. +def match_ANY(ctx, pattern, ptr, ppos): # dot wildcard. return not rsre_char.is_linebreak(ctx.str(ptr)) -def match_ANY_ALL(ctx, ptr, ppos): +def match_ANY_ALL(ctx, pattern, ptr, ppos): return True # match anything (including a newline) @specializectx -def match_IN(ctx, ptr, ppos): - return rsre_char.check_charset(ctx, ppos+2, ctx.str(ptr)) +def match_IN(ctx, pattern, ptr, ppos): + return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.str(ptr)) @specializectx -def match_IN_IGNORE(ctx, ptr, ppos): - return rsre_char.check_charset(ctx, ppos+2, ctx.lowstr(ptr)) +def match_IN_IGNORE(ctx, pattern, ptr, ppos): + return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.lowstr(ptr)) @specializectx -def match_LITERAL(ctx, ptr, ppos): - return ctx.str(ptr) == ctx.pat(ppos+1) +def match_LITERAL(ctx, pattern, ptr, ppos): + return ctx.str(ptr) == pattern.pat(ppos+1) @specializectx -def match_LITERAL_IGNORE(ctx, ptr, ppos): - return ctx.lowstr(ptr) == ctx.pat(ppos+1) +def match_LITERAL_IGNORE(ctx, pattern, ptr, ppos): + return ctx.lowstr(ptr) == pattern.pat(ppos+1) @specializectx -def match_NOT_LITERAL(ctx, ptr, ppos): - return ctx.str(ptr) != ctx.pat(ppos+1) +def match_NOT_LITERAL(ctx, pattern, ptr, ppos): + return ctx.str(ptr) != pattern.pat(ppos+1) @specializectx -def match_NOT_LITERAL_IGNORE(ctx, ptr, ppos): - return ctx.lowstr(ptr) != ctx.pat(ppos+1) +def match_NOT_LITERAL_IGNORE(ctx, pattern, ptr, ppos): + return ctx.lowstr(ptr) != pattern.pat(ppos+1) def _make_fre(checkerfn): if checkerfn == match_ANY_ALL: - def fre(ctx, ptr, end, ppos): + def fre(ctx, pattern, ptr, end, ppos): return end elif checkerfn == match_IN: install_jitdriver_spec('MatchIn', - greens=['ppos', 'ctx.pattern'], + greens=['ppos', 'pattern'], reds=['ptr', 'end', 'ctx'], debugprint=(1, 0)) @specializectx - def fre(ctx, ptr, end, ppos): + def fre(ctx, pattern, ptr, end, ppos): while True: ctx.jitdriver_MatchIn.jit_merge_point(ctx=ctx, ptr=ptr, - end=end, ppos=ppos) - if ptr < end and checkerfn(ctx, ptr, ppos): + end=end, ppos=ppos, + pattern=pattern) + if ptr < end and checkerfn(ctx, pattern, ptr, ppos): ptr += 1 else: return ptr elif checkerfn == match_IN_IGNORE: install_jitdriver_spec('MatchInIgnore', - greens=['ppos', 'ctx.pattern'], + greens=['ppos', 'pattern'], reds=['ptr', 'end', 'ctx'], debugprint=(1, 0)) @specializectx - def fre(ctx, ptr, end, ppos): + def fre(ctx, pattern, ptr, end, ppos): while True: ctx.jitdriver_MatchInIgnore.jit_merge_point(ctx=ctx, ptr=ptr, - end=end, ppos=ppos) - if ptr < end and checkerfn(ctx, ptr, ppos): + end=end, ppos=ppos, + pattern=pattern) + if ptr < end and checkerfn(ctx, pattern, ptr, ppos): ptr += 1 else: return ptr @@ -925,8 +938,8 @@ # in the other cases, the fre() function is not JITted at all # and is present as a residual call. @specializectx - def fre(ctx, ptr, end, ppos): - while ptr < end and checkerfn(ctx, ptr, ppos): + def fre(ctx, pattern, ptr, end, ppos): + while ptr < end and checkerfn(ctx, pattern, ptr, ppos): ptr += 1 return ptr fre = func_with_new_name(fre, 'fre_' + checkerfn.__name__) @@ -1037,10 +1050,11 @@ return start, end def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False): + assert isinstance(pattern, CompiledPattern) start, end = _adjust(start, end, len(string)) - ctx = StrMatchContext(pattern, string, start, end, flags) + ctx = StrMatchContext(string, start, end, flags) ctx.fullmatch_only = fullmatch - if match_context(ctx): + if match_context(ctx, pattern): return ctx else: return None @@ -1049,105 +1063,106 @@ return match(pattern, string, start, end, flags, fullmatch=True) def search(pattern, string, start=0, end=sys.maxint, flags=0): + assert isinstance(pattern, CompiledPattern) start, end = _adjust(start, end, len(string)) - ctx = StrMatchContext(pattern, string, start, end, flags) - if search_context(ctx): + ctx = StrMatchContext(string, start, end, flags) + if search_context(ctx, pattern): return ctx else: return None install_jitdriver('Match', - greens=['ctx.pattern'], reds=['ctx'], + greens=['pattern'], reds=['ctx'], debugprint=(0,)) -def match_context(ctx): +def match_context(ctx, pattern): ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: return False - ctx.jitdriver_Match.jit_merge_point(ctx=ctx) - return sre_match(ctx, 0, ctx.match_start, None) is not None + ctx.jitdriver_Match.jit_merge_point(ctx=ctx, pattern=pattern) + return sre_match(ctx, pattern, 0, ctx.match_start, None) is not None -def search_context(ctx): +def search_context(ctx, pattern): ctx.original_pos = ctx.match_start if ctx.end < ctx.match_start: return False base = 0 charset = False - if ctx.pat(base) == OPCODE_INFO: - flags = ctx.pat(2) + if pattern.pat(base) == OPCODE_INFO: + flags = pattern.pat(2) if flags & rsre_char.SRE_INFO_PREFIX: - if ctx.pat(5) > 1: - return fast_search(ctx) + if pattern.pat(5) > 1: + return fast_search(ctx, pattern) else: charset = (flags & rsre_char.SRE_INFO_CHARSET) - base += 1 + ctx.pat(1) - if ctx.pat(base) == OPCODE_LITERAL: - return literal_search(ctx, base) + base += 1 + pattern.pat(1) + if pattern.pat(base) == OPCODE_LITERAL: + return literal_search(ctx, pattern, base) if charset: - return charset_search(ctx, base) - return regular_search(ctx, base) + return charset_search(ctx, pattern, base) + return regular_search(ctx, pattern, base) install_jitdriver('RegularSearch', - greens=['base', 'ctx.pattern'], + greens=['base', 'pattern'], reds=['start', 'ctx'], debugprint=(1, 0)) -def regular_search(ctx, base): +def regular_search(ctx, pattern, base): start = ctx.match_start while start <= ctx.end: ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, start=start, - base=base) - if sre_match(ctx, base, start, None) is not None: + base=base, pattern=pattern) + if sre_match(ctx, pattern, base, start, None) is not None: ctx.match_start = start return True start += 1 return False install_jitdriver_spec("LiteralSearch", - greens=['base', 'character', 'ctx.pattern'], + greens=['base', 'character', 'pattern'], reds=['start', 'ctx'], debugprint=(2, 0, 1)) @specializectx -def literal_search(ctx, base): +def literal_search(ctx, pattern, base): # pattern starts with a literal character. this is used # for short prefixes, and if fast search is disabled - character = ctx.pat(base + 1) + character = pattern.pat(base + 1) base += 2 start = ctx.match_start while start < ctx.end: ctx.jitdriver_LiteralSearch.jit_merge_point(ctx=ctx, start=start, - base=base, character=character) + base=base, character=character, pattern=pattern) if ctx.str(start) == character: - if sre_match(ctx, base, start + 1, None) is not None: + if sre_match(ctx, pattern, base, start + 1, None) is not None: ctx.match_start = start return True start += 1 return False install_jitdriver_spec("CharsetSearch", - greens=['base', 'ctx.pattern'], + greens=['base', 'pattern'], reds=['start', 'ctx'], debugprint=(1, 0)) @specializectx -def charset_search(ctx, base): +def charset_search(ctx, pattern, base): # pattern starts with a character from a known set start = ctx.match_start while start < ctx.end: ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start, - base=base) - if rsre_char.check_charset(ctx, 5, ctx.str(start)): - if sre_match(ctx, base, start, None) is not None: + base=base, pattern=pattern) + if rsre_char.check_charset(ctx, pattern, 5, ctx.str(start)): + if sre_match(ctx, pattern, base, start, None) is not None: ctx.match_start = start return True start += 1 return False install_jitdriver_spec('FastSearch', - greens=['i', 'prefix_len', 'ctx.pattern'], + greens=['i', 'prefix_len', 'pattern'], reds=['string_position', 'ctx'], debugprint=(2, 0)) @specializectx -def fast_search(ctx): +def fast_search(ctx, pattern): # skips forward in a string as fast as possible using information from # an optimization info block # <1=skip> <2=flags> <3=min> <4=...> @@ -1155,17 +1170,18 @@ string_position = ctx.match_start if string_position >= ctx.end: return False - prefix_len = ctx.pat(5) + prefix_len = pattern.pat(5) assert prefix_len >= 0 i = 0 while True: ctx.jitdriver_FastSearch.jit_merge_point(ctx=ctx, - string_position=string_position, i=i, prefix_len=prefix_len) + string_position=string_position, i=i, prefix_len=prefix_len, + pattern=pattern) char_ord = ctx.str(string_position) - if char_ord != ctx.pat(7 + i): + if char_ord != pattern.pat(7 + i): if i > 0: overlap_offset = prefix_len + (7 - 1) - i = ctx.pat(overlap_offset + i) + i = pattern.pat(overlap_offset + i) continue else: i += 1 @@ -1173,22 +1189,22 @@ # found a potential match start = string_position + 1 - prefix_len assert start >= 0 - prefix_skip = ctx.pat(6) + prefix_skip = pattern.pat(6) ptr = start + prefix_skip - #flags = ctx.pat(2) + #flags = pattern.pat(2) #if flags & rsre_char.SRE_INFO_LITERAL: # # matched all of pure literal pattern # ctx.match_start = start # ctx.match_end = ptr # ctx.match_marks = None # return True - pattern_offset = ctx.pat(1) + 1 + pattern_offset = pattern.pat(1) + 1 ppos_start = pattern_offset + 2 * prefix_skip - if sre_match(ctx, ppos_start, ptr, None) is not None: + if sre_match(ctx, pattern, ppos_start, ptr, None) is not None: ctx.match_start = start return True overlap_offset = prefix_len + (7 - 1) - i = ctx.pat(overlap_offset + i) + i = pattern.pat(overlap_offset + i) string_position += 1 if string_position >= ctx.end: return False diff --git a/rpython/rlib/rsre/test/test_char.py b/rpython/rlib/rsre/test/test_char.py --- a/rpython/rlib/rsre/test/test_char.py +++ b/rpython/rlib/rsre/test/test_char.py @@ -1,10 +1,16 @@ -from rpython.rlib.rsre import rsre_char +from rpython.rlib.rsre import rsre_char, rsre_core from rpython.rlib.rsre.rsre_char import SRE_FLAG_LOCALE, SRE_FLAG_UNICODE def setup_module(mod): from rpython.rlib.unicodedata import unicodedb rsre_char.set_unicode_db(unicodedb) + +def check_charset(pattern, idx, char): + p = rsre_core.CompiledPattern(pattern) + return rsre_char.check_charset(Ctx(p), p, idx, char) + + UPPER_PI = 0x3a0 LOWER_PI = 0x3c0 INDIAN_DIGIT = 0x966 @@ -157,12 +163,12 @@ pat_neg = [70, ord(cat) | 0x80, 0] for c in positive: assert unicodedb.category(ord(c)).startswith(cat) - assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert check_charset(pat_pos, 0, ord(c)) + assert not check_charset(pat_neg, 0, ord(c)) for c in negative: assert not unicodedb.category(ord(c)).startswith(cat) - assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert not check_charset(pat_pos, 0, ord(c)) + assert check_charset(pat_neg, 0, ord(c)) def cat2num(cat): return ord(cat[0]) | (ord(cat[1]) << 8) @@ -173,17 +179,16 @@ pat_neg = [70, cat2num(cat) | 0x80, 0] for c in positive: assert unicodedb.category(ord(c)) == cat - assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert check_charset(pat_pos, 0, ord(c)) + assert not check_charset(pat_neg, 0, ord(c)) for c in negative: assert unicodedb.category(ord(c)) != cat - assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c)) - assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c)) + assert not check_charset(pat_pos, 0, ord(c)) + assert check_charset(pat_neg, 0, ord(c)) # test for how the common 'L&' pattern might be compiled pat = [70, cat2num('Lu'), 70, cat2num('Ll'), 70, cat2num('Lt'), 0] - assert rsre_char.check_charset(Ctx(pat), 0, 65) # Lu - assert rsre_char.check_charset(Ctx(pat), 0, 99) # Ll - assert rsre_char.check_charset(Ctx(pat), 0, 453) # Lt - assert not rsre_char.check_charset(Ctx(pat), 0, 688) # Lm - assert not rsre_char.check_charset(Ctx(pat), 0, 5870) # Nl + assert check_charset(pat, 0, 65) # Lu + assert check_charset(pat, 0, 99) # Lcheck_charset(pat, 0, 453) # Lt + assert not check_charset(pat, 0, 688) # Lm + assert not check_charset(pat, 0, 5870) # Nl diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py --- a/rpython/rlib/rsre/test/test_ext_opcode.py +++ b/rpython/rlib/rsre/test/test_ext_opcode.py @@ -17,10 +17,10 @@ # it's a valid optimization because \1 is always one character long r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, GROUPREF, 0, SUCCESS, SUCCESS] - assert rsre_core.match(r, "aaa").match_end == 3 + assert rsre_core.match(rsre_core.CompiledPattern(r), "aaa").match_end == 3 def test_min_repeat_one_with_backref(): # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT, GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS] - assert rsre_core.match(r, "aaab").match_end == 4 + assert rsre_core.match(rsre_core.CompiledPattern(r), "aaab").match_end == 4 diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py --- a/rpython/rlib/rsre/test/test_match.py +++ b/rpython/rlib/rsre/test/test_match.py @@ -9,7 +9,7 @@ def test_get_code_repetition(): c1 = get_code(r"a+") c2 = get_code(r"a+") - assert c1 == c2 + assert c1.pattern == c2.pattern class TestMatch: @@ -305,6 +305,6 @@ rsre_char.set_unicode_db(unicodedb) # r = get_code(u"[\U00010428-\U0001044f]", re.I) - assert r.count(27) == 1 # OPCODE_RANGE - r[r.index(27)] = 32 # => OPCODE_RANGE_IGNORE + assert r.pattern.count(27) == 1 # OPCODE_RANGE + r.pattern[r.pattern.index(27)] = 32 # => OPCODE_RANGE_IGNORE assert rsre_core.match(r, u"\U00010428") diff --git a/rpython/rlib/rsre/test/test_re.py b/rpython/rlib/rsre/test/test_re.py --- a/rpython/rlib/rsre/test/test_re.py +++ b/rpython/rlib/rsre/test/test_re.py @@ -426,31 +426,6 @@ assert pat.match(p) is not None assert pat.match(p).span() == (0,256) - def test_pickling(self): - import pickle - self.pickle_test(pickle) - import cPickle - self.pickle_test(cPickle) - # old pickles expect the _compile() reconstructor in sre module - import warnings - original_filters = warnings.filters[:] - try: - warnings.filterwarnings("ignore", "The sre module is deprecated", - DeprecationWarning) - from sre import _compile - finally: - warnings.filters = original_filters - - def pickle_test(self, pickle): - oldpat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)') - s = pickle.dumps(oldpat) - newpat = pickle.loads(s) - # Not using object identity for _sre.py, since some Python builds do - # not seem to preserve that in all cases (observed on an UCS-4 build - # of 2.4.1). - #self.assertEqual(oldpat, newpat) - assert oldpat.__dict__ == newpat.__dict__ - def test_constants(self): assert re.I == re.IGNORECASE assert re.L == re.LOCALE diff --git a/rpython/rlib/rsre/test/test_zinterp.py b/rpython/rlib/rsre/test/test_zinterp.py --- a/rpython/rlib/rsre/test/test_zinterp.py +++ b/rpython/rlib/rsre/test/test_zinterp.py @@ -11,6 +11,7 @@ rsre_core.search(pattern, string) # unicodestr = unichr(n) * n + pattern = rsre_core.CompiledPattern(pattern) ctx = rsre_core.UnicodeMatchContext(pattern, unicodestr, 0, len(unicodestr), 0) rsre_core.search_context(ctx) diff --git a/rpython/rlib/rsre/test/test_zjit.py b/rpython/rlib/rsre/test/test_zjit.py --- a/rpython/rlib/rsre/test/test_zjit.py +++ b/rpython/rlib/rsre/test/test_zjit.py @@ -6,18 +6,20 @@ from rpython.rtyper.annlowlevel import llstr, hlstr def entrypoint1(r, string, repeat): - r = array2list(r) + r = rsre_core.CompiledPattern(array2list(r)) string = hlstr(string) match = None for i in range(repeat): match = rsre_core.match(r, string) + if match is None: + return -1 if match is None: return -1 else: return match.match_end def entrypoint2(r, string, repeat): - r = array2list(r) + r = rsre_core.CompiledPattern(array2list(r)) string = hlstr(string) match = None for i in range(repeat): @@ -48,13 +50,13 @@ def meta_interp_match(self, pattern, string, repeat=1): r = get_code(pattern) - return self.meta_interp(entrypoint1, [list2array(r), llstr(string), + return self.meta_interp(entrypoint1, [list2array(r.pattern), llstr(string), repeat], listcomp=True, backendopt=True) def meta_interp_search(self, pattern, string, repeat=1): r = get_code(pattern) - return self.meta_interp(entrypoint2, [list2array(r), llstr(string), + return self.meta_interp(entrypoint2, [list2array(r.pattern), llstr(string), repeat], listcomp=True, backendopt=True) @@ -166,3 +168,9 @@ res = self.meta_interp_search(r"b+", "a"*30 + "b") assert res == 30 self.check_resops(call=0) + + def test_match_jit_bug(self): + pattern = ".a" * 2500 + text = "a" * 6000 + res = self.meta_interp_match(pattern, text, repeat=10) + assert res != -1 diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -225,8 +225,10 @@ def test_green_field(self): def get_printable_location(xfoo): return str(ord(xfoo)) # xfoo must be annotated as a character - myjitdriver = JitDriver(greens=['x.foo'], reds=['n', 'x'], + # green fields are disabled! + pytest.raises(ValueError, JitDriver, greens=['x.foo'], reds=['n', 'x'], get_printable_location=get_printable_location) + return class A(object): _immutable_fields_ = ['foo'] def fn(n): From pypy.commits at gmail.com Thu Mar 29 02:04:21 2018 From: pypy.commits at gmail.com (arigo) Date: Wed, 28 Mar 2018 23:04:21 -0700 (PDT) Subject: [pypy-commit] pypy.org extradoc: Clarify how to build PyPy3 from sources Message-ID: <5abc81e5.ab87df0a.8736b.9bd5@mx.google.com> Author: Armin Rigo Branch: extradoc Changeset: r919:b010de333e74 Date: 2018-03-29 08:04 +0200 http://bitbucket.org/pypy/pypy.org/changeset/b010de333e74/ Log: Clarify how to build PyPy3 from sources diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -263,10 +263,16 @@ 
         hg clone https://bitbucket.org/pypy/pypy
        -

        Alternatively, the following smaller package contains the source at +

        The trunk contains PyPy 2. For PyPy 3, switch to the correct branch:

        +
        +# for PyPy 3: switch to the branch of PyPy that implements Python 3.5
+hg update py3.5
+
        +

        Alternatively, get one of the following smaller packages for the source at the same revision as the above binaries:

      • Make sure you installed the dependencies. See the list here.

        @@ -278,7 +284,7 @@

      • Run the rpython script. Here are the common combinations of options (works also with python instead of pypy; -requires CPython 2.7 or PyPy 2):

        +requires CPython 2.7 or PyPy 2, even to build PyPy 3):

         pypy ../../rpython/bin/rpython -Ojit targetpypystandalone           # get the JIT version
 pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -301,12 +301,19 @@
 
      hg clone https://bitbucket.org/pypy/pypy
 
-   Alternatively, the following smaller package contains the source at
+   The trunk contains PyPy 2.  For PyPy 3, switch to the correct branch::
+
+     # for PyPy 3: switch to the branch of PyPy that implements Python 3.5
+     hg update py3.5
+
+   Alternatively, get one of the following smaller packages for the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.10.1-src.tar.bz2`__ (sources)
+   * `pypy2-v5.10.1-src.tar.bz2`__ (sources, PyPy 2 only)
+   * `pypy3-v5.10.1-src.tar.bz2`__ (sources, PyPy 3 only)
 
    .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.10.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.10.1-src.tar.bz2
 
 
 2. Make sure you **installed the dependencies.**  See the list here__.
@@ -319,7 +326,7 @@
 
 4. Run the ``rpython`` script.  Here are the common combinations
    of options (works also with ``python`` instead of ``pypy``;
-   requires CPython 2.7 or PyPy 2)::
+   requires CPython 2.7 or PyPy 2, even to build PyPy 3)::
 
      pypy ../../rpython/bin/rpython -Ojit targetpypystandalone           # get the JIT version
      pypy ../../rpython/bin/rpython -O2 targetpypystandalone             # get the no-jit version

From pypy.commits at gmail.com  Thu Mar 29 05:49:39 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Mar 2018 02:49:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default into branch
Message-ID: <5abcb6b3.d2d1df0a.568d5.a37a@mx.google.com>

Author: Matti Picus 
Branch: py3.5
Changeset: r94166:c3b4518f2322
Date: 2018-03-29 12:48 +0300
http://bitbucket.org/pypy/pypy/changeset/c3b4518f2322/

Log:	merge default into branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -62,3 +62,13 @@
 .. branch: rpython-sprint
 
 Refactor in rpython signatures
+
+.. branch: cpyext-tls-operror2
+
+Store error state thread-locally in executioncontext, fixes issue #2764
+
+.. branch: cpyext-fast-typecheck
+
+Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify
+`W_PyCWrapperObject` which is used to call slots from the C-API, greatly
+improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -5,7 +5,8 @@
 from pypy.objspace.std.listobject import (
     ListStrategy, UNROLL_CUTOFF, W_ListObject, ObjectListStrategy)
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP)
+    cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP,
+    generic_cpy_call)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.pyobject import as_pyobj, incref
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -145,21 +146,26 @@
     # XXX we should call Py*_GET_ITEM() instead of Py*_GetItem()
     # from here, but we cannot because we are also called from
     # PySequence_GetItem()
+    py_obj = as_pyobj(space, w_obj)
     if isinstance(w_obj, tupleobject.W_TupleObject):
         from pypy.module.cpyext.tupleobject import PyTuple_GetItem
-        py_obj = as_pyobj(space, w_obj)
         py_res = PyTuple_GetItem(space, py_obj, i)
         incref(space, py_res)
         keepalive_until_here(w_obj)
         return py_res
     if isinstance(w_obj, W_ListObject):
         from pypy.module.cpyext.listobject import PyList_GetItem
-        py_obj = as_pyobj(space, w_obj)
         py_res = PyList_GetItem(space, py_obj, i)
         incref(space, py_res)
         keepalive_until_here(w_obj)
         return py_res
-    return make_ref(space, space.getitem(w_obj, space.newint(i)))
+    
+    as_sequence = py_obj.c_ob_type.c_tp_as_sequence
+    if not as_sequence or not as_sequence.c_sq_item:
+        raise oefmt(space.w_TypeError,
+                    "'%T' object does not support indexing", w_obj)
+    ret = generic_cpy_call(space, as_sequence.c_sq_item, w_obj, i)
+    return make_ref(space, ret)
 
 @cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PySequence_GetItem(space, w_obj, i):
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2638,6 +2638,16 @@
     Py_RETURN_NONE;
 };
 
+static PyObject *
+getitem(PyObject* self, PyObject * args) {
+    PyObject * obj;
+    int i;
+    if (!PyArg_ParseTuple(args, "Oi", &obj, &i)) {
+        return NULL;
+    }
+    return PySequence_ITEM(obj, i);
+}
+
 PyDoc_STRVAR(module_doc,
 "This module defines an object type which can efficiently represent\n\
 an array of basic values: characters, integers, floating point\n\
@@ -2937,6 +2947,7 @@
     {"create_and_release_buffer",   (PyCFunction)create_and_release_buffer, METH_O, NULL},
     {"write_buffer_len", write_buffer_len, METH_O, NULL},
     {"same_dealloc",   (PyCFunction)same_dealloc, METH_VARARGS, NULL},
+    {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL},
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -167,3 +167,15 @@
         fd = BytesIO()
         # only test that it works
         fd.write(a)
+
+    def test_getitem_via_PySequence_GetItem(self):
+        module = self.import_module(name='array')
+        a = module.array('i', range(10))
+        # call via tp_as_mapping.mp_subscript
+        assert 5 == a[-5]
+        # PySequence_ITEM used to call space.getitem() which
+        # prefers tp_as_mapping.mp_subscript over tp_as_sequence.sq_item
+        # Now fixed so this test raises (array_item does not add len(a),
+        # array_subscr does)
+        raises(IndexError, module.getitem, a, -5)
+
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -8,6 +8,7 @@
 from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr
 
 
+
 class AppTestTypeObject(AppTestCpythonExtensionBase):
 
     def setup_class(cls):
@@ -917,6 +918,10 @@
         res = "foo" in obj
         assert res is True
 
+            #if PY_MAJOR_VERSION > 2
+            #define PyInt_Check PyLong_Check
+            #define PyInt_AsLong PyLong_AsLong
+            #endif
     def test_sq_ass_item(self):
         module = self.import_extension('foo', [
            ("new_obj", "METH_NOARGS",
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -375,28 +375,58 @@
         log = self.run(main, [300])
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i161 = int_lt(i160, i43)
+        i106 = getfield_gc_i(p20, descr=...)
+        i161 = int_lt(i106, i43)
         guard_true(i161, descr=...)
-        i162 = int_add(i160, 1)
-        setfield_gc(p22, i162, descr=)
+        i162 = int_add(i106, 1)
+        p110 = getfield_gc_r(p16, descr=...)
+        setfield_gc(p20, i162, descr=...)
+        guard_value(p110, ConstPtr(ptr111), descr=...)
         guard_not_invalidated(descr=...)
         p163 = force_token()
         p164 = force_token()
-        p167 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=)
+        p118 = getfield_gc_r(p16, descr=...)
+        p120 = getarrayitem_gc_r(p118, 0, descr=...)
+        guard_value(p120, ConstPtr(ptr121), descr=...)
+        p122 = getfield_gc_r(p120, descr=...)
+        guard_value(p122, ConstPtr(ptr123), descr=...)
+        p125 = getfield_gc_r(p16, descr=...)
+        guard_nonnull_class(p125, ..., descr=...)
+        p127 = getfield_gc_r(p125, descr=...)
+        guard_value(p127, ConstPtr(ptr128), descr=...)
+        p129 = getfield_gc_r(p127, descr=...)
+        guard_value(p129, ConstPtr(ptr130), descr=...)
+        p132 = call_r(ConstClass(_ll_0_alloc_with_del___), descr=...)
         guard_no_exception(descr=...)
-        i112 = int_signext(i160, 2)
-        setfield_gc(p167, ConstPtr(ptr85), descr=)
-        setfield_gc(p167, -1, descr=)
-        i114 = int_ne(i160, i112)
-        guard_false(i114, descr=...)
-        # NB. we get threads because '_hashlib' uses ffi callback/def_extern
-        --THREAD-TICK--
-        i123 = arraylen_gc(p67, descr=)
-        i119 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero_mpressure__Signed), 6, descr=)
-        check_memory_error(i119)
-        raw_store(i119, 0, i160, descr=)
-        raw_store(i119, 2, i160, descr=)
-        raw_store(i119, 4, i160, descr=)
-        setfield_gc(p167, i119, descr=)
+        p133 = force_token()
+        p134 = new_with_vtable(descr=...)
+        setfield_gc(p134, ..., descr=...)
+        setfield_gc(p134, ConstPtr(null), descr=...)
+        setfield_gc(p48, p134, descr=...)
+        setfield_gc(p132, ..., descr=...)
+        i138 = call_i(ConstClass(_ll_1_raw_malloc_varsize_zero__Signed), 6, descr=...)
+        check_memory_error(i138)
+        setfield_gc(p132, i138, descr=...)
+        setfield_gc(p132, ConstPtr(ptr139), descr=...)
+        setfield_gc(p132, -1, descr=...)
+        setfield_gc(p0, p133, descr=...)
+        call_may_force_n(ConstClass(_ll_2_gc_add_memory_pressure__Signed_pypy_module__cffi_backend_cdataobj_W_CDataNewStdPtr), 6, p132, descr=...)
+        guard_not_forced(descr=...)
+        guard_no_exception(descr=...)
+        i144 = int_add(i138, 0)
+        i146 = int_signext(i106, 2)
+        i147 = int_ne(i106, i146)
+        guard_false(i147, descr=...)
+        setarrayitem_raw(i144, 0, i106, descr=...)
+        i150 = int_add(i138, 2)
+        setarrayitem_raw(i150, 0, i106, descr=...)
+        i153 = int_add(i138, 4)
+        setarrayitem_raw(i153, 0, i106, descr=...)
+        p156 = getfield_gc_r(p48, descr=...)
+        i158 = getfield_raw_i(..., descr=...)
+        setfield_gc(p48, p49, descr=...)
+        setfield_gc(p134, ConstPtr(null), descr=...)
+        i160 = int_lt(i158, 0)
+        guard_false(i160, descr=...)
         jump(..., descr=...)
         """)
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -742,6 +742,9 @@
     def op_zero_gc_pointers_inside(self, obj):
         raise NotImplementedError("zero_gc_pointers_inside")
 
+    def op_gc_get_stats(self, obj):
+        raise NotImplementedError("gc_get_stats")
+
     def op_gc_writebarrier_before_copy(self, source, dest,
                                        source_start, dest_start, length):
         if hasattr(self.heap, 'writebarrier_before_copy'):

From pypy.commits at gmail.com  Thu Mar 29 05:49:36 2018
From: pypy.commits at gmail.com (mattip)
Date: Thu, 29 Mar 2018 02:49:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: one more missing file
Message-ID: <5abcb6b0.0ea7df0a.87aa2.e457@mx.google.com>

Author: Matti Picus 
Branch: py3.5
Changeset: r94165:c06e1a903116
Date: 2018-03-29 12:38 +0300
http://bitbucket.org/pypy/pypy/changeset/c06e1a903116/

Log:	one more missing file

diff --git a/pypy/module/cpyext/test/sre_constants.h b/pypy/module/cpyext/test/sre_constants.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/sre_constants.h
@@ -0,0 +1,86 @@
+/*
+ * Secret Labs' Regular Expression Engine
+ *
+ * regular expression matching engine
+ *
+ * NOTE: This file is generated by sre_constants.py.  If you need
+ * to change anything in here, edit sre_constants.py and run it.
+ *
+ * Copyright (c) 1997-2001 by Secret Labs AB.  All rights reserved.
+ *
+ * See the _sre.c file for information on usage and redistribution.
+ */
+
+#define SRE_MAGIC 20031017
+#define SRE_OP_FAILURE 0
+#define SRE_OP_SUCCESS 1
+#define SRE_OP_ANY 2
+#define SRE_OP_ANY_ALL 3
+#define SRE_OP_ASSERT 4
+#define SRE_OP_ASSERT_NOT 5
+#define SRE_OP_AT 6
+#define SRE_OP_BRANCH 7
+#define SRE_OP_CALL 8
+#define SRE_OP_CATEGORY 9
+#define SRE_OP_CHARSET 10
+#define SRE_OP_BIGCHARSET 11
+#define SRE_OP_GROUPREF 12
+#define SRE_OP_GROUPREF_EXISTS 13
+#define SRE_OP_GROUPREF_IGNORE 14
+#define SRE_OP_IN 15
+#define SRE_OP_IN_IGNORE 16
+#define SRE_OP_INFO 17
+#define SRE_OP_JUMP 18
+#define SRE_OP_LITERAL 19
+#define SRE_OP_LITERAL_IGNORE 20
+#define SRE_OP_MARK 21
+#define SRE_OP_MAX_UNTIL 22
+#define SRE_OP_MIN_UNTIL 23
+#define SRE_OP_NOT_LITERAL 24
+#define SRE_OP_NOT_LITERAL_IGNORE 25
+#define SRE_OP_NEGATE 26
+#define SRE_OP_RANGE 27
+#define SRE_OP_REPEAT 28
+#define SRE_OP_REPEAT_ONE 29
+#define SRE_OP_SUBPATTERN 30
+#define SRE_OP_MIN_REPEAT_ONE 31
+#define SRE_AT_BEGINNING 0
+#define SRE_AT_BEGINNING_LINE 1
+#define SRE_AT_BEGINNING_STRING 2
+#define SRE_AT_BOUNDARY 3
+#define SRE_AT_NON_BOUNDARY 4
+#define SRE_AT_END 5
+#define SRE_AT_END_LINE 6
+#define SRE_AT_END_STRING 7
+#define SRE_AT_LOC_BOUNDARY 8
+#define SRE_AT_LOC_NON_BOUNDARY 9
+#define SRE_AT_UNI_BOUNDARY 10
+#define SRE_AT_UNI_NON_BOUNDARY 11
+#define SRE_CATEGORY_DIGIT 0
+#define SRE_CATEGORY_NOT_DIGIT 1
+#define SRE_CATEGORY_SPACE 2
+#define SRE_CATEGORY_NOT_SPACE 3
+#define SRE_CATEGORY_WORD 4
+#define SRE_CATEGORY_NOT_WORD 5
+#define SRE_CATEGORY_LINEBREAK 6
+#define SRE_CATEGORY_NOT_LINEBREAK 7
+#define SRE_CATEGORY_LOC_WORD 8
+#define SRE_CATEGORY_LOC_NOT_WORD 9
+#define SRE_CATEGORY_UNI_DIGIT 10
+#define SRE_CATEGORY_UNI_NOT_DIGIT 11
+#define SRE_CATEGORY_UNI_SPACE 12
+#define SRE_CATEGORY_UNI_NOT_SPACE 13
+#define SRE_CATEGORY_UNI_WORD 14
+#define SRE_CATEGORY_UNI_NOT_WORD 15
+#define SRE_CATEGORY_UNI_LINEBREAK 16
+#define SRE_CATEGORY_UNI_NOT_LINEBREAK 17
+#define SRE_FLAG_TEMPLATE 1
+#define SRE_FLAG_IGNORECASE 2
+#define SRE_FLAG_LOCALE 4
+#define SRE_FLAG_MULTILINE 8
+#define SRE_FLAG_DOTALL 16
+#define SRE_FLAG_UNICODE 32
+#define SRE_FLAG_VERBOSE 64
+#define SRE_INFO_PREFIX 1
+#define SRE_INFO_LITERAL 2
+#define SRE_INFO_CHARSET 4

From pypy.commits at gmail.com  Thu Mar 29 05:50:07 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 02:50:07 -0700 (PDT)
Subject: [pypy-commit] pypy pyparser-improvements: move bench target to test/
Message-ID: <5abcb6cf.77a9df0a.5b00d.5ca2@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: pyparser-improvements
Changeset: r94167:e5c117bc3cbf
Date: 2018-03-29 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e5c117bc3cbf/

Log:	move bench target to test/

diff --git a/pypy/interpreter/pyparser/targetparse.py b/pypy/interpreter/pyparser/test/targetparse.py
rename from pypy/interpreter/pyparser/targetparse.py
rename to pypy/interpreter/pyparser/test/targetparse.py

From pypy.commits at gmail.com  Thu Mar 29 05:50:13 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 02:50:13 -0700 (PDT)
Subject: [pypy-commit] pypy pyparser-improvements: document branch
Message-ID: <5abcb6d5.ab87df0a.8736b.e719@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: pyparser-improvements
Changeset: r94169:ec7d9790f8f9
Date: 2018-03-29 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/ec7d9790f8f9/

Log:	document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -79,3 +79,8 @@
 
 - in rare "trace is too long" situations, the JIT could break behaviour
   arbitrarily.
+
+
+.. branch: pyparser-improvements
+
+Improve speed of Python parser, improve ParseError messages slightly.

From pypy.commits at gmail.com  Thu Mar 29 05:50:11 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 02:50:11 -0700 (PDT)
Subject: [pypy-commit] pypy pyparser-improvements: merge default
Message-ID: <5abcb6d3.5b88df0a.be1c9.cc41@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: pyparser-improvements
Changeset: r94168:fbbadbbb888b
Date: 2018-03-29 10:43 +0200
http://bitbucket.org/pypy/pypy/changeset/fbbadbbb888b/

Log:	merge default

diff too long, truncating to 2000 out of 4540 lines

diff --git a/README.rst b/README.rst
--- a/README.rst
+++ b/README.rst
@@ -4,42 +4,40 @@
 
 Welcome to PyPy!
 
-PyPy is both an implementation of the Python programming language, and
-an extensive compiler framework for dynamic language implementations.
-You can build self-contained Python implementations which execute
-independently from CPython.
+PyPy is an interperter that implements the Python programming language, based
+on the RPython compiler framework for dynamic language implementations.
 
-The home page is:
+The home page for the interpreter is:
 
     http://pypy.org/
 
-If you want to help developing PyPy, this document might help you:
+If you want to help developing PyPy, this documentation might help you:
 
     http://doc.pypy.org/
 
-It will also point you to the rest of the documentation which is generated
-from files in the pypy/doc directory within the source repositories. Enjoy
-and send us feedback!
+More documentation about the RPython framework can be found here
 
-    the pypy-dev team 
+    http://rpython.readthedocs.io
 
+The source for the documentation is in the pypy/doc directory 
+
+Using PyPy instead of CPython
+=============================
+
+Please read the information at http://pypy.org to find the correct way to
+download and use PyPy as an alternative to CPython. 
 
 Building
 ========
 
-First switch to or download the correct branch.  The basic choices are
-``default`` for Python 2.7 and, for Python 3.X, the corresponding py3.X
-branch (e.g. ``py3.5``).
+Building PyPy is not the recommended way to obtain the PyPy alternative python
+interpreter. It is time-consuming and requires significant computing resources.
+More information can be found here
 
-Build with:
+    http://doc.pypy.org/en/latest/build.html
 
-.. code-block:: console
+Enjoy and send us feedback!
 
-    $ rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+    the pypy-dev team 
 
-This ends up with a ``pypy-c`` or ``pypy3-c`` binary in the main pypy
-directory.  We suggest to use virtualenv with the resulting
-pypy-c/pypy3-c as the interpreter; you can find more details about
-various installation schemes here:
 
-    http://doc.pypy.org/en/latest/install.html
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -17,13 +17,18 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The quickest way to start using PyPy is to download a prebuilt binary for your
-OS and architecture.  You can either use the `most recent release`_ or one of
-our `development nightly build`_.  Please note that the nightly builds are not
+OS and architecture.  You may be able to use either use the
+`most recent release`_ or one of our `development nightly build`_. These
+builds depend on dynamically linked libraries that may not be available on your
+OS. See the section about `Linux binaries` for more info and alternatives that
+may work on your system.
+
+Please note that the nightly builds are not
 guaranteed to be as stable as official releases, use them at your own risk.
 
 .. _most recent release: http://pypy.org/download.html
 .. _development nightly build: http://buildbot.pypy.org/nightly/trunk/
-
+.. _Linux binaries: http://pypy.org/download.html#linux-binaries-and-common-distributions
 
 Installing PyPy
 ~~~~~~~~~~~~~~~
@@ -69,9 +74,9 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 It is often convenient to run pypy inside a virtualenv.  To do this
-you need a recent version of virtualenv -- 1.6.1 or greater.  You can
+you need a version of virtualenv -- 1.6.1 or greater.  You can
 then install PyPy both from a precompiled tarball or from a mercurial
-checkout::
+checkout after translation::
 
 	# from a tarball
 	$ virtualenv -p /opt/pypy-xxx/bin/pypy my-pypy-env
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -54,3 +54,28 @@
 
 Speed up branchy code that does a lot of function inlining by saving one call
 to read the TLS in most bridges.
+
+.. branch: rpython-sprint
+
+Refactor in rpython signatures
+
+.. branch: cpyext-tls-operror2
+
+Store error state thread-locally in executioncontext, fixes issue #2764
+
+.. branch: cpyext-fast-typecheck
+
+Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify
+`W_PyCWrapperObject` which is used to call slots from the C-API, greatly
+improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks
+
+
+.. branch: fix-sre-problems
+
+Fix two (unrelated) JIT bugs manifesting in the re module:
+
+- green fields are broken and were thus disabled, plus their usage removed from
+  the _sre implementation
+
+- in rare "trace is too long" situations, the JIT could break behaviour
+  arbitrarily.
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -232,7 +232,9 @@
                 "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
+        key_pycode = self.key_pycode
         jitdriver1.jit_merge_point(callback=self,
+                                   key_pycode=key_pycode,
                                    ll_res=ll_res,
                                    ll_args=ll_args)
         self.do_invoke(ll_res, ll_args)
@@ -294,7 +296,7 @@
     return 'cffi_callback ' + key_pycode.get_repr()
 
 jitdriver1 = jit.JitDriver(name='cffi_callback',
-                           greens=['callback.key_pycode'],
+                           greens=['key_pycode'],
                            reds=['ll_res', 'll_args', 'callback'],
                            get_printable_location=get_printable_location1)
 
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -77,15 +77,15 @@
     w_import = space.getattr(w_builtin, space.newtext("__import__"))
     return space.call_function(w_import, space.newtext("re"))
 
-def matchcontext(space, ctx):
+def matchcontext(space, ctx, pattern):
     try:
-        return rsre_core.match_context(ctx)
+        return rsre_core.match_context(ctx, pattern)
     except rsre_core.Error as e:
         raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
-def searchcontext(space, ctx):
+def searchcontext(space, ctx, pattern):
     try:
-        return rsre_core.search_context(ctx)
+        return rsre_core.search_context(ctx, pattern)
     except rsre_core.Error as e:
         raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
@@ -114,7 +114,7 @@
                 pos = len(unicodestr)
             if endpos > len(unicodestr):
                 endpos = len(unicodestr)
-            return rsre_core.UnicodeMatchContext(self.code, unicodestr,
+            return rsre_core.UnicodeMatchContext(unicodestr,
                                                  pos, endpos, self.flags)
         elif space.isinstance_w(w_string, space.w_bytes):
             str = space.bytes_w(w_string)
@@ -122,7 +122,7 @@
                 pos = len(str)
             if endpos > len(str):
                 endpos = len(str)
-            return rsre_core.StrMatchContext(self.code, str,
+            return rsre_core.StrMatchContext(str,
                                              pos, endpos, self.flags)
         else:
             buf = space.readbuf_w(w_string)
@@ -132,7 +132,7 @@
                 pos = size
             if endpos > size:
                 endpos = size
-            return rsre_core.BufMatchContext(self.code, buf,
+            return rsre_core.BufMatchContext(buf,
                                              pos, endpos, self.flags)
 
     def getmatch(self, ctx, found):
@@ -144,12 +144,12 @@
     @unwrap_spec(pos=int, endpos=int)
     def match_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
-        return self.getmatch(ctx, matchcontext(self.space, ctx))
+        return self.getmatch(ctx, matchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def search_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
-        return self.getmatch(ctx, searchcontext(self.space, ctx))
+        return self.getmatch(ctx, searchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def findall_w(self, w_string, pos=0, endpos=sys.maxint):
@@ -157,7 +157,7 @@
         matchlist_w = []
         ctx = self.make_ctx(w_string, pos, endpos)
         while ctx.match_start <= ctx.end:
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, self.code):
                 break
             num_groups = self.num_groups
             w_emptystr = space.newtext("")
@@ -182,7 +182,7 @@
         # this also works as the implementation of the undocumented
         # scanner() method.
         ctx = self.make_ctx(w_string, pos, endpos)
-        scanner = W_SRE_Scanner(self, ctx)
+        scanner = W_SRE_Scanner(self, ctx, self.code)
         return scanner
 
     @unwrap_spec(maxsplit=int)
@@ -193,7 +193,7 @@
         last = 0
         ctx = self.make_ctx(w_string)
         while not maxsplit or n < maxsplit:
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, self.code):
                 break
             if ctx.match_start == ctx.match_end:     # zero-width match
                 if ctx.match_start == ctx.end:       # or end of string
@@ -274,13 +274,14 @@
         else:
             sublist_w = []
         n = last_pos = 0
+        pattern = self.code
         while not count or n < count:
             sub_jitdriver.jit_merge_point(
                 self=self,
                 use_builder=use_builder,
                 filter_is_callable=filter_is_callable,
                 filter_type=type(w_filter),
-                ctx=ctx,
+                ctx=ctx, pattern=pattern,
                 w_filter=w_filter,
                 strbuilder=strbuilder,
                 unicodebuilder=unicodebuilder,
@@ -291,7 +292,7 @@
                 n=n, last_pos=last_pos, sublist_w=sublist_w
                 )
             space = self.space
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, pattern):
                 break
             if last_pos < ctx.match_start:
                 _sub_append_slice(
@@ -355,7 +356,7 @@
             filter_as_unicode
             w_string sublist_w
             self""".split(),
-    greens=["filter_is_callable", "use_builder", "filter_type", "ctx.pattern"])
+    greens=["filter_is_callable", "use_builder", "filter_type", "pattern"])
 
 
 def _sub_append_slice(ctx, space, use_builder, sublist_w,
@@ -387,7 +388,11 @@
     srepat.space = space
     srepat.w_pattern = w_pattern      # the original uncompiled pattern
     srepat.flags = flags
-    srepat.code = code
+    # note: we assume that the app-level is caching SRE_Pattern objects,
+    # so that we don't need to do it here.  Creating new SRE_Pattern
+    # objects all the time would be bad for the JIT, which relies on the
+    # identity of the CompiledPattern() object.
+    srepat.code = rsre_core.CompiledPattern(code)
     srepat.num_groups = groups
     srepat.w_groupindex = w_groupindex
     srepat.w_indexgroup = w_indexgroup
@@ -610,10 +615,11 @@
 # Our version is also directly iterable, to make finditer() easier.
 
 class W_SRE_Scanner(W_Root):
-    def __init__(self, pattern, ctx):
+    def __init__(self, pattern, ctx, code):
         self.space = pattern.space
         self.srepat = pattern
         self.ctx = ctx
+        self.code = code
         # 'self.ctx' is always a fresh context in which no searching
         # or matching succeeded so far.
 
@@ -623,19 +629,19 @@
     def next_w(self):
         if self.ctx.match_start > self.ctx.end:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        if not searchcontext(self.space, self.ctx):
+        if not searchcontext(self.space, self.ctx, self.code):
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
         return self.getmatch(True)
 
     def match_w(self):
         if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
-        return self.getmatch(matchcontext(self.space, self.ctx))
+        return self.getmatch(matchcontext(self.space, self.ctx, self.code))
 
     def search_w(self):
         if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
-        return self.getmatch(searchcontext(self.space, self.ctx))
+        return self.getmatch(searchcontext(self.space, self.ctx, self.code))
 
     def getmatch(self, found):
         if found:
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -133,6 +133,11 @@
              'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3
     constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name)
 
+# PyPy-specific flags
+for name in ('FLOAT',):
+    constant_names.append('Py_TPPYPYFLAGS_%s_SUBCLASS' % name)
+
+
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
 globals().update(rffi_platform.configure(CConfig_constants))
diff --git a/pypy/module/cpyext/boolobject.py b/pypy/module/cpyext/boolobject.py
--- a/pypy/module/cpyext/boolobject.py
+++ b/pypy/module/cpyext/boolobject.py
@@ -1,9 +1,5 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
-from pypy.module.cpyext.api import (cpython_api, PyObject, CANNOT_FAIL,
-                                    build_type_checkers)
-
-# Inheriting from bool isn't actually possible.
-PyBool_Check = build_type_checkers("Bool")[1]
+from rpython.rtyper.lltypesystem import rffi
+from pypy.module.cpyext.api import cpython_api, PyObject
 
 @cpython_api([rffi.LONG], PyObject)
 def PyBool_FromLong(space, value):
diff --git a/pypy/module/cpyext/floatobject.py b/pypy/module/cpyext/floatobject.py
--- a/pypy/module/cpyext/floatobject.py
+++ b/pypy/module/cpyext/floatobject.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (PyObjectFields, bootstrap_function,
     cpython_struct,
-    CANNOT_FAIL, cpython_api, PyObject, build_type_checkers, CONST_STRING)
+    CANNOT_FAIL, cpython_api, PyObject, CONST_STRING)
 from pypy.module.cpyext.pyobject import (
     make_typedescr, track_reference, from_ref)
 from pypy.interpreter.error import OperationError
@@ -38,8 +38,6 @@
     track_reference(space, obj, w_obj)
     return w_obj
 
-PyFloat_Check, PyFloat_CheckExact = build_type_checkers("Float")
-
 @cpython_api([lltype.Float], PyObject)
 def PyFloat_FromDouble(space, value):
     return space.newfloat(value)
diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py
--- a/pypy/module/cpyext/frameobject.py
+++ b/pypy/module/cpyext/frameobject.py
@@ -82,10 +82,10 @@
 def PyTraceBack_Here(space, w_frame):
     from pypy.interpreter.pytraceback import record_application_traceback
     state = space.fromcache(State)
-    if state.operror is None:
+    if state.get_exception() is None:
         return -1
     frame = space.interp_w(PyFrame, w_frame)
-    record_application_traceback(space, state.operror, frame, 0)
+    record_application_traceback(space, state.get_exception(), frame, 0)
     return 0
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
diff --git a/pypy/module/cpyext/include/boolobject.h b/pypy/module/cpyext/include/boolobject.h
--- a/pypy/module/cpyext/include/boolobject.h
+++ b/pypy/module/cpyext/include/boolobject.h
@@ -16,6 +16,8 @@
 #define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True
 #define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False
 
+#define PyBool_Check(op) ((op)->ob_type == &PyBool_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h
--- a/pypy/module/cpyext/include/floatobject.h
+++ b/pypy/module/cpyext/include/floatobject.h
@@ -32,6 +32,11 @@
                 return PyFloat_FromDouble(-Py_HUGE_VAL);        \
         } while(0)
 
+#define PyFloat_Check(op) \
+		 _PyPy_Type_FastSubclass((op)->ob_type, Py_TPPYPYFLAGS_FLOAT_SUBCLASS)
+#define PyFloat_CheckExact(op) ((op)->ob_type == &PyFloat_Type)
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -228,6 +228,11 @@
 #define Py_TPFLAGS_BASE_EXC_SUBCLASS	(1L<<30)
 #define Py_TPFLAGS_TYPE_SUBCLASS	(1L<<31)
 
+/* These are conceptually the same as the flags above, but they are
+   PyPy-specific and are stored inside tp_pypy_flags */
+#define Py_TPPYPYFLAGS_FLOAT_SUBCLASS (1L<<0)
+
+    
 #define Py_TPFLAGS_DEFAULT_EXTERNAL ( \
                              Py_TPFLAGS_HAVE_GETCHARBUFFER | \
                              Py_TPFLAGS_HAVE_SEQUENCE_IN | \
@@ -247,6 +252,8 @@
 #define PyType_HasFeature(t,f)  (((t)->tp_flags & (f)) != 0)
 #define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
 
+#define _PyPy_Type_FastSubclass(t,f) (((t)->tp_pypy_flags & (f)) != 0)
+    
 #define PyType_Check(op) \
     PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
 #define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type)
diff --git a/pypy/module/cpyext/include/sliceobject.h b/pypy/module/cpyext/include/sliceobject.h
--- a/pypy/module/cpyext/include/sliceobject.h
+++ b/pypy/module/cpyext/include/sliceobject.h
@@ -17,6 +17,8 @@
     PyObject *step;
 } PySliceObject;
 
+#define PySlice_Check(op) ((op)->ob_type == &PySlice_Type)
+    
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -45,6 +45,18 @@
     from pypy.module.cpyext.object import _dealloc
     _dealloc(space, py_obj)
 
+def w_kwargs_from_args(space, __args__):
+    w_kwargs = None
+    if __args__.keywords:
+        # CCC: we should probably have a @jit.look_inside_iff if the
+        # keyword count is constant, as we do in Arguments.unpack
+        w_kwargs = space.newdict()
+        for i in range(len(__args__.keywords)):
+            key = __args__.keywords[i]
+            w_obj = __args__.keywords_w[i]
+            space.setitem(w_kwargs, space.newtext(key), w_obj)
+    return w_kwargs
+
 class W_PyCFunctionObject(W_Root):
     _immutable_fields_ = ["flags"]
 
@@ -103,15 +115,7 @@
     def call_keywords(self, space, w_self, __args__):
         func = rffi.cast(PyCFunctionKwArgs, self.ml.c_ml_meth)
         py_args = tuple_from_args_w(space, __args__.arguments_w)
-        w_kwargs = None
-        if __args__.keywords:
-            # CCC: we should probably have a @jit.look_inside_iff if the
-            # keyword count is constant, as we do in Arguments.unpack
-            w_kwargs = space.newdict()
-            for i in range(len(__args__.keywords)):
-                key = __args__.keywords[i]
-                w_obj = __args__.keywords_w[i]
-                space.setitem(w_kwargs, space.newtext(key), w_obj)
+        w_kwargs = w_kwargs_from_args(space, __args__)
         try:
             return generic_cpy_call(space, func, w_self, py_args, w_kwargs)
         finally:
@@ -213,14 +217,15 @@
                             (self.name, self.w_objclass.getname(self.space)))
 
 
+class W_PyCWrapperObject(W_Root):
+    """
+    Abstract class; for concrete subclasses, see slotdefs.py
+    """
+    _immutable_fields_ = ['offset[*]']
 
-class W_PyCWrapperObject(W_Root):
-    def __init__(self, space, pto, method_name, wrapper_func,
-                 wrapper_func_kwds, doc, func, offset=None):
+    def __init__(self, space, pto, method_name, doc, func, offset):
         self.space = space
         self.method_name = method_name
-        self.wrapper_func = wrapper_func
-        self.wrapper_func_kwds = wrapper_func_kwds
         self.doc = doc
         self.func = func
         self.offset = offset
@@ -229,10 +234,17 @@
         assert isinstance(w_type, W_TypeObject)
         self.w_objclass = w_type
 
-    def call(self, space, w_self, w_args, w_kw):
+    def descr_call(self, space, w_self, __args__):
+        return self.call(space, w_self, __args__)
+
+    def call(self, space, w_self, __args__):
+        raise NotImplementedError
+
+    @jit.unroll_safe
+    def get_func_to_call(self):
         func_to_call = self.func
         if self.offset:
-            pto = as_pyobj(space, self.w_objclass)
+            pto = as_pyobj(self.space, self.w_objclass)
             # make ptr the equivalent of this, using the offsets
             #func_to_call = rffi.cast(rffi.VOIDP, ptr.c_tp_as_number.c_nb_multiply)
             if pto:
@@ -246,31 +258,33 @@
                 assert False, "failed to convert w_type %s to PyObject" % str(
                                                               self.w_objclass)
         assert func_to_call
-        if self.wrapper_func is None:
-            assert self.wrapper_func_kwds is not None
-            return self.wrapper_func_kwds(space, w_self, w_args, func_to_call,
-                                          w_kw)
-        if space.is_true(w_kw):
-            raise oefmt(space.w_TypeError,
+        return func_to_call
+
+    def check_args(self, __args__, arity):
+        length = len(__args__.arguments_w)
+        if length != arity:
+            raise oefmt(self.space.w_TypeError, "expected %d arguments, got %d",
+                        arity, length)
+        if __args__.keywords:
+            raise oefmt(self.space.w_TypeError,
                         "wrapper %s doesn't take any keyword arguments",
                         self.method_name)
-        return self.wrapper_func(space, w_self, w_args, func_to_call)
+
+    def check_argsv(self, __args__, min, max):
+        length = len(__args__.arguments_w)
+        if not min <= length <= max:
+            raise oefmt(self.space.w_TypeError, "expected %d-%d arguments, got %d",
+                        min, max, length)
+        if __args__.keywords:
+            raise oefmt(self.space.w_TypeError,
+                        "wrapper %s doesn't take any keyword arguments",
+                        self.method_name)
 
     def descr_method_repr(self):
         return self.space.newtext("" %
                                   (self.method_name,
                                    self.w_objclass.name))
 
- at jit.dont_look_inside
-def cwrapper_descr_call(space, w_self, __args__):
-    self = space.interp_w(W_PyCWrapperObject, w_self)
-    args_w, kw_w = __args__.unpack()
-    w_args = space.newtuple(args_w[1:])
-    w_self = args_w[0]
-    w_kw = space.newdict()
-    for key, w_obj in kw_w.items():
-        space.setitem(w_kw, space.newtext(key), w_obj)
-    return self.call(space, w_self, w_args, w_kw)
 
 def cmethod_descr_get(space, w_function, w_obj, w_cls=None):
     asking_for_bound = (space.is_none(w_cls) or
@@ -323,7 +337,7 @@
 
 W_PyCWrapperObject.typedef = TypeDef(
     'wrapper_descriptor',
-    __call__ = interp2app(cwrapper_descr_call),
+    __call__ = interp2app(W_PyCWrapperObject.descr_call),
     __get__ = interp2app(cmethod_descr_get),
     __name__ = interp_attrproperty('method_name', cls=W_PyCWrapperObject,
         wrapfn="newtext_or_none"),
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -311,6 +311,10 @@
 	/* Type attribute cache version tag. Added in version 2.6 */
 	unsigned int tp_version_tag;
 
+    /* PyPy specific extra fields: make sure that they are ALWAYS at the end,
+       for compatibility with CPython */
+    long tp_pypy_flags;
+
 } PyTypeObject;
 
 typedef struct _heaptypeobject {
diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -31,9 +31,10 @@
 @cpython_api([], PyObject, result_borrowed=True)
 def PyErr_Occurred(space):
     state = space.fromcache(State)
-    if state.operror is None:
+    operror = state.get_exception()
+    if operror is None:
         return None
-    return state.operror.w_type     # borrowed ref
+    return operror.w_type     # borrowed ref
 
 @cpython_api([], lltype.Void)
 def PyErr_Clear(space):
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -5,7 +5,8 @@
 from pypy.objspace.std.listobject import (
     ListStrategy, UNROLL_CUTOFF, W_ListObject, ObjectListStrategy)
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP)
+    cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t, PyObject, PyObjectP,
+    generic_cpy_call)
 from pypy.module.cpyext.pyobject import PyObject, make_ref, from_ref
 from pypy.module.cpyext.pyobject import as_pyobj, incref
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -145,21 +146,26 @@
     # XXX we should call Py*_GET_ITEM() instead of Py*_GetItem()
     # from here, but we cannot because we are also called from
     # PySequence_GetItem()
+    py_obj = as_pyobj(space, w_obj)
     if isinstance(w_obj, tupleobject.W_TupleObject):
         from pypy.module.cpyext.tupleobject import PyTuple_GetItem
-        py_obj = as_pyobj(space, w_obj)
         py_res = PyTuple_GetItem(space, py_obj, i)
         incref(space, py_res)
         keepalive_until_here(w_obj)
         return py_res
     if isinstance(w_obj, W_ListObject):
         from pypy.module.cpyext.listobject import PyList_GetItem
-        py_obj = as_pyobj(space, w_obj)
         py_res = PyList_GetItem(space, py_obj, i)
         incref(space, py_res)
         keepalive_until_here(w_obj)
         return py_res
-    return make_ref(space, space.getitem(w_obj, space.newint(i)))
+    
+    as_sequence = py_obj.c_ob_type.c_tp_as_sequence
+    if not as_sequence or not as_sequence.c_sq_item:
+        raise oefmt(space.w_TypeError,
+                    "'%T' object does not support indexing", w_obj)
+    ret = generic_cpy_call(space, as_sequence.c_sq_item, w_obj, i)
+    return make_ref(space, ret)
 
 @cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PySequence_GetItem(space, w_obj, i):
diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py
--- a/pypy/module/cpyext/sliceobject.py
+++ b/pypy/module/cpyext/sliceobject.py
@@ -47,7 +47,6 @@
     from pypy.module.cpyext.object import _dealloc
     _dealloc(space, py_obj)
 
-PySlice_Check, PySlice_CheckExact = build_type_checkers("Slice")
 
 @cpython_api([PyObject, PyObject, PyObject], PyObject)
 def PySlice_New(space, w_start, w_stop, w_step):
@@ -75,9 +74,8 @@
     normal slices.
 
     Returns 0 on success and -1 on error with exception set."""
-    if not PySlice_Check(space, w_slice):
+    if not isinstance(w_slice, W_SliceObject):
         PyErr_BadInternalCall(space)
-    assert isinstance(w_slice, W_SliceObject)
     start_p[0], stop_p[0], step_p[0], slicelength_p[0] = \
             w_slice.indices4(space, length)
     return 0
@@ -97,9 +95,8 @@
     objects in versions of Python prior to 2.3, you would probably do well to
     incorporate the source of PySlice_GetIndicesEx(), suitably renamed,
     in the source of your extension."""
-    if not PySlice_Check(space, w_slice):
+    if not isinstance(w_slice, W_SliceObject):
         PyErr_BadInternalCall(space)
-    assert isinstance(w_slice, W_SliceObject)
     start_p[0], stop_p[0], step_p[0] = \
             w_slice.indices3(space, length)
     return 0
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -19,6 +19,8 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import userslot
 from pypy.module.cpyext.buffer import CBuffer, CPyBuffer, fq
+from pypy.module.cpyext.methodobject import (W_PyCWrapperObject, tuple_from_args_w,
+                                             w_kwargs_from_args)
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
 from rpython.rlib.unroll import unrolling_iterable
@@ -38,29 +40,6 @@
 Py_GT = 4
 Py_GE = 5
 
-
-def check_num_args(space, w_ob, n):
-    from pypy.module.cpyext.tupleobject import PyTuple_CheckExact
-    if not PyTuple_CheckExact(space, w_ob):
-        raise oefmt(space.w_SystemError,
-                    "PyArg_UnpackTuple() argument list is not a tuple")
-    if n == space.len_w(w_ob):
-        return
-    raise oefmt(space.w_TypeError,
-                "expected %d arguments, got %d",
-                n, space.len_w(w_ob))
-
-def check_num_argsv(space, w_ob, low, high):
-    from pypy.module.cpyext.tupleobject import PyTuple_CheckExact
-    if not PyTuple_CheckExact(space, w_ob):
-        raise oefmt(space.w_SystemError,
-                    "PyArg_UnpackTuple() argument list is not a tuple")
-    if low <=space.len_w(w_ob) <= high:
-        return
-    raise oefmt(space.w_TypeError,
-                "expected %d-%d arguments, got %d",
-                low, high, space.len_w(w_ob))
-
 @not_rpython
 def llslot(space, func):
     return func.api_func.get_llhelper(space)
@@ -71,337 +50,413 @@
     get_llhelper = v_func.value.api_func.get_llhelper
     return ctx.appcall(get_llhelper, v_space)
 
+# NOTE: the following wrap_* are subclasses of W_PyCWrapperObject, even if
+# they don't follow the usual W_* naming convention for subclasses of W_Root:
+# we do this because we automatically generate most of the slots from the
+# CPython code copy&pasted inside slotdefs_str, and thus we need to keep the
+# same names as they are used in C.
 
-def wrap_init(space, w_self, w_args, func, w_kwargs):
-    func_init = rffi.cast(initproc, func)
-    res = generic_cpy_call(space, func_init, w_self, w_args, w_kwargs)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return None
+class wrap_init(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        func = self.get_func_to_call()
+        func_init = rffi.cast(initproc, func)
+        py_args = tuple_from_args_w(space, __args__.arguments_w)
+        w_kwargs = w_kwargs_from_args(space, __args__)
+        res = generic_cpy_call(space, func_init, w_self, py_args, w_kwargs)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return None
 
-def wrap_unaryfunc(space, w_self, w_args, func):
-    func_unary = rffi.cast(unaryfunc, func)
-    check_num_args(space, w_args, 0)
-    return generic_cpy_call(space, func_unary, w_self)
+class wrap_unaryfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 0)
+        func = self.get_func_to_call()
+        func_unary = rffi.cast(unaryfunc, func)
+        return generic_cpy_call(space, func_unary, w_self)
 
-def wrap_binaryfunc(space, w_self, w_args, func):
-    func_binary = rffi.cast(binaryfunc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    return generic_cpy_call(space, func_binary, w_self, args_w[0])
+class wrap_binaryfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_binary = rffi.cast(binaryfunc, func)
+        w_x = __args__.arguments_w[0]
+        return generic_cpy_call(space, func_binary, w_self, w_x)
 
 def _get_ob_type(space, w_obj):
     # please ensure that w_obj stays alive
     ob_type = as_pyobj(space, space.type(w_obj))
     return rffi.cast(PyTypeObjectPtr, ob_type)
 
-def wrap_binaryfunc_l(space, w_self, w_args, func):
-    func_binary = rffi.cast(binaryfunc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    ob_type = _get_ob_type(space, w_self)
-    if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
-        not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
-        return space.w_NotImplemented
-    return generic_cpy_call(space, func_binary, w_self, args_w[0])
+class wrap_binaryfunc_l(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_binary = rffi.cast(binaryfunc, func)
+        w_value = __args__.arguments_w[0]
+        ob_type = _get_ob_type(space, w_self)
+        if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
+            not space.issubtype_w(space.type(w_value), space.type(w_self))):
+            return space.w_NotImplemented
+        return generic_cpy_call(space, func_binary, w_self, w_value)
 
-def wrap_binaryfunc_r(space, w_self, w_args, func):
-    func_binary = rffi.cast(binaryfunc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    ob_type = _get_ob_type(space, w_self)
-    if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
-        not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
-        return space.w_NotImplemented
-    return generic_cpy_call(space, func_binary, args_w[0], w_self)
+class wrap_binaryfunc_r(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_binary = rffi.cast(binaryfunc, func)
+        w_value = __args__.arguments_w[0]
+        ob_type = _get_ob_type(space, w_self)
+        if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
+            not space.issubtype_w(space.type(w_value), space.type(w_self))):
+            return space.w_NotImplemented
+        return generic_cpy_call(space, func_binary, w_value, w_self)
 
-def wrap_ternaryfunc(space, w_self, w_args, func):
-    # The third argument is optional
-    func_ternary = rffi.cast(ternaryfunc, func)
-    check_num_argsv(space, w_args, 1, 2)
-    args_w = space.fixedview(w_args)
-    arg3 = space.w_None
-    if len(args_w) > 1:
-        arg3 = args_w[1]
-    return generic_cpy_call(space, func_ternary, w_self, args_w[0], arg3)
+class wrap_ternaryfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        # The third argument is optional
+        self.check_argsv(__args__, 1, 2)
+        func = self.get_func_to_call()
+        func_ternary = rffi.cast(ternaryfunc, func)
+        w_arg0 = __args__.arguments_w[0]
+        if len(__args__.arguments_w) == 2:
+            w_arg1 = __args__.arguments_w[1]
+        else:
+            w_arg1 = space.w_None
+        return generic_cpy_call(space, func_ternary, w_self, w_arg0, w_arg1)
 
-def wrap_ternaryfunc_r(space, w_self, w_args, func):
-    # The third argument is optional
-    func_ternary = rffi.cast(ternaryfunc, func)
-    check_num_argsv(space, w_args, 1, 2)
-    args_w = space.fixedview(w_args)
-    ob_type = _get_ob_type(space, w_self)
-    if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
-        not space.issubtype_w(space.type(args_w[0]), space.type(w_self))):
-        return space.w_NotImplemented
-    arg3 = space.w_None
-    if len(args_w) > 1:
-        arg3 = args_w[1]
-    return generic_cpy_call(space, func_ternary, args_w[0], w_self, arg3)
+class wrap_ternaryfunc_r(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):    
+        # The third argument is optional
+        self.check_argsv(__args__, 1, 2)
+        func = self.get_func_to_call()
+        func_ternary = rffi.cast(ternaryfunc, func)
+        w_arg0 = __args__.arguments_w[0]
+        if len(__args__.arguments_w) == 2:
+            w_arg1 = __args__.arguments_w[1]
+        else:
+            w_arg1 = space.w_None
+        ob_type = _get_ob_type(space, w_self)
+        if (not ob_type.c_tp_flags & Py_TPFLAGS_CHECKTYPES and
+            not space.issubtype_w(space.type(w_arg0), space.type(w_self))):
+            return space.w_NotImplemented
+        return generic_cpy_call(space, func_ternary, w_arg0, w_self, w_arg1)
 
+class wrap_inquirypred(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 0)
+        func = self.get_func_to_call()
+        func_inquiry = rffi.cast(inquiry, func)
+        res = generic_cpy_call(space, func_inquiry, w_self)
+        res = rffi.cast(lltype.Signed, res)
+        if res == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.newbool(bool(res))
 
-def wrap_inquirypred(space, w_self, w_args, func):
-    func_inquiry = rffi.cast(inquiry, func)
-    check_num_args(space, w_args, 0)
-    res = generic_cpy_call(space, func_inquiry, w_self)
-    res = rffi.cast(lltype.Signed, res)
-    if res == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.newbool(bool(res))
+class wrap_getattr(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(getattrfunc, func)
+        w_name = __args__.arguments_w[0]
+        name_ptr = rffi.str2charp(space.text_w(w_name))
+        try:
+            return generic_cpy_call(space, func_target, w_self, name_ptr)
+        finally:
+            rffi.free_charp(name_ptr)
 
-def wrap_getattr(space, w_self, w_args, func):
-    func_target = rffi.cast(getattrfunc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    name_ptr = rffi.str2charp(space.text_w(args_w[0]))
-    try:
-        return generic_cpy_call(space, func_target, w_self, name_ptr)
-    finally:
-        rffi.free_charp(name_ptr)
+class wrap_getattro(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(getattrofunc, func)
+        w_name = __args__.arguments_w[0]
+        return generic_cpy_call(space, func_target, w_self, w_name)
 
-def wrap_getattro(space, w_self, w_args, func):
-    func_target = rffi.cast(getattrofunc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    return generic_cpy_call(space, func_target, w_self, args_w[0])
+class wrap_setattr(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 2)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(setattrofunc, func)
+        w_name = __args__.arguments_w[0]
+        w_value = __args__.arguments_w[1]
+        # XXX "Carlo Verre hack"?
+        res = generic_cpy_call(space, func_target, w_self, w_name, w_value)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
-def wrap_setattr(space, w_self, w_args, func):
-    func_target = rffi.cast(setattrofunc, func)
-    check_num_args(space, w_args, 2)
-    w_name, w_value = space.fixedview(w_args)
-    # XXX "Carlo Verre hack"?
-    res = generic_cpy_call(space, func_target, w_self, w_name, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+class wrap_delattr(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(setattrofunc, func)
+        w_name = __args__.arguments_w[0]
+        # XXX "Carlo Verre hack"?
+        res = generic_cpy_call(space, func_target, w_self, w_name, None)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
-def wrap_delattr(space, w_self, w_args, func):
-    func_target = rffi.cast(setattrofunc, func)
-    check_num_args(space, w_args, 1)
-    w_name, = space.fixedview(w_args)
-    # XXX "Carlo Verre hack"?
-    res = generic_cpy_call(space, func_target, w_self, w_name, None)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+class wrap_descr_get(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        func = self.get_func_to_call()
+        func_target = rffi.cast(descrgetfunc, func)
+        length = len(__args__.arguments_w)
+        if length == 1:
+            w_obj = __args__.arguments_w[0]
+            w_type = None
+        elif length == 2:
+            w_obj = __args__.arguments_w[0]
+            w_type = __args__.arguments_w[1]
+        else:
+            raise oefmt(space.w_TypeError,
+                        "expected 1 or 2 arguments, got %d", len(__args__.arguments_w))
+        if w_obj is space.w_None:
+            w_obj = None
+        if w_type is space.w_None:
+            w_type = None
+        if w_obj is None and w_type is None:
+            raise oefmt(space.w_TypeError, "__get__(None, None) is invalid")
+        return generic_cpy_call(space, func_target, w_self, w_obj, w_type)
 
-def wrap_descr_get(space, w_self, w_args, func):
-    func_target = rffi.cast(descrgetfunc, func)
-    args_w = space.fixedview(w_args)
-    if len(args_w) == 1:
-        w_obj, = args_w
-        w_type = None
-    elif len(args_w) == 2:
-        w_obj, w_type = args_w
-    else:
-        raise oefmt(space.w_TypeError,
-                    "expected 1 or 2 arguments, got %d", len(args_w))
-    if w_obj is space.w_None:
-        w_obj = None
-    if w_type is space.w_None:
-        w_type = None
-    if w_obj is None and w_type is None:
-        raise oefmt(space.w_TypeError, "__get__(None, None) is invalid")
-    return generic_cpy_call(space, func_target, w_self, w_obj, w_type)
+class wrap_descr_set(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 2)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(descrsetfunc, func)
+        w_obj = __args__.arguments_w[0]
+        w_value = __args__.arguments_w[1]
+        res = generic_cpy_call(space, func_target, w_self, w_obj, w_value)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
-def wrap_descr_set(space, w_self, w_args, func):
-    func_target = rffi.cast(descrsetfunc, func)
-    check_num_args(space, w_args, 2)
-    w_obj, w_value = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_obj, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+class wrap_descr_delete(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(descrsetfunc, func)
+        w_obj = __args__.arguments_w[0]
+        res = generic_cpy_call(space, func_target, w_self, w_obj, None)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
-def wrap_descr_delete(space, w_self, w_args, func):
-    func_target = rffi.cast(descrsetfunc, func)
-    check_num_args(space, w_args, 1)
-    w_obj, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_obj, None)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+class wrap_call(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        func = self.get_func_to_call()
+        func_target = rffi.cast(ternaryfunc, func)
+        py_args = tuple_from_args_w(space, __args__.arguments_w)
+        w_kwargs = w_kwargs_from_args(space, __args__)
+        return generic_cpy_call(space, func_target, w_self, py_args, w_kwargs)
 
-def wrap_call(space, w_self, w_args, func, w_kwds):
-    func_target = rffi.cast(ternaryfunc, func)
-    return generic_cpy_call(space, func_target, w_self, w_args, w_kwds)
+class wrap_ssizessizeobjargproc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 3)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(ssizessizeobjargproc, func)
+        i = space.int_w(space.index(__args__.arguments_w[0]))
+        j = space.int_w(space.index(__args__.arguments_w[1]))
+        w_y = __args__.arguments_w[2]
+        res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
-def wrap_ssizessizeobjargproc(space, w_self, w_args, func):
-    func_target = rffi.cast(ssizessizeobjargproc, func)
-    check_num_args(space, w_args, 3)
-    args_w = space.fixedview(w_args)
-    i = space.int_w(space.index(args_w[0]))
-    j = space.int_w(space.index(args_w[1]))
-    w_y = args_w[2]
-    res = generic_cpy_call(space, func_target, w_self, i, j, w_y)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+class wrap_lenfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 0)
+        func = self.get_func_to_call()
+        func_len = rffi.cast(lenfunc, func)
+        res = generic_cpy_call(space, func_len, w_self)
+        if widen(res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.newint(res)
 
-def wrap_lenfunc(space, w_self, w_args, func):
-    func_len = rffi.cast(lenfunc, func)
-    check_num_args(space, w_args, 0)
-    res = generic_cpy_call(space, func_len, w_self)
-    if widen(res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.newint(res)
+class wrap_sq_item(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(ssizeargfunc, func)
+        w_index = __args__.arguments_w[0]
+        index = space.int_w(space.index(w_index))
+        return generic_cpy_call(space, func_target, w_self, index)
 
-def wrap_sq_item(space, w_self, w_args, func):
-    func_target = rffi.cast(ssizeargfunc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    index = space.int_w(space.index(args_w[0]))
-    return generic_cpy_call(space, func_target, w_self, index)
+class wrap_sq_setitem(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 2)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(ssizeobjargproc, func)
+        w_index = __args__.arguments_w[0]
+        w_value = __args__.arguments_w[1]
+        index = space.int_w(space.index(w_index))
+        res = generic_cpy_call(space, func_target, w_self, index, w_value)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
-def wrap_sq_setitem(space, w_self, w_args, func):
-    func_target = rffi.cast(ssizeobjargproc, func)
-    check_num_args(space, w_args, 2)
-    args_w = space.fixedview(w_args)
-    index = space.int_w(space.index(args_w[0]))
-    res = generic_cpy_call(space, func_target, w_self, index, args_w[1])
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-
-def wrap_sq_delitem(space, w_self, w_args, func):
-    func_target = rffi.cast(ssizeobjargproc, func)
-    check_num_args(space, w_args, 1)
-    args_w = space.fixedview(w_args)
-    index = space.int_w(space.index(args_w[0]))
-    null = rffi.cast(PyObject, 0)
-    res = generic_cpy_call(space, func_target, w_self, index, null)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
+class wrap_sq_delitem(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(ssizeobjargproc, func)
+        w_index = __args__.arguments_w[0]
+        index = space.int_w(space.index(w_index))
+        null = rffi.cast(PyObject, 0)
+        res = generic_cpy_call(space, func_target, w_self, index, null)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
 
 # Warning, confusing function name (like CPython).  Used only for sq_contains.
-def wrap_objobjproc(space, w_self, w_args, func):
-    func_target = rffi.cast(objobjproc, func)
-    check_num_args(space, w_args, 1)
-    w_value, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_value)
-    res = rffi.cast(lltype.Signed, res)
-    if res == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.newbool(bool(res))
+class wrap_objobjproc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(objobjproc, func)
+        w_value = __args__.arguments_w[0]
+        res = generic_cpy_call(space, func_target, w_self, w_value)
+        res = rffi.cast(lltype.Signed, res)
+        if res == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.newbool(bool(res))
 
-def wrap_objobjargproc(space, w_self, w_args, func):
-    func_target = rffi.cast(objobjargproc, func)
-    check_num_args(space, w_args, 2)
-    w_key, w_value = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.w_None
+class wrap_objobjargproc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 2)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(objobjargproc, func)
+        w_key = __args__.arguments_w[0]
+        w_value = __args__.arguments_w[1]
+        res = generic_cpy_call(space, func_target, w_self, w_key, w_value)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.w_None
 
-def wrap_delitem(space, w_self, w_args, func):
-    func_target = rffi.cast(objobjargproc, func)
-    check_num_args(space, w_args, 1)
-    w_key, = space.fixedview(w_args)
-    null = rffi.cast(PyObject, 0)
-    res = generic_cpy_call(space, func_target, w_self, w_key, null)
-    if rffi.cast(lltype.Signed, res) == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.w_None
+class wrap_delitem(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(objobjargproc, func)
+        w_key = __args__.arguments_w[0]
+        null = rffi.cast(PyObject, 0)
+        res = generic_cpy_call(space, func_target, w_self, w_key, null)
+        if rffi.cast(lltype.Signed, res) == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.w_None
 
-def wrap_ssizessizeargfunc(space, w_self, w_args, func):
-    func_target = rffi.cast(ssizessizeargfunc, func)
-    check_num_args(space, w_args, 2)
-    args_w = space.fixedview(w_args)
-    start = space.int_w(args_w[0])
-    end = space.int_w(args_w[1])
-    return generic_cpy_call(space, func_target, w_self, start, end)
+class wrap_ssizessizeargfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 2)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(ssizessizeargfunc, func)
+        start = space.int_w(__args__.arguments_w[0])
+        end = space.int_w(__args__.arguments_w[1])
+        return generic_cpy_call(space, func_target, w_self, start, end)
 
-def wrap_next(space, w_self, w_args, func):
-    from pypy.module.cpyext.api import generic_cpy_call_expect_null
-    func_target = rffi.cast(iternextfunc, func)
-    check_num_args(space, w_args, 0)
-    w_res = generic_cpy_call_expect_null(space, func_target, w_self)
-    if not w_res and not PyErr_Occurred(space):
-        raise OperationError(space.w_StopIteration, space.w_None)
-    return w_res
+class wrap_next(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        from pypy.module.cpyext.api import generic_cpy_call_expect_null
+        self.check_args(__args__, 0)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(iternextfunc, func)
+        w_res = generic_cpy_call_expect_null(space, func_target, w_self)
+        if not w_res and not PyErr_Occurred(space):
+            raise OperationError(space.w_StopIteration, space.w_None)
+        return w_res
 
-def wrap_hashfunc(space, w_self, w_args, func):
-    func_target = rffi.cast(hashfunc, func)
-    check_num_args(space, w_args, 0)
-    res = generic_cpy_call(space, func_target, w_self)
-    if res == -1:
-        space.fromcache(State).check_and_raise_exception(always=True)
-    return space.newint(res)
+class wrap_hashfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 0)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(hashfunc, func)
+        res = generic_cpy_call(space, func_target, w_self)
+        if res == -1:
+            space.fromcache(State).check_and_raise_exception(always=True)
+        return space.newint(res)
 
-def wrap_getreadbuffer(space, w_self, w_args, func):
-    func_target = rffi.cast(readbufferproc, func)
-    py_type = _get_ob_type(space, w_self)
-    rbp = rffi.cast(rffi.VOIDP, 0)
-    if py_type.c_tp_as_buffer:
-        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
-    with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
-        index = rffi.cast(Py_ssize_t, 0)
-        size = generic_cpy_call(space, func_target, w_self, index, ptr)
-        if size < 0:
-            space.fromcache(State).check_and_raise_exception(always=True)
-        view = CPyBuffer(space, ptr[0], size, w_self,
-                               releasebufferproc=rbp)
-        fq.register_finalizer(view)
-        return space.newbuffer(CBuffer(view))
+class wrap_getreadbuffer(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        func = self.get_func_to_call()
+        func_target = rffi.cast(readbufferproc, func)
+        py_type = _get_ob_type(space, w_self)
+        rbp = rffi.cast(rffi.VOIDP, 0)
+        if py_type.c_tp_as_buffer:
+            rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
+            index = rffi.cast(Py_ssize_t, 0)
+            size = generic_cpy_call(space, func_target, w_self, index, ptr)
+            if size < 0:
+                space.fromcache(State).check_and_raise_exception(always=True)
+            view = CPyBuffer(space, ptr[0], size, w_self,
+                                   releasebufferproc=rbp)
+            fq.register_finalizer(view)
+            return space.newbuffer(CBuffer(view))
 
-def wrap_getwritebuffer(space, w_self, w_args, func):
-    func_target = rffi.cast(readbufferproc, func)
-    py_type = _get_ob_type(space, w_self)
-    rbp = rffi.cast(rffi.VOIDP, 0)
-    if py_type.c_tp_as_buffer:
-        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
-    with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
-        index = rffi.cast(Py_ssize_t, 0)
-        size = generic_cpy_call(space, func_target, w_self, index, ptr)
-        if size < 0:
-            space.fromcache(State).check_and_raise_exception(always=True)
-        view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
-                               releasebufferproc=rbp)
-        fq.register_finalizer(view)
-        return space.newbuffer(CBuffer(view))
+class wrap_getwritebuffer(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        func = self.get_func_to_call()
+        func_target = rffi.cast(readbufferproc, func)
+        py_type = _get_ob_type(space, w_self)
+        rbp = rffi.cast(rffi.VOIDP, 0)
+        if py_type.c_tp_as_buffer:
+            rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        with lltype.scoped_alloc(rffi.VOIDPP.TO, 1) as ptr:
+            index = rffi.cast(Py_ssize_t, 0)
+            size = generic_cpy_call(space, func_target, w_self, index, ptr)
+            if size < 0:
+                space.fromcache(State).check_and_raise_exception(always=True)
+            view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
+                                   releasebufferproc=rbp)
+            fq.register_finalizer(view)
+            return space.newbuffer(CBuffer(view))
 
-def wrap_getbuffer(space, w_self, w_args, func):
-    func_target = rffi.cast(getbufferproc, func)
-    py_type = _get_ob_type(space, w_self)
-    rbp = rffi.cast(rffi.VOIDP, 0)
-    if py_type.c_tp_as_buffer:
-        rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
-    with lltype.scoped_alloc(Py_buffer) as pybuf:
-        _flags = 0
-        if space.len_w(w_args) > 0:
-            _flags = space.int_w(space.listview(w_args)[0])
-        flags = rffi.cast(rffi.INT_real,_flags)
-        size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
-        if widen(size) < 0:
-            space.fromcache(State).check_and_raise_exception(always=True)
-        ptr = pybuf.c_buf
-        size = pybuf.c_len
-        ndim = widen(pybuf.c_ndim)
-        shape = None
-        if pybuf.c_shape:
-            shape = [pybuf.c_shape[i]   for i in range(ndim)]
-        strides = None
-        if pybuf.c_strides:
-            strides = [pybuf.c_strides[i] for i in range(ndim)]
-        if pybuf.c_format:
-            format = rffi.charp2str(pybuf.c_format)
-        else:
-            format = 'B'
-        # the CPython docs mandates that you do an incref whenever you call
-        # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't
-        # leak we release the buffer:
-        # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer
-        buf = CPyBuffer(space, ptr, size, w_self, format=format,
-                            ndim=ndim, shape=shape, strides=strides,
-                            itemsize=pybuf.c_itemsize,
-                            readonly=widen(pybuf.c_readonly),
-                            needs_decref=True,
-                            releasebufferproc = rbp)
-        fq.register_finalizer(buf)
-        return buf.wrap(space)
+
+class wrap_getbuffer(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        func = self.get_func_to_call()
+        func_target = rffi.cast(getbufferproc, func)
+        py_type = _get_ob_type(space, w_self)
+        rbp = rffi.cast(rffi.VOIDP, 0)
+        if py_type.c_tp_as_buffer:
+            rbp = rffi.cast(rffi.VOIDP, py_type.c_tp_as_buffer.c_bf_releasebuffer)
+        with lltype.scoped_alloc(Py_buffer) as pybuf:
+            _flags = 0
+            if len(__args__.arguments_w) > 0:
+                _flags = space.int_w(__args__.arguments_w[0])
+            flags = rffi.cast(rffi.INT_real,_flags)
+            size = generic_cpy_call(space, func_target, w_self, pybuf, flags)
+            if widen(size) < 0:
+                space.fromcache(State).check_and_raise_exception(always=True)
+            ptr = pybuf.c_buf
+            size = pybuf.c_len
+            ndim = widen(pybuf.c_ndim)
+            shape = None
+            if pybuf.c_shape:
+                shape = [pybuf.c_shape[i]   for i in range(ndim)]
+            strides = None
+            if pybuf.c_strides:
+                strides = [pybuf.c_strides[i] for i in range(ndim)]
+            if pybuf.c_format:
+                format = rffi.charp2str(pybuf.c_format)
+            else:
+                format = 'B'
+            # the CPython docs mandates that you do an incref whenever you call
+            # bf_getbuffer; so, we pass needs_decref=True to ensure that we don't
+            # leak we release the buffer:
+            # https://docs.python.org/3.5/c-api/typeobj.html#c.PyBufferProcs.bf_getbuffer
+            buf = CPyBuffer(space, ptr, size, w_self, format=format,
+                                ndim=ndim, shape=shape, strides=strides,
+                                itemsize=pybuf.c_itemsize,
+                                readonly=widen(pybuf.c_readonly),
+                                needs_decref=True,
+                                releasebufferproc = rbp)
+            fq.register_finalizer(buf)
+            return buf.wrap(space)
 
 def get_richcmp_func(OP_CONST):
-    def inner(space, w_self, w_args, func):
-        func_target = rffi.cast(richcmpfunc, func)
-        check_num_args(space, w_args, 1)
-        w_other, = space.fixedview(w_args)
-        return generic_cpy_call(space, func_target,
-            w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST))
-    return inner
+    class wrap_richcmp(W_PyCWrapperObject):
+        def call(self, space, w_self, __args__):
+            self.check_args(__args__, 1)
+            func = self.get_func_to_call()
+            func_target = rffi.cast(richcmpfunc, func)
+            w_other = __args__.arguments_w[0]
+            return generic_cpy_call(space, func_target,
+                w_self, w_other, rffi.cast(rffi.INT_real, OP_CONST))
+    return wrap_richcmp
 
 richcmp_eq = get_richcmp_func(Py_EQ)
 richcmp_ne = get_richcmp_func(Py_NE)
@@ -410,17 +465,19 @@
 richcmp_gt = get_richcmp_func(Py_GT)
 richcmp_ge = get_richcmp_func(Py_GE)
 
-def wrap_cmpfunc(space, w_self, w_args, func):
-    func_target = rffi.cast(cmpfunc, func)
-    check_num_args(space, w_args, 1)
-    w_other, = space.fixedview(w_args)
+class wrap_cmpfunc(W_PyCWrapperObject):
+    def call(self, space, w_self, __args__):
+        self.check_args(__args__, 1)
+        func = self.get_func_to_call()
+        func_target = rffi.cast(cmpfunc, func)
+        w_other = __args__.arguments_w[0]
 
-    if not space.issubtype_w(space.type(w_self), space.type(w_other)):
-        raise oefmt(space.w_TypeError,
-                    "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'",
-                    w_self, w_self, w_other)
+        if not space.issubtype_w(space.type(w_self), space.type(w_other)):
+            raise oefmt(space.w_TypeError,
+                        "%T.__cmp__(x,y) requires y to be a '%T', not a '%T'",
+                        w_self, w_self, w_other)
 
-    return space.newint(generic_cpy_call(space, func_target, w_self, w_other))
+        return space.newint(generic_cpy_call(space, func_target, w_self, w_other))
 
 SLOT_FACTORIES = {}
 def slot_factory(tp_name):
@@ -804,9 +861,10 @@
 missing_wrappers = ['wrap_indexargfunc', 'wrap_delslice', 'wrap_coercefunc']
 for name in missing_wrappers:
     assert name not in globals()
-    def missing_wrapper(space, w_self, w_args, func):
-        print "cpyext: missing slot wrapper " + name
-        raise NotImplementedError("Slot wrapper " + name)
+    class missing_wrapper(W_PyCWrapperObject):
+        def call(self, space, w_self, __args__):
+            print "cpyext: missing slot wrapper " + name
+            raise NotImplementedError("Slot wrapper " + name)
     missing_wrapper.__name__ = name
     globals()[name] = missing_wrapper
 
@@ -836,13 +894,12 @@
 PyWrapperFlag_KEYWORDS = 1
 
 class TypeSlot:
-    def __init__(self, method_name, slot_name, function, wrapper1, wrapper2, doc):
+    def __init__(self, method_name, slot_name, function, wrapper, doc):
         self.method_name = method_name
         self.slot_name = slot_name
         self.slot_names = tuple(("c_" + slot_name).split("."))
         self.slot_func = function
-        self.wrapper_func = wrapper1
-        self.wrapper_func_kwds = wrapper2
+        self.wrapper_class = wrapper
         self.doc = doc
 
 # adapted from typeobject.c
@@ -863,13 +920,7 @@
 
     function = getattr(userslot, FUNCTION or '!missing', None)
     assert FLAGS == 0 or FLAGS == PyWrapperFlag_KEYWORDS
-    if FLAGS:
-        wrapper1 = None
-        wrapper2 = wrapper
-    else:
-        wrapper1 = wrapper
-        wrapper2 = None
-    return TypeSlot(NAME, SLOT, function, wrapper1, wrapper2, DOC)
+    return TypeSlot(NAME, SLOT, function, wrapper, DOC)
 
 def TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC):
     return FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, 0)
@@ -1158,7 +1209,7 @@
       x.slot_func.api_func if x.slot_func else None) for x in slotdefs])
 
 slotdefs_for_wrappers = unrolling_iterable(
-    [(x.method_name, x.slot_names, x.wrapper_func, x.wrapper_func_kwds, x.doc)
+    [(x.method_name, x.slot_names, x.wrapper_class, x.doc)
      for x in slotdefs])
 
 if __name__ == "__main__":
diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py
--- a/pypy/module/cpyext/state.py
+++ b/pypy/module/cpyext/state.py
@@ -2,11 +2,18 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter import executioncontext
+from pypy.interpreter.executioncontext import ExecutionContext
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.rdynload import DLLHANDLE
 from rpython.rlib import rawrefcount
 import sys
 
+
+# Keep track of exceptions raised in cpyext for a particular execution
+# context.
+ExecutionContext.cpyext_operror = None
+
+
 class State:
     def __init__(self, space):
         self.space = space
@@ -18,7 +25,8 @@
 
     def reset(self):
         from pypy.module.cpyext.modsupport import PyMethodDef
-        self.operror = None
+        ec = self.space.getexecutioncontext()
+        ec.cpyext_operror = None
         self.new_method_def = lltype.nullptr(PyMethodDef)
 
         # When importing a package, use this to keep track
@@ -37,17 +45,24 @@
 
     def set_exception(self, operror):
         self.clear_exception()
-        self.operror = operror
+        ec = self.space.getexecutioncontext()
+        ec.cpyext_operror = operror
 
     def clear_exception(self):
         """Clear the current exception state, and return the operror."""
-        operror = self.operror
-        self.operror = None
+        ec = self.space.getexecutioncontext()
+        operror = ec.cpyext_operror
+        ec.cpyext_operror = None
         return operror
 
+    def get_exception(self):
+        ec = self.space.getexecutioncontext()
+        return ec.cpyext_operror
+
     @specialize.arg(1)
     def check_and_raise_exception(self, always=False):
-        operror = self.operror
+        ec = self.space.getexecutioncontext()
+        operror = ec.cpyext_operror
         if operror:
             self.clear_exception()
             raise operror
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2202,6 +2202,16 @@
     Py_RETURN_NONE;
 };
 
+static PyObject *
+getitem(PyObject* self, PyObject * args) {
+    PyObject * obj;
+    int i;
+    if (!PyArg_ParseTuple(args, "Oi", &obj, &i)) {
+        return NULL;
+    }
+    return PySequence_ITEM(obj, i);
+}
+
 PyDoc_STRVAR(module_doc,
 "This module defines an object type which can efficiently represent\n\
 an array of basic values: characters, integers, floating point\n\
@@ -2491,6 +2501,7 @@
     {"get_releasebuffer_cnt",   (PyCFunction)get_releasebuffer_cnt, METH_NOARGS, NULL},
     {"create_and_release_buffer",   (PyCFunction)create_and_release_buffer, METH_O, NULL},
     {"same_dealloc",   (PyCFunction)same_dealloc, METH_VARARGS, NULL},
+    {"getitem", (PyCFunction)getitem, METH_VARARGS, NULL},
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
 
diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py
--- a/pypy/module/cpyext/test/test_api.py
+++ b/pypy/module/cpyext/test/test_api.py
@@ -39,7 +39,7 @@
             raise Exception("%s is not callable" % (f,))
         f(*args)
         state = space.fromcache(State)
-        operror = state.operror
+        operror = state.get_exception()
         if not operror:
             raise Exception("DID NOT RAISE")
         if getattr(space, 'w_' + expected_exc.__name__) is not operror.w_type:
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -76,7 +76,9 @@
         else:
             expected = '\x01\0\0\0' '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0'
         assert str(buf) == expected
-        assert str(buffer('') + arr) == expected
+        assert str(buffer('a') + arr) == "a" + expected
+        # python2 special cases empty-buffer + obj
+        assert str(buffer('') + arr) == "array('i', [1, 2, 3, 4])"
 
     def test_releasebuffer(self):
         module = self.import_module(name='array')
@@ -172,3 +174,15 @@
         fd = BytesIO()
         # only test that it works
         fd.write(a)
+
+    def test_getitem_via_PySequence_GetItem(self):
+        module = self.import_module(name='array')
+        a = module.array('i', range(10))
+        # call via tp_as_mapping.mp_subscript
+        assert 5 == a[-5]
+        # PySequence_ITEM used to call space.getitem() which
+        # prefers tp_as_mapping.mp_subscript over tp_as_sequence.sq_item
+        # Now fixed so this test raises (array_item does not add len(a),
+        # array_subscr does)
+        raises(IndexError, module.getitem, a, -5)
+
diff --git a/pypy/module/cpyext/test/test_boolobject.py b/pypy/module/cpyext/test/test_boolobject.py
--- a/pypy/module/cpyext/test/test_boolobject.py
+++ b/pypy/module/cpyext/test/test_boolobject.py
@@ -1,7 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.boolobject import PyBool_Check, PyBool_FromLong
-from pypy.module.cpyext.floatobject import PyFloat_FromDouble
+from pypy.module.cpyext.boolobject import PyBool_FromLong
 
 class TestBoolObject(BaseApiTest):
     def test_fromlong(self, space):
@@ -12,12 +11,6 @@
             else:
                 assert obj is space.w_False
 
-    def test_check(self, space):
-        assert PyBool_Check(space, space.w_True)
-        assert PyBool_Check(space, space.w_False)
-        assert not PyBool_Check(space, space.w_None)
-        assert not PyBool_Check(space, PyFloat_FromDouble(space, 1.0))
-
 class AppTestBoolMacros(AppTestCpythonExtensionBase):
     def test_macros(self):
         module = self.import_extension('foo', [
@@ -42,4 +35,14 @@
         assert module.to_int(False) == 0
         assert module.to_int(True) == 1
 
-            
+    def test_check(self):
+        module = self.import_extension('foo', [
+            ("type_check", "METH_O",
+             '''
+                return PyLong_FromLong(PyBool_Check(args));
+             ''')])
+        assert module.type_check(True)
+        assert module.type_check(False)
+        assert not module.type_check(None)
+        assert not module.type_check(1.0)
+             
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -636,7 +636,8 @@
             Py_ssize_t refcnt_after;
             Py_INCREF(true_obj);
             Py_INCREF(true_obj);
-            PyBool_Check(true_obj);
+            if (!PyBool_Check(true_obj))
+                Py_RETURN_NONE;
             refcnt_after = true_obj->ob_refcnt;
             Py_DECREF(true_obj);
             Py_DECREF(true_obj);
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -102,9 +102,11 @@
              """
              PyObject* pyobj = PyFloat_FromDouble(1.0);
              PyFloatObject* pfo = (PyFloatObject*)pyobj;
-             int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) &&
-                PyFloat_Check(pfo) && PyFloat_CheckExact(pfo);
+             int res = (PyFloat_Check(pyobj) +
+                        PyFloat_CheckExact(pyobj) * 10 +
+                        PyFloat_Check(pfo) * 100 +
+                        PyFloat_CheckExact(pfo) * 1000);
              Py_DecRef(pyobj);
              return PyLong_FromLong(res);"""),
             ])
-        assert module.test() == 1
+        assert module.test() == 1111
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -11,7 +11,6 @@
     PyNumber_Index, PyNumber_Coerce, PyNumber_CoerceEx, PyNumber_Add,
     PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute,
     PyNumber_Power, PyNumber_InPlacePower)
-from pypy.module.cpyext.floatobject import PyFloat_Check
 from pypy.module.cpyext.intobject import PyInt_CheckExact
 from pypy.module.cpyext.longobject import PyLong_CheckExact
 from pypy.module.cpyext.object import PyObject_Size
@@ -86,7 +85,7 @@
 
         w_res = from_ref(space, ppl[0])
 
-        assert PyFloat_Check(space, w_res)
+        assert space.isinstance_w(w_res, space.w_float)
         assert space.unwrap(w_res) == 123.
         decref(space, pl)
         decref(space, pf)
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -52,7 +52,8 @@
         api.PyErr_SetObject(space.w_ValueError, space.wrap("a value"))
         assert api.PyErr_Occurred() is space.w_ValueError
         state = space.fromcache(State)
-        assert space.eq_w(state.operror.get_w_value(space),
+        operror = state.get_exception()
+        assert space.eq_w(operror.get_w_value(space),
                           space.wrap("a value"))
 
         api.PyErr_Clear()
@@ -60,12 +61,14 @@
     def test_SetNone(self, space, api):
         api.PyErr_SetNone(space.w_KeyError)
         state = space.fromcache(State)
-        assert space.eq_w(state.operror.w_type, space.w_KeyError)
-        assert space.eq_w(state.operror.get_w_value(space), space.w_None)
+        operror = state.get_exception()
+        assert space.eq_w(operror.w_type, space.w_KeyError)
+        assert space.eq_w(operror.get_w_value(space), space.w_None)
         api.PyErr_Clear()
 
         api.PyErr_NoMemory()
-        assert space.eq_w(state.operror.w_type, space.w_MemoryError)
+        operror = state.get_exception()
+        assert space.eq_w(operror.w_type, space.w_MemoryError)
         api.PyErr_Clear()
 
     def test_Warning(self, space, api, capfd):
@@ -437,3 +440,59 @@
              '''),
             ])
         raises(SystemError, module.oops)
+
+    def test_error_thread_race(self):
+        # Check race condition: thread 0 returns from cpyext with error set,
+        # after thread 1 has set an error but before it returns.
+        module = self.import_extension('foo', [
+            ("emit_error", "METH_VARARGS",
+             '''
+             PyThreadState *save = NULL;
+             PyGILState_STATE gilsave;
+
+             /* NB. synchronization due to GIL */
+             static volatile int flag = 0;
+             int id;
+
+             if (!PyArg_ParseTuple(args, "i", &id))
+                 return NULL;
+
+             /* Proceed in thread 1 first */
+             save = PyEval_SaveThread();
+             while (id == 0 && flag == 0);
+             gilsave = PyGILState_Ensure();
+
+             PyErr_Format(PyExc_ValueError, "%d", id);
+
+             /* Proceed in thread 0 first */
+             if (id == 1) flag = 1;
+             PyGILState_Release(gilsave);
+             while (id == 1 && flag == 1);
+             PyEval_RestoreThread(save);
+
+             if (id == 0) flag = 0;
+             return NULL;
+             '''
+             ),
+            ])
+
+        import threading
+
+        failures = []
+
+        def worker(arg):
+            try:
+                module.emit_error(arg)
+                failures.append(True)
+            except Exception as exc:
+                if str(exc) != str(arg):
+                    failures.append(exc)
+
+        threads = [threading.Thread(target=worker, args=(j,))
+                   for j in (0, 1)]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+        assert not failures
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py
--- a/pypy/module/cpyext/test/test_sliceobject.py
+++ b/pypy/module/cpyext/test/test_sliceobject.py
@@ -2,14 +2,8 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP
-from pypy.module.cpyext.sliceobject import PySlice_Check
 
 class TestSliceObject(BaseApiTest):
-    def test_slice(self, space):
-        w_i = space.wrap(10)
-        w_slice = space.newslice(w_i, w_i, w_i)
-        assert PySlice_Check(space, w_slice)
-        assert not PySlice_Check(space, w_i)
 
     def test_GetIndicesEx(self, space, api):
         w = space.wrap
@@ -79,3 +73,14 @@
              """),
             ])
         assert module.get_ellipsis() is Ellipsis
+
+    def test_typecheck(self):
+        module = self.import_extension('foo', [
+            ("check", "METH_O",
+             """
+                 PySliceObject *slice = (PySliceObject *)args;
+                 return PyLong_FromLong(PySlice_Check(slice));
+             """),
+            ])
+        s = slice(10, 20, 30)
+        assert module.check(s)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1,3 +1,4 @@
+import pytest
 from pypy.interpreter import gateway
 from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
@@ -6,6 +7,7 @@
 from pypy.module.cpyext.pyobject import make_ref, from_ref, decref, as_pyobj
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 
+
 class AppTestTypeObject(AppTestCpythonExtensionBase):
 
     def setup_class(cls):
@@ -136,8 +138,10 @@
         module = self.import_module(name='foo')
         descr = module.fooType.copy
         assert type(descr).__name__ == 'method_descriptor'
-        assert str(descr) == ""
-        assert repr(descr) == ""
+        assert str(descr) in ("",
+            "")
+        assert repr(descr) in ("",
+            "")
         raises(TypeError, descr, None)
 
     def test_cython_fake_classmethod(self):
@@ -250,7 +254,7 @@
         import re
         assert re.sre_compile._sre is module
         s = u"Foo " * 1000 + u"Bar"
-        prog = re.compile(ur"Foo.*Bar")
+        prog = re.compile(u"Foo.*Bar")
         assert prog.match(s)
         m = re.search(u"xyz", u"xyzxyz")
         assert m
@@ -319,7 +323,7 @@
     def test_tp_dict(self):
         foo = self.import_module("foo")
         module = self.import_extension('test', [
-           ("read_tp_dict", "METH_O",
+            ("read_tp_dict", "METH_O",
             '''
                  PyObject *method;
                  if (!args->ob_type->tp_dict)
@@ -420,7 +424,7 @@
                      return NULL;
                  Py_DECREF(a1);
                  PyType_Modified(type);
-                 value = PyObject_GetAttrString((PyObject*)type, "a");
+                 value = PyObject_GetAttrString((PyObject *)type, "a");
                  Py_DECREF(value);
 
                  if (PyDict_SetItemString(type->tp_dict, "a",
@@ -428,7 +432,7 @@
                      return NULL;
                  Py_DECREF(a2);
                  PyType_Modified(type);
-                 value = PyObject_GetAttrString((PyObject*)type, "a");
+                 value = PyObject_GetAttrString((PyObject *)type, "a");
                  return value;
              '''
              )
@@ -529,7 +533,7 @@
 
         py_type = rffi.cast(PyTypeObjectPtr, ref)
         w_dict = from_ref(space, py_type.c_tp_dict)
-        w_name = space.wrap('a')
+        w_name = space.newtext('a')
         space.setitem(w_dict, w_name, space.wrap(1))
         assert space.int_w(space.getattr(w_class, w_name)) == 1
         space.delitem(w_dict, w_name)
@@ -611,16 +615,21 @@
         module = self.import_extension('foo', [
             ("test_tp_getattro", "METH_VARARGS",
              '''
+                 #if PY_MAJOR_VERSION > 2
+                 #define PyString_FromString PyUnicode_FromString
+                 #define PyIntObject PyLongObject
+                 #define PyInt_AsLong PyLong_AsLong
+                 #endif
                  PyObject *name, *obj = PyTuple_GET_ITEM(args, 0);
-                 PyIntObject *attr, *value = (PyIntObject*) PyTuple_GET_ITEM(args, 1);
+                 PyObject *attr, *value = PyTuple_GET_ITEM(args, 1);
                  if (!obj->ob_type->tp_getattro)
                  {
                      PyErr_SetString(PyExc_ValueError, "missing tp_getattro");
                      return NULL;
                  }
                  name = PyString_FromString("attr1");
-                 attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name);
-                 if (attr->ob_ival != value->ob_ival)
+                 attr = obj->ob_type->tp_getattro(obj, name);
+                 if (PyInt_AsLong(attr) != PyInt_AsLong(value))
                  {
                      PyErr_SetString(PyExc_ValueError,
                                      "tp_getattro returned wrong value");
@@ -629,7 +638,7 @@
                  Py_DECREF(name);
                  Py_DECREF(attr);
                  name = PyString_FromString("attr2");
-                 attr = (PyIntObject*) obj->ob_type->tp_getattro(obj, name);
+                 attr = obj->ob_type->tp_getattro(obj, name);
                  if (attr == NULL && PyErr_ExceptionMatches(PyExc_AttributeError))
                  {
                      PyErr_Clear();
@@ -652,6 +661,9 @@
         module = self.import_extension('foo', [
             ("get_foo", "METH_O",
              '''
+             #if PY_MAJOR_VERSION > 2
+             #define PyString_FromString PyUnicode_FromString

From pypy.commits at gmail.com  Thu Mar 29 06:03:38 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 03:03:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default
Message-ID: <5abcb9fa.0ea6df0a.a5146.d01e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r94170:ffbcc29df485
Date: 2018-03-29 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/ffbcc29df485/

Log:	merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -72,3 +72,14 @@
 Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify
 `W_PyCWrapperObject` which is used to call slots from the C-API, greatly
 improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks
+
+
+.. branch: fix-sre-problems
+
+Fix two (unrelated) JIT bugs manifesting in the re module:
+
+- green fields are broken and were thus disabled, plus their usage removed from
+  the _sre implementation
+
+- in rare "trace is too long" situations, the JIT could break behaviour
+  arbitrarily.
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -232,7 +232,9 @@
                 "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
+        key_pycode = self.key_pycode
         jitdriver1.jit_merge_point(callback=self,
+                                   key_pycode=key_pycode,
                                    ll_res=ll_res,
                                    ll_args=ll_args)
         self.do_invoke(ll_res, ll_args)
@@ -294,7 +296,7 @@
     return 'cffi_callback ' + key_pycode.get_repr()
 
 jitdriver1 = jit.JitDriver(name='cffi_callback',
-                           greens=['callback.key_pycode'],
+                           greens=['key_pycode'],
                            reds=['ll_res', 'll_args', 'callback'],
                            get_printable_location=get_printable_location1)
 
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -76,15 +76,15 @@
     w_import = space.getattr(space.builtin, space.newtext("__import__"))
     return space.call_function(w_import, space.newtext("re"))
 
-def matchcontext(space, ctx):
+def matchcontext(space, ctx, pattern):
     try:
-        return rsre_core.match_context(ctx)
+        return rsre_core.match_context(ctx, pattern)
     except rsre_core.Error as e:
         raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
-def searchcontext(space, ctx):
+def searchcontext(space, ctx, pattern):
     try:
-        return rsre_core.search_context(ctx)
+        return rsre_core.search_context(ctx, pattern)
     except rsre_core.Error as e:
         raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
@@ -189,7 +189,7 @@
                 raise oefmt(space.w_TypeError,
                             "can't use a bytes pattern on a string-like "
                             "object")
-            return rsre_core.UnicodeMatchContext(self.code, unicodestr,
+            return rsre_core.UnicodeMatchContext(unicodestr,
                                                  pos, endpos, flags)
         else:
             if self.is_known_unicode():
@@ -197,10 +197,10 @@
                             "can't use a string pattern on a bytes-like "
                             "object")
             if string is not None:
-                return rsre_core.StrMatchContext(self.code, string,
+                return rsre_core.StrMatchContext(string,
                                                  pos, endpos, flags)
             else:
-                return rsre_core.BufMatchContext(self.code, buf,
+                return rsre_core.BufMatchContext(buf,
                                                  pos, endpos, flags)
 
     def getmatch(self, ctx, found):
@@ -212,18 +212,18 @@
     @unwrap_spec(pos=int, endpos=int)
     def match_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
-        return self.getmatch(ctx, matchcontext(self.space, ctx))
+        return self.getmatch(ctx, matchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def fullmatch_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
         ctx.fullmatch_only = True
-        return self.getmatch(ctx, matchcontext(self.space, ctx))
+        return self.getmatch(ctx, matchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def search_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
-        return self.getmatch(ctx, searchcontext(self.space, ctx))
+        return self.getmatch(ctx, searchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def findall_w(self, w_string, pos=0, endpos=sys.maxint):
@@ -231,7 +231,7 @@
         matchlist_w = []
         ctx = self.make_ctx(w_string, pos, endpos)
         while ctx.match_start <= ctx.end:
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, self.code):
                 break
             num_groups = self.num_groups
             w_emptystr = space.newtext("")
@@ -256,14 +256,15 @@
         # this also works as the implementation of the undocumented
         # scanner() method.
         ctx = self.make_ctx(w_string, pos, endpos)
-        scanner = W_SRE_Scanner(self, ctx)
+        scanner = W_SRE_Scanner(self, ctx, self.code)
         return scanner
 
     @unwrap_spec(maxsplit=int)
     def split_w(self, w_string, maxsplit=0):
         space = self.space
-        if self.code[0] != rsre_core.OPCODE_INFO or self.code[3] == 0:
-            if self.code[0] == rsre_core.OPCODE_INFO and self.code[4] == 0:
+
+        if self.code.pattern[0] != rsre_core.OPCODE_INFO or self.code.pattern[3] == 0:
+            if self.code.pattern[0] == rsre_core.OPCODE_INFO and self.code.pattern[4] == 0:
                 raise oefmt(space.w_ValueError,
                             "split() requires a non-empty pattern match.")
             space.warn(
@@ -275,7 +276,7 @@
         last = 0
         ctx = self.make_ctx(w_string)
         while not maxsplit or n < maxsplit:
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, self.code):
                 break
             if ctx.match_start == ctx.match_end:     # zero-width match
                 if ctx.match_start == ctx.end:       # or end of string
@@ -356,8 +357,8 @@
         else:
             sublist_w = []
         n = last_pos = 0
+        pattern = self.code
         while not count or n < count:
-            pattern = ctx.pattern
             sub_jitdriver.jit_merge_point(
                 self=self,
                 use_builder=use_builder,
@@ -374,7 +375,7 @@
                 n=n, last_pos=last_pos, sublist_w=sublist_w
                 )
             space = self.space
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, pattern):
                 break
             if last_pos < ctx.match_start:
                 _sub_append_slice(
@@ -474,7 +475,11 @@
         space.readbuf_w(w_pattern)
     srepat.w_pattern = w_pattern      # the original uncompiled pattern
     srepat.flags = flags
-    srepat.code = code
+    # note: we assume that the app-level is caching SRE_Pattern objects,
+    # so that we don't need to do it here.  Creating new SRE_Pattern
+    # objects all the time would be bad for the JIT, which relies on the
+    # identity of the CompiledPattern() object.
+    srepat.code = rsre_core.CompiledPattern(code)
     srepat.num_groups = groups
     srepat.w_groupindex = w_groupindex
     srepat.w_indexgroup = w_indexgroup
@@ -711,10 +716,11 @@
 # Our version is also directly iterable, to make finditer() easier.
 
 class W_SRE_Scanner(W_Root):
-    def __init__(self, pattern, ctx):
+    def __init__(self, pattern, ctx, code):
         self.space = pattern.space
         self.srepat = pattern
         self.ctx = ctx
+        self.code = code
         # 'self.ctx' is always a fresh context in which no searching
         # or matching succeeded so far.
 
@@ -724,19 +730,19 @@
     def next_w(self):
         if self.ctx.match_start > self.ctx.end:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        if not searchcontext(self.space, self.ctx):
+        if not searchcontext(self.space, self.ctx, self.code):
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
         return self.getmatch(True)
 
     def match_w(self):
         if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
-        return self.getmatch(matchcontext(self.space, self.ctx))
+        return self.getmatch(matchcontext(self.space, self.ctx, self.code))
 
     def search_w(self):
         if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
-        return self.getmatch(searchcontext(self.space, self.ctx))
+        return self.getmatch(searchcontext(self.space, self.ctx, self.code))
 
     def getmatch(self, found):
         if found:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -701,6 +701,9 @@
     def length(self):
         return self.trace._count - len(self.trace.inputargs)
 
+    def trace_tag_overflow(self):
+        return self.trace.tag_overflow
+
     def get_trace_position(self):
         return self.trace.cut_point()
 
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -49,13 +49,6 @@
     way up to lltype.Signed for indexes everywhere
     """
 
-def frontend_tag_overflow():
-    # Minor abstraction leak: raise directly the right exception
-    # expected by the rest of the machinery
-    from rpython.jit.metainterp import history
-    from rpython.rlib.jit import Counters
-    raise history.SwitchToBlackhole(Counters.ABORT_TOO_LONG)
-
 class BaseTrace(object):
     pass
 
@@ -293,6 +286,7 @@
         self._start = len(inputargs)
         self._pos = self._start
         self.inputargs = inputargs
+        self.tag_overflow = False
 
     def append(self, v):
         model = get_model(self)
@@ -300,12 +294,14 @@
             # grow by 2X
             self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops)
         if not model.MIN_VALUE <= v <= model.MAX_VALUE:
-            raise frontend_tag_overflow()
+            v = 0 # broken value, but that's fine, tracing will stop soon
+            self.tag_overflow = True
         self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v)
         self._pos += 1
 
-    def done(self):
+    def tracing_done(self):
         from rpython.rlib.debug import debug_start, debug_stop, debug_print
+        assert not self.tag_overflow
 
         self._bigints_dict = {}
         self._refs_dict = llhelper.new_ref_dict_3()
@@ -317,8 +313,6 @@
         debug_print(" ref consts: " + str(self._consts_ptr) + " " + str(len(self._refs)))
         debug_print(" descrs: " + str(len(self._descrs)))
         debug_stop("jit-trace-done")
-        return 0 # completely different than TraceIter.done, but we have to
-        # share the base class
 
     def length(self):
         return self._pos
@@ -379,6 +373,7 @@
 
     def record_op(self, opnum, argboxes, descr=None):
         pos = self._index
+        old_pos = self._pos
         self.append(opnum)
         expected_arity = oparity[opnum]
         if expected_arity == -1:
@@ -397,6 +392,10 @@
         self._count += 1
         if opclasses[opnum].type != 'v':
             self._index += 1
+        if self.tag_overflow:
+            # potentially a broken op is left behind
+            # clean it up
+            self._pos = old_pos
         return pos
 
     def _encode_descr(self, descr):
@@ -424,10 +423,11 @@
         vref_array = self._list_of_boxes(vref_boxes)
         s = TopSnapshot(combine_uint(jitcode.index, pc), array, vable_array,
                         vref_array)
-        assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
         # guards have no descr
         self._snapshots.append(s)
-        self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
+        if not self.tag_overflow: # otherwise we're broken anyway
+            assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
+            self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
         return s
 
     def create_empty_top_snapshot(self, vable_boxes, vref_boxes):
@@ -436,10 +436,11 @@
         vref_array = self._list_of_boxes(vref_boxes)
         s = TopSnapshot(combine_uint(2**16 - 1, 0), [], vable_array,
                         vref_array)
-        assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
         # guards have no descr
         self._snapshots.append(s)
-        self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
+        if not self.tag_overflow: # otherwise we're broken anyway
+            assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
+            self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
         return s
 
     def create_snapshot(self, jitcode, pc, frame, flag):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2384,9 +2384,9 @@
 
     def blackhole_if_trace_too_long(self):
         warmrunnerstate = self.jitdriver_sd.warmstate
-        if self.history.length() > warmrunnerstate.trace_limit:
+        if (self.history.length() > warmrunnerstate.trace_limit or
+                self.history.trace_tag_overflow()):
             jd_sd, greenkey_of_huge_function = self.find_biggest_function()
-            self.history.trace.done()
             self.staticdata.stats.record_aborted(greenkey_of_huge_function)
             self.portal_trace_positions = None
             if greenkey_of_huge_function is not None:
@@ -2689,7 +2689,9 @@
                      try_disabling_unroll=False, exported_state=None):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         if not self.partial_trace:
             ptoken = self.get_procedure_token(greenkey)
             if ptoken is not None and ptoken.target_tokens is not None:
@@ -2742,7 +2744,9 @@
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
                             descr=target_jitcell_token)
         self.history.ends_with_jump = True
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         try:
             target_token = compile.compile_trace(self, self.resumekey,
                 live_arg_boxes[num_green_args:])
@@ -2776,7 +2780,9 @@
             assert False
         # FIXME: can we call compile_trace?
         self.history.record(rop.FINISH, exits, None, descr=token)
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         target_token = compile.compile_trace(self, self.resumekey, exits)
         if target_token is not token:
             compile.giveup()
@@ -2802,7 +2808,9 @@
         sd = self.staticdata
         token = sd.exit_frame_with_exception_descr_ref
         self.history.record(rop.FINISH, [valuebox], None, descr=token)
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         target_token = compile.compile_trace(self, self.resumekey, [valuebox])
         if target_token is not token:
             compile.giveup()
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4661,3 +4661,36 @@
 
         f() # finishes
         self.meta_interp(f, [])
+
+    def test_trace_too_long_bug(self):
+        driver = JitDriver(greens=[], reds=['i'])
+        @unroll_safe
+        def match(s):
+            l = len(s)
+            p = 0
+            for i in range(2500): # produces too long trace
+                c = s[p]
+                if c != 'a':
+                    return False
+                p += 1
+                if p >= l:
+                    return True
+                c = s[p]
+                if c != '\n':
+                    p += 1
+                    if p >= l:
+                        return True
+                else:
+                    return False
+            return True
+
+        def f(i):
+            while i > 0:
+                driver.jit_merge_point(i=i)
+                match('a' * (500 * i))
+                i -= 1
+            return i
+
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py
--- a/rpython/jit/metainterp/test/test_greenfield.py
+++ b/rpython/jit/metainterp/test/test_greenfield.py
@@ -1,6 +1,17 @@
+import pytest
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib.jit import JitDriver, assert_green
 
+pytest.skip("this feature is disabled at the moment!")
+
+# note why it is disabled: before d721da4573ad
+# there was a failing assert when inlining python -> sre -> python:
+# https://bitbucket.org/pypy/pypy/issues/2775/
+# this shows, that the interaction of greenfields and virtualizables is broken,
+# because greenfields use MetaInterp.virtualizable_boxes, which confuses
+# MetaInterp._nonstandard_virtualizable somehow (and makes no sense
+# conceptually anyway). to fix greenfields, the two mechanisms would have to be
+# disentangled.
 
 class GreenFieldsTests:
 
diff --git a/rpython/jit/metainterp/test/test_opencoder.py b/rpython/jit/metainterp/test/test_opencoder.py
--- a/rpython/jit/metainterp/test/test_opencoder.py
+++ b/rpython/jit/metainterp/test/test_opencoder.py
@@ -209,5 +209,8 @@
     def test_tag_overflow(self):
         t = Trace([], metainterp_sd)
         i0 = FakeOp(100000)
-        py.test.raises(SwitchToBlackhole, t.record_op, rop.FINISH, [i0])
-        assert t.unpack() == ([], [])
+        # if we overflow, we can keep recording
+        for i in range(10):
+            t.record_op(rop.FINISH, [i0])
+            assert t.unpack() == ([], [])
+        assert t.tag_overflow
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -653,6 +653,9 @@
         self._make_extregistryentries()
         assert get_jitcell_at is None, "get_jitcell_at no longer used"
         assert set_jitcell_at is None, "set_jitcell_at no longer used"
+        for green in self.greens:
+            if "." in green:
+                raise ValueError("green fields are buggy! if you need them fixed, please talk to us")
         self.get_printable_location = get_printable_location
         self.get_location = get_location
         self.has_unique_id = (get_unique_id is not None)
diff --git a/rpython/rlib/rsre/rpy/_sre.py b/rpython/rlib/rsre/rpy/_sre.py
--- a/rpython/rlib/rsre/rpy/_sre.py
+++ b/rpython/rlib/rsre/rpy/_sre.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rsre import rsre_char
+from rpython.rlib.rsre import rsre_char, rsre_core
 from rpython.rlib.rarithmetic import intmask
 
 VERSION = "2.7.6"
@@ -12,7 +12,7 @@
     pass
 
 def compile(pattern, flags, code, *args):
-    raise GotIt([intmask(i) for i in code], flags, args)
+    raise GotIt(rsre_core.CompiledPattern([intmask(i) for i in code]), flags, args)
 
 
 def get_code(regexp, flags=0, allargs=False):
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -152,17 +152,16 @@
 ##### Charset evaluation
 
 @jit.unroll_safe
-def check_charset(ctx, ppos, char_code):
+def check_charset(ctx, pattern, ppos, char_code):
     """Checks whether a character matches set of arbitrary length.
     The set starts at pattern[ppos]."""
     negated = False
     result = False
-    pattern = ctx.pattern
     while True:
-        opcode = pattern[ppos]
+        opcode = pattern.pattern[ppos]
         for i, function in set_dispatch_unroll:
             if opcode == i:
-                newresult, ppos = function(ctx, ppos, char_code)
+                newresult, ppos = function(ctx, pattern, ppos, char_code)
                 result |= newresult
                 break
         else:
@@ -177,50 +176,44 @@
         return not result
     return result
 
-def set_literal(ctx, index, char_code):
+def set_literal(ctx, pattern, index, char_code):
     #  
-    pat = ctx.pattern
-    match = pat[index+1] == char_code
+    match = pattern.pattern[index+1] == char_code
     return match, index + 2
 
-def set_category(ctx, index, char_code):
+def set_category(ctx, pattern, index, char_code):
     #  
-    pat = ctx.pattern
-    match = category_dispatch(pat[index+1], char_code)
+    match = category_dispatch(pattern.pattern[index+1], char_code)
     return match, index + 2
 
-def set_charset(ctx, index, char_code):
+def set_charset(ctx, pattern, index, char_code):
     #   (16 bits per code word)
-    pat = ctx.pattern
     if CODESIZE == 2:
         match = char_code < 256 and \
-                (pat[index+1+(char_code >> 4)] & (1 << (char_code & 15)))
+                (pattern.pattern[index+1+(char_code >> 4)] & (1 << (char_code & 15)))
         return match, index + 17  # skip bitmap
     else:
         match = char_code < 256 and \
-                (pat[index+1+(char_code >> 5)] & (1 << (char_code & 31)))
+                (pattern.pattern[index+1+(char_code >> 5)] & (1 << (char_code & 31)))
         return match, index + 9   # skip bitmap
 
-def set_range(ctx, index, char_code):
+def set_range(ctx, pattern, index, char_code):
     #   
-    pat = ctx.pattern
-    match = int_between(pat[index+1], char_code, pat[index+2] + 1)
+    match = int_between(pattern.pattern[index+1], char_code, pattern.pattern[index+2] + 1)
     return match, index + 3
 
-def set_range_ignore(ctx, index, char_code):
+def set_range_ignore(ctx, pattern, index, char_code):
     #   
     # the char_code is already lower cased
-    pat = ctx.pattern
-    lower = pat[index + 1]
-    upper = pat[index + 2]
+    lower = pattern.pattern[index + 1]
+    upper = pattern.pattern[index + 2]
     match1 = int_between(lower, char_code, upper + 1)
     match2 = int_between(lower, getupper(char_code, ctx.flags), upper + 1)
     return match1 | match2, index + 3
 
-def set_bigcharset(ctx, index, char_code):
+def set_bigcharset(ctx, pattern, index, char_code):
     #   <256 blockindices> 
-    pat = ctx.pattern
-    count = pat[index+1]
+    count = pattern.pattern[index+1]
     index += 2
 
     if CODESIZE == 2:
@@ -238,7 +231,7 @@
             return False, index
         shift = 5
 
-    block = pat[index + (char_code >> (shift + 5))]
+    block = pattern.pattern[index + (char_code >> (shift + 5))]
 
     block_shift = char_code >> 5
     if BIG_ENDIAN:
@@ -247,23 +240,22 @@
     block = (block >> block_shift) & 0xFF
 
     index += 256 / CODESIZE
-    block_value = pat[index+(block * (32 / CODESIZE)
+    block_value = pattern.pattern[index+(block * (32 / CODESIZE)
                              + ((char_code & 255) >> shift))]
     match = (block_value & (1 << (char_code & ((8 * CODESIZE) - 1))))
     index += count * (32 / CODESIZE)  # skip blocks
     return match, index
 
-def set_unicode_general_category(ctx, index, char_code):
+def set_unicode_general_category(ctx, pattern, index, char_code):
     # Unicode "General category property code" (not used by Python).
-    # A general category is two letters.  'pat[index+1]' contains both
+    # A general category is two letters.  'pattern.pattern[index+1]' contains both
     # the first character, and the second character shifted by 8.
     # http://en.wikipedia.org/wiki/Unicode_character_property#General_Category
     # Also supports single-character categories, if the second character is 0.
     # Negative matches are triggered by bit number 7.
     assert unicodedb is not None
     cat = unicodedb.category(char_code)
-    pat = ctx.pattern
-    category_code = pat[index + 1]
+    category_code = pattern.pattern[index + 1]
     first_character = category_code & 0x7F
     second_character = (category_code >> 8) & 0x7F
     negative_match = category_code & 0x80
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -83,35 +83,19 @@
     def __init__(self, msg):
         self.msg = msg
 
-class AbstractMatchContext(object):
-    """Abstract base class"""
-    _immutable_fields_ = ['pattern[*]', 'flags', 'end']
-    match_start = 0
-    match_end = 0
-    match_marks = None
-    match_marks_flat = None
-    fullmatch_only = False
 
-    def __init__(self, pattern, match_start, end, flags):
-        # 'match_start' and 'end' must be known to be non-negative
-        # and they must not be more than len(string).
-        check_nonneg(match_start)
-        check_nonneg(end)
+class CompiledPattern(object):
+    _immutable_fields_ = ['pattern[*]']
+
+    def __init__(self, pattern):
         self.pattern = pattern
-        self.match_start = match_start
-        self.end = end
-        self.flags = flags
         # check we don't get the old value of MAXREPEAT
         # during the untranslated tests
         if not we_are_translated():
             assert 65535 not in pattern
 
-    def reset(self, start):
-        self.match_start = start
-        self.match_marks = None
-        self.match_marks_flat = None
-
     def pat(self, index):
+        jit.promote(self)
         check_nonneg(index)
         result = self.pattern[index]
         # Check that we only return non-negative integers from this helper.
@@ -121,6 +105,29 @@
         assert result >= 0
         return result
 
+class AbstractMatchContext(object):
+    """Abstract base class"""
+    _immutable_fields_ = ['flags', 'end']
+    match_start = 0
+    match_end = 0
+    match_marks = None
+    match_marks_flat = None
+    fullmatch_only = False
+
+    def __init__(self, match_start, end, flags):
+        # 'match_start' and 'end' must be known to be non-negative
+        # and they must not be more than len(string).
+        check_nonneg(match_start)
+        check_nonneg(end)
+        self.match_start = match_start
+        self.end = end
+        self.flags = flags
+
+    def reset(self, start):
+        self.match_start = start
+        self.match_marks = None
+        self.match_marks_flat = None
+
     @not_rpython
     def str(self, index):
         """Must be overridden in a concrete subclass.
@@ -183,8 +190,8 @@
 
     _immutable_fields_ = ["_buffer"]
 
-    def __init__(self, pattern, buf, match_start, end, flags):
-        AbstractMatchContext.__init__(self, pattern, match_start, end, flags)
+    def __init__(self, buf, match_start, end, flags):
+        AbstractMatchContext.__init__(self, match_start, end, flags)
         self._buffer = buf
 
     def str(self, index):
@@ -196,7 +203,7 @@
         return rsre_char.getlower(c, self.flags)
 
     def fresh_copy(self, start):
-        return BufMatchContext(self.pattern, self._buffer, start,
+        return BufMatchContext(self._buffer, start,
                                self.end, self.flags)
 
 class StrMatchContext(AbstractMatchContext):
@@ -204,8 +211,8 @@
 
     _immutable_fields_ = ["_string"]
 
-    def __init__(self, pattern, string, match_start, end, flags):
-        AbstractMatchContext.__init__(self, pattern, match_start, end, flags)
+    def __init__(self, string, match_start, end, flags):
+        AbstractMatchContext.__init__(self, match_start, end, flags)
         self._string = string
         if not we_are_translated() and isinstance(string, unicode):
             self.flags |= rsre_char.SRE_FLAG_UNICODE   # for rsre_re.py
@@ -219,7 +226,7 @@
         return rsre_char.getlower(c, self.flags)
 
     def fresh_copy(self, start):
-        return StrMatchContext(self.pattern, self._string, start,
+        return StrMatchContext(self._string, start,
                                self.end, self.flags)
 
 class UnicodeMatchContext(AbstractMatchContext):
@@ -227,8 +234,8 @@
 
     _immutable_fields_ = ["_unicodestr"]
 
-    def __init__(self, pattern, unicodestr, match_start, end, flags):
-        AbstractMatchContext.__init__(self, pattern, match_start, end, flags)
+    def __init__(self, unicodestr, match_start, end, flags):
+        AbstractMatchContext.__init__(self, match_start, end, flags)
         self._unicodestr = unicodestr
 
     def str(self, index):
@@ -240,7 +247,7 @@
         return rsre_char.getlower(c, self.flags)
 
     def fresh_copy(self, start):
-        return UnicodeMatchContext(self.pattern, self._unicodestr, start,
+        return UnicodeMatchContext(self._unicodestr, start,
                                    self.end, self.flags)
 
 # ____________________________________________________________
@@ -265,16 +272,16 @@
 class MatchResult(object):
     subresult = None
 
-    def move_to_next_result(self, ctx):
+    def move_to_next_result(self, ctx, pattern):
         # returns either 'self' or None
         result = self.subresult
         if result is None:
             return
-        if result.move_to_next_result(ctx):
+        if result.move_to_next_result(ctx, pattern):
             return self
-        return self.find_next_result(ctx)
+        return self.find_next_result(ctx, pattern)
 
-    def find_next_result(self, ctx):
+    def find_next_result(self, ctx, pattern):
         raise NotImplementedError
 
 MATCHED_OK = MatchResult()
@@ -287,11 +294,11 @@
         self.start_marks = marks
 
     @jit.unroll_safe
-    def find_first_result(self, ctx):
+    def find_first_result(self, ctx, pattern):
         ppos = jit.hint(self.ppos, promote=True)
-        while ctx.pat(ppos):
-            result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks)
-            ppos += ctx.pat(ppos)
+        while pattern.pat(ppos):
+            result = sre_match(ctx, pattern, ppos + 1, self.start_ptr, self.start_marks)
+            ppos += pattern.pat(ppos)
             if result is not None:
                 self.subresult = result
                 self.ppos = ppos
@@ -300,7 +307,7 @@
 
 class RepeatOneMatchResult(MatchResult):
     install_jitdriver('RepeatOne',
-                      greens=['nextppos', 'ctx.pattern'],
+                      greens=['nextppos', 'pattern'],
                       reds=['ptr', 'self', 'ctx'],
                       debugprint=(1, 0))   # indices in 'greens'
 
@@ -310,13 +317,14 @@
         self.start_ptr = ptr
         self.start_marks = marks
 
-    def find_first_result(self, ctx):
+    def find_first_result(self, ctx, pattern):
         ptr = self.start_ptr
         nextppos = self.nextppos
         while ptr >= self.minptr:
             ctx.jitdriver_RepeatOne.jit_merge_point(
-                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos)
-            result = sre_match(ctx, nextppos, ptr, self.start_marks)
+                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos,
+                pattern=pattern)
+            result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks)
             ptr -= 1
             if result is not None:
                 self.subresult = result
@@ -327,7 +335,7 @@
 
 class MinRepeatOneMatchResult(MatchResult):
     install_jitdriver('MinRepeatOne',
-                      greens=['nextppos', 'ppos3', 'ctx.pattern'],
+                      greens=['nextppos', 'ppos3', 'pattern'],
                       reds=['ptr', 'self', 'ctx'],
                       debugprint=(2, 0))   # indices in 'greens'
 
@@ -338,39 +346,40 @@
         self.start_ptr = ptr
         self.start_marks = marks
 
-    def find_first_result(self, ctx):
+    def find_first_result(self, ctx, pattern):
         ptr = self.start_ptr
         nextppos = self.nextppos
         ppos3 = self.ppos3
         while ptr <= self.maxptr:
             ctx.jitdriver_MinRepeatOne.jit_merge_point(
-                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3)
-            result = sre_match(ctx, nextppos, ptr, self.start_marks)
+                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3,
+                pattern=pattern)
+            result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks)
             if result is not None:
                 self.subresult = result
                 self.start_ptr = ptr
                 return self
-            if not self.next_char_ok(ctx, ptr, ppos3):
+            if not self.next_char_ok(ctx, pattern, ptr, ppos3):
                 break
             ptr += 1
 
-    def find_next_result(self, ctx):
+    def find_next_result(self, ctx, pattern):
         ptr = self.start_ptr
-        if not self.next_char_ok(ctx, ptr, self.ppos3):
+        if not self.next_char_ok(ctx, pattern, ptr, self.ppos3):
             return
         self.start_ptr = ptr + 1
-        return self.find_first_result(ctx)
+        return self.find_first_result(ctx, pattern)
 
-    def next_char_ok(self, ctx, ptr, ppos):
+    def next_char_ok(self, ctx, pattern, ptr, ppos):
         if ptr == ctx.end:
             return False
-        op = ctx.pat(ppos)
+        op = pattern.pat(ppos)
         for op1, checkerfn in unroll_char_checker:
             if op1 == op:
-                return checkerfn(ctx, ptr, ppos)
+                return checkerfn(ctx, pattern, ptr, ppos)
         # obscure case: it should be a single char pattern, but isn't
         # one of the opcodes in unroll_char_checker (see test_ext_opcode)
-        return sre_match(ctx, ppos, ptr, self.start_marks) is not None
+        return sre_match(ctx, pattern, ppos, ptr, self.start_marks) is not None
 
 class AbstractUntilMatchResult(MatchResult):
 
@@ -391,17 +400,17 @@
 
 class MaxUntilMatchResult(AbstractUntilMatchResult):
     install_jitdriver('MaxUntil',
-                      greens=['ppos', 'tailppos', 'match_more', 'ctx.pattern'],
+                      greens=['ppos', 'tailppos', 'match_more', 'pattern'],
                       reds=['ptr', 'marks', 'self', 'ctx'],
                       debugprint=(3, 0, 2))
 
-    def find_first_result(self, ctx):
-        return self.search_next(ctx, match_more=True)
+    def find_first_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, match_more=True)
 
-    def find_next_result(self, ctx):
-        return self.search_next(ctx, match_more=False)
+    def find_next_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, match_more=False)
 
-    def search_next(self, ctx, match_more):
+    def search_next(self, ctx, pattern, match_more):
         ppos = self.ppos
         tailppos = self.tailppos
         ptr = self.cur_ptr
@@ -409,12 +418,13 @@
         while True:
             ctx.jitdriver_MaxUntil.jit_merge_point(
                 ppos=ppos, tailppos=tailppos, match_more=match_more,
-                ptr=ptr, marks=marks, self=self, ctx=ctx)
+                ptr=ptr, marks=marks, self=self, ctx=ctx,
+                pattern=pattern)
             if match_more:
-                max = ctx.pat(ppos+2)
+                max = pattern.pat(ppos+2)
                 if max == rsre_char.MAXREPEAT or self.num_pending < max:
                     # try to match one more 'item'
-                    enum = sre_match(ctx, ppos + 3, ptr, marks)
+                    enum = sre_match(ctx, pattern, ppos + 3, ptr, marks)
                 else:
                     enum = None    # 'max' reached, no more matches
             else:
@@ -425,9 +435,9 @@
                 self.num_pending -= 1
                 ptr = p.ptr
                 marks = p.marks
-                enum = p.enum.move_to_next_result(ctx)
+                enum = p.enum.move_to_next_result(ctx, pattern)
             #
-            min = ctx.pat(ppos+1)
+            min = pattern.pat(ppos+1)
             if enum is not None:
                 # matched one more 'item'.  record it and continue.
                 last_match_length = ctx.match_end - ptr
@@ -447,7 +457,7 @@
             # 'item' no longer matches.
             if self.num_pending >= min:
                 # try to match 'tail' if we have enough 'item'
-                result = sre_match(ctx, tailppos, ptr, marks)
+                result = sre_match(ctx, pattern, tailppos, ptr, marks)
                 if result is not None:
                     self.subresult = result
                     self.cur_ptr = ptr
@@ -457,23 +467,23 @@
 
 class MinUntilMatchResult(AbstractUntilMatchResult):
 
-    def find_first_result(self, ctx):
-        return self.search_next(ctx, resume=False)
+    def find_first_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, resume=False)
 
-    def find_next_result(self, ctx):
-        return self.search_next(ctx, resume=True)
+    def find_next_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, resume=True)
 
-    def search_next(self, ctx, resume):
+    def search_next(self, ctx, pattern, resume):
         # XXX missing jit support here
         ppos = self.ppos
-        min = ctx.pat(ppos+1)
-        max = ctx.pat(ppos+2)
+        min = pattern.pat(ppos+1)
+        max = pattern.pat(ppos+2)
         ptr = self.cur_ptr
         marks = self.cur_marks
         while True:
             # try to match 'tail' if we have enough 'item'
             if not resume and self.num_pending >= min:
-                result = sre_match(ctx, self.tailppos, ptr, marks)
+                result = sre_match(ctx, pattern, self.tailppos, ptr, marks)
                 if result is not None:
                     self.subresult = result
                     self.cur_ptr = ptr
@@ -483,12 +493,12 @@
 
             if max == rsre_char.MAXREPEAT or self.num_pending < max:
                 # try to match one more 'item'
-                enum = sre_match(ctx, ppos + 3, ptr, marks)
+                enum = sre_match(ctx, pattern, ppos + 3, ptr, marks)
                 #
                 # zero-width match protection
                 if self.num_pending >= min:
                     while enum is not None and ptr == ctx.match_end:
-                        enum = enum.move_to_next_result(ctx)
+                        enum = enum.move_to_next_result(ctx, pattern)
             else:
                 enum = None    # 'max' reached, no more matches
 
@@ -502,7 +512,7 @@
                 self.num_pending -= 1
                 ptr = p.ptr
                 marks = p.marks
-                enum = p.enum.move_to_next_result(ctx)
+                enum = p.enum.move_to_next_result(ctx, pattern)
 
             # matched one more 'item'.  record it and continue
             self.pending = Pending(ptr, marks, enum, self.pending)
@@ -514,13 +524,13 @@
 
 @specializectx
 @jit.unroll_safe
-def sre_match(ctx, ppos, ptr, marks):
+def sre_match(ctx, pattern, ppos, ptr, marks):
     """Returns either None or a MatchResult object.  Usually we only need
     the first result, but there is the case of REPEAT...UNTIL where we
     need all results; in that case we use the method move_to_next_result()
     of the MatchResult."""
     while True:
-        op = ctx.pat(ppos)
+        op = pattern.pat(ppos)
         ppos += 1
 
         #jit.jit_debug("sre_match", op, ppos, ptr)
@@ -563,33 +573,33 @@
         elif op == OPCODE_ASSERT:
             # assert subpattern
             #  <0=skip> <1=back> 
-            ptr1 = ptr - ctx.pat(ppos+1)
+            ptr1 = ptr - pattern.pat(ppos+1)
             saved = ctx.fullmatch_only
             ctx.fullmatch_only = False
-            stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None
+            stop = ptr1 < 0 or sre_match(ctx, pattern, ppos + 2, ptr1, marks) is None
             ctx.fullmatch_only = saved
             if stop:
                 return
             marks = ctx.match_marks
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_ASSERT_NOT:
             # assert not subpattern
             #  <0=skip> <1=back> 
-            ptr1 = ptr - ctx.pat(ppos+1)
+            ptr1 = ptr - pattern.pat(ppos+1)
             saved = ctx.fullmatch_only
             ctx.fullmatch_only = False
-            stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks)
+            stop = (ptr1 >= 0 and sre_match(ctx, pattern, ppos + 2, ptr1, marks)
                                       is not None)
             ctx.fullmatch_only = saved
             if stop:
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_AT:
             # match at given position (e.g. at beginning, at boundary, etc.)
             #  
-            if not sre_at(ctx, ctx.pat(ppos), ptr):
+            if not sre_at(ctx, pattern.pat(ppos), ptr):
                 return
             ppos += 1
 
@@ -597,14 +607,14 @@
             # alternation
             #  <0=skip> code  ... 
             result = BranchMatchResult(ppos, ptr, marks)
-            return result.find_first_result(ctx)
+            return result.find_first_result(ctx, pattern)
 
         elif op == OPCODE_CATEGORY:
             # seems to be never produced, but used by some tests from
             # pypy/module/_sre/test
             #  
             if (ptr == ctx.end or
-                not rsre_char.category_dispatch(ctx.pat(ppos), ctx.str(ptr))):
+                not rsre_char.category_dispatch(pattern.pat(ppos), ctx.str(ptr))):
                 return
             ptr += 1
             ppos += 1
@@ -612,7 +622,7 @@
         elif op == OPCODE_GROUPREF:
             # match backreference
             #  
-            startptr, length = get_group_ref(marks, ctx.pat(ppos))
+            startptr, length = get_group_ref(marks, pattern.pat(ppos))
             if length < 0:
                 return     # group was not previously defined
             if not match_repeated(ctx, ptr, startptr, length):
@@ -623,7 +633,7 @@
         elif op == OPCODE_GROUPREF_IGNORE:
             # match backreference
             #  
-            startptr, length = get_group_ref(marks, ctx.pat(ppos))
+            startptr, length = get_group_ref(marks, pattern.pat(ppos))
             if length < 0:
                 return     # group was not previously defined
             if not match_repeated_ignore(ctx, ptr, startptr, length):
@@ -634,44 +644,44 @@
         elif op == OPCODE_GROUPREF_EXISTS:
             # conditional match depending on the existence of a group
             #    codeyes  codeno ...
-            _, length = get_group_ref(marks, ctx.pat(ppos))
+            _, length = get_group_ref(marks, pattern.pat(ppos))
             if length >= 0:
                 ppos += 2                  # jump to 'codeyes'
             else:
-                ppos += ctx.pat(ppos+1)    # jump to 'codeno'
+                ppos += pattern.pat(ppos+1)    # jump to 'codeno'
 
         elif op == OPCODE_IN:
             # match set member (or non_member)
             #   
-            if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1,
+            if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1,
                                                              ctx.str(ptr)):
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
             ptr += 1
 
         elif op == OPCODE_IN_IGNORE:
             # match set member (or non_member), ignoring case
             #   
-            if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1,
+            if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1,
                                                              ctx.lowstr(ptr)):
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
             ptr += 1
 
         elif op == OPCODE_INFO:
             # optimization info block
             #  <0=skip> <1=flags> <2=min> ...
-            if (ctx.end - ptr) < ctx.pat(ppos+2):
+            if (ctx.end - ptr) < pattern.pat(ppos+2):
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_JUMP:
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_LITERAL:
             # match literal string
             #  
-            if ptr >= ctx.end or ctx.str(ptr) != ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.str(ptr) != pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -679,7 +689,7 @@
         elif op == OPCODE_LITERAL_IGNORE:
             # match literal string, ignoring case
             #  
-            if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.lowstr(ptr) != pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -687,14 +697,14 @@
         elif op == OPCODE_MARK:
             # set mark
             #  
-            gid = ctx.pat(ppos)
+            gid = pattern.pat(ppos)
             marks = Mark(gid, ptr, marks)
             ppos += 1
 
         elif op == OPCODE_NOT_LITERAL:
             # match if it's not a literal string
             #  
-            if ptr >= ctx.end or ctx.str(ptr) == ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.str(ptr) == pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -702,7 +712,7 @@
         elif op == OPCODE_NOT_LITERAL_IGNORE:
             # match if it's not a literal string, ignoring case
             #  
-            if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.lowstr(ptr) == pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -715,22 +725,22 @@
 
             # decode the later UNTIL operator to see if it is actually
             # a MAX_UNTIL or MIN_UNTIL
-            untilppos = ppos + ctx.pat(ppos)
+            untilppos = ppos + pattern.pat(ppos)
             tailppos = untilppos + 1
-            op = ctx.pat(untilppos)
+            op = pattern.pat(untilppos)
             if op == OPCODE_MAX_UNTIL:
                 # the hard case: we have to match as many repetitions as
                 # possible, followed by the 'tail'.  we do this by
                 # remembering each state for each possible number of
                 # 'item' matching.
                 result = MaxUntilMatchResult(ppos, tailppos, ptr, marks)
-                return result.find_first_result(ctx)
+                return result.find_first_result(ctx, pattern)
 
             elif op == OPCODE_MIN_UNTIL:
                 # first try to match the 'tail', and if it fails, try
                 # to match one more 'item' and try again
                 result = MinUntilMatchResult(ppos, tailppos, ptr, marks)
-                return result.find_first_result(ctx)
+                return result.find_first_result(ctx, pattern)
 
             else:
                 raise Error("missing UNTIL after REPEAT")
@@ -743,17 +753,18 @@
             # use the MAX_REPEAT operator.
             #   <1=min> <2=max> item  tail
             start = ptr
-            minptr = start + ctx.pat(ppos+1)
+            minptr = start + pattern.pat(ppos+1)
             if minptr > ctx.end:
                 return    # cannot match
-            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2),
+            ptr = find_repetition_end(ctx, pattern, ppos+3, start,
+                                      pattern.pat(ppos+2),
                                       marks)
             # when we arrive here, ptr points to the tail of the target
             # string.  check if the rest of the pattern matches,
             # and backtrack if not.
-            nextppos = ppos + ctx.pat(ppos)
+            nextppos = ppos + pattern.pat(ppos)
             result = RepeatOneMatchResult(nextppos, minptr, ptr, marks)
-            return result.find_first_result(ctx)
+            return result.find_first_result(ctx, pattern)
 
         elif op == OPCODE_MIN_REPEAT_ONE:
             # match repeated sequence (minimizing regexp).
@@ -763,26 +774,26 @@
             # use the MIN_REPEAT operator.
             #   <1=min> <2=max> item  tail
             start = ptr
-            min = ctx.pat(ppos+1)
+            min = pattern.pat(ppos+1)
             if min > 0:
                 minptr = ptr + min
                 if minptr > ctx.end:
                     return   # cannot match
                 # count using pattern min as the maximum
-                ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks)
+                ptr = find_repetition_end(ctx, pattern, ppos+3, ptr, min, marks)
                 if ptr < minptr:
                     return   # did not match minimum number of times
 
             maxptr = ctx.end
-            max = ctx.pat(ppos+2)
+            max = pattern.pat(ppos+2)
             if max != rsre_char.MAXREPEAT:
                 maxptr1 = start + max
                 if maxptr1 <= maxptr:
                     maxptr = maxptr1
-            nextppos = ppos + ctx.pat(ppos)
+            nextppos = ppos + pattern.pat(ppos)
             result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr,
                                              ptr, marks)
-            return result.find_first_result(ctx)
+            return result.find_first_result(ctx, pattern)
 
         else:
             raise Error("bad pattern code %d" % op)
@@ -816,7 +827,7 @@
     return True
 
 @specializectx
-def find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+def find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks):
     end = ctx.end
     ptrp1 = ptr + 1
     # First get rid of the cases where we don't have room for any match.
@@ -826,16 +837,16 @@
     # The idea is to be fast for cases like re.search("b+"), where we expect
     # the common case to be a non-match.  It's much faster with the JIT to
     # have the non-match inlined here rather than detect it in the fre() call.
-    op = ctx.pat(ppos)
+    op = pattern.pat(ppos)
     for op1, checkerfn in unroll_char_checker:
         if op1 == op:
-            if checkerfn(ctx, ptr, ppos):
+            if checkerfn(ctx, pattern, ptr, ppos):
                 break
             return ptr
     else:
         # obscure case: it should be a single char pattern, but isn't
         # one of the opcodes in unroll_char_checker (see test_ext_opcode)
-        return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks)
+        return general_find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks)
     # It matches at least once.  If maxcount == 1 (relatively common),
     # then we are done.
     if maxcount == 1:
@@ -846,14 +857,14 @@
         end1 = ptr + maxcount
         if end1 <= end:
             end = end1
-    op = ctx.pat(ppos)
+    op = pattern.pat(ppos)
     for op1, fre in unroll_fre_checker:
         if op1 == op:
-            return fre(ctx, ptrp1, end, ppos)
+            return fre(ctx, pattern, ptrp1, end, ppos)
     raise Error("rsre.find_repetition_end[%d]" % op)
 
 @specializectx
-def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+def general_find_repetition_end(ctx, patern, ppos, ptr, maxcount, marks):
     # moved into its own JIT-opaque function
     end = ctx.end
     if maxcount != rsre_char.MAXREPEAT:
@@ -861,63 +872,65 @@
         end1 = ptr + maxcount
         if end1 <= end:
             end = end1
-    while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None:
+    while ptr < end and sre_match(ctx, patern, ppos, ptr, marks) is not None:
         ptr += 1
     return ptr
 
 @specializectx
-def match_ANY(ctx, ptr, ppos):   # dot wildcard.
+def match_ANY(ctx, pattern, ptr, ppos):   # dot wildcard.
     return not rsre_char.is_linebreak(ctx.str(ptr))
-def match_ANY_ALL(ctx, ptr, ppos):
+def match_ANY_ALL(ctx, pattern, ptr, ppos):
     return True    # match anything (including a newline)
 @specializectx
-def match_IN(ctx, ptr, ppos):
-    return rsre_char.check_charset(ctx, ppos+2, ctx.str(ptr))
+def match_IN(ctx, pattern, ptr, ppos):
+    return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.str(ptr))
 @specializectx
-def match_IN_IGNORE(ctx, ptr, ppos):
-    return rsre_char.check_charset(ctx, ppos+2, ctx.lowstr(ptr))
+def match_IN_IGNORE(ctx, pattern, ptr, ppos):
+    return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.lowstr(ptr))
 @specializectx
-def match_LITERAL(ctx, ptr, ppos):
-    return ctx.str(ptr) == ctx.pat(ppos+1)
+def match_LITERAL(ctx, pattern, ptr, ppos):
+    return ctx.str(ptr) == pattern.pat(ppos+1)
 @specializectx
-def match_LITERAL_IGNORE(ctx, ptr, ppos):
-    return ctx.lowstr(ptr) == ctx.pat(ppos+1)
+def match_LITERAL_IGNORE(ctx, pattern, ptr, ppos):
+    return ctx.lowstr(ptr) == pattern.pat(ppos+1)
 @specializectx
-def match_NOT_LITERAL(ctx, ptr, ppos):
-    return ctx.str(ptr) != ctx.pat(ppos+1)
+def match_NOT_LITERAL(ctx, pattern, ptr, ppos):
+    return ctx.str(ptr) != pattern.pat(ppos+1)
 @specializectx
-def match_NOT_LITERAL_IGNORE(ctx, ptr, ppos):
-    return ctx.lowstr(ptr) != ctx.pat(ppos+1)
+def match_NOT_LITERAL_IGNORE(ctx, pattern, ptr, ppos):
+    return ctx.lowstr(ptr) != pattern.pat(ppos+1)
 
 def _make_fre(checkerfn):
     if checkerfn == match_ANY_ALL:
-        def fre(ctx, ptr, end, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
             return end
     elif checkerfn == match_IN:
         install_jitdriver_spec('MatchIn',
-                               greens=['ppos', 'ctx.pattern'],
+                               greens=['ppos', 'pattern'],
                                reds=['ptr', 'end', 'ctx'],
                                debugprint=(1, 0))
         @specializectx
-        def fre(ctx, ptr, end, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
             while True:
                 ctx.jitdriver_MatchIn.jit_merge_point(ctx=ctx, ptr=ptr,
-                                                      end=end, ppos=ppos)
-                if ptr < end and checkerfn(ctx, ptr, ppos):
+                                                      end=end, ppos=ppos,
+                                                      pattern=pattern)
+                if ptr < end and checkerfn(ctx, pattern, ptr, ppos):
                     ptr += 1
                 else:
                     return ptr
     elif checkerfn == match_IN_IGNORE:
         install_jitdriver_spec('MatchInIgnore',
-                               greens=['ppos', 'ctx.pattern'],
+                               greens=['ppos', 'pattern'],
                                reds=['ptr', 'end', 'ctx'],
                                debugprint=(1, 0))
         @specializectx
-        def fre(ctx, ptr, end, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
             while True:
                 ctx.jitdriver_MatchInIgnore.jit_merge_point(ctx=ctx, ptr=ptr,
-                                                            end=end, ppos=ppos)
-                if ptr < end and checkerfn(ctx, ptr, ppos):
+                                                            end=end, ppos=ppos,
+                                                            pattern=pattern)
+                if ptr < end and checkerfn(ctx, pattern, ptr, ppos):
                     ptr += 1
                 else:
                     return ptr
@@ -925,8 +938,8 @@
         # in the other cases, the fre() function is not JITted at all
         # and is present as a residual call.
         @specializectx
-        def fre(ctx, ptr, end, ppos):
-            while ptr < end and checkerfn(ctx, ptr, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
+            while ptr < end and checkerfn(ctx, pattern, ptr, ppos):
                 ptr += 1
             return ptr
     fre = func_with_new_name(fre, 'fre_' + checkerfn.__name__)
@@ -1037,10 +1050,11 @@
     return start, end
 
 def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False):
+    assert isinstance(pattern, CompiledPattern)
     start, end = _adjust(start, end, len(string))
-    ctx = StrMatchContext(pattern, string, start, end, flags)
+    ctx = StrMatchContext(string, start, end, flags)
     ctx.fullmatch_only = fullmatch
-    if match_context(ctx):
+    if match_context(ctx, pattern):
         return ctx
     else:
         return None
@@ -1049,105 +1063,106 @@
     return match(pattern, string, start, end, flags, fullmatch=True)
 
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
+    assert isinstance(pattern, CompiledPattern)
     start, end = _adjust(start, end, len(string))
-    ctx = StrMatchContext(pattern, string, start, end, flags)
-    if search_context(ctx):
+    ctx = StrMatchContext(string, start, end, flags)
+    if search_context(ctx, pattern):
         return ctx
     else:
         return None
 
 install_jitdriver('Match',
-                  greens=['ctx.pattern'], reds=['ctx'],
+                  greens=['pattern'], reds=['ctx'],
                   debugprint=(0,))
 
-def match_context(ctx):
+def match_context(ctx, pattern):
     ctx.original_pos = ctx.match_start
     if ctx.end < ctx.match_start:
         return False
-    ctx.jitdriver_Match.jit_merge_point(ctx=ctx)
-    return sre_match(ctx, 0, ctx.match_start, None) is not None
+    ctx.jitdriver_Match.jit_merge_point(ctx=ctx, pattern=pattern)
+    return sre_match(ctx, pattern, 0, ctx.match_start, None) is not None
 
-def search_context(ctx):
+def search_context(ctx, pattern):
     ctx.original_pos = ctx.match_start
     if ctx.end < ctx.match_start:
         return False
     base = 0
     charset = False
-    if ctx.pat(base) == OPCODE_INFO:
-        flags = ctx.pat(2)
+    if pattern.pat(base) == OPCODE_INFO:
+        flags = pattern.pat(2)
         if flags & rsre_char.SRE_INFO_PREFIX:
-            if ctx.pat(5) > 1:
-                return fast_search(ctx)
+            if pattern.pat(5) > 1:
+                return fast_search(ctx, pattern)
         else:
             charset = (flags & rsre_char.SRE_INFO_CHARSET)
-        base += 1 + ctx.pat(1)
-    if ctx.pat(base) == OPCODE_LITERAL:
-        return literal_search(ctx, base)
+        base += 1 + pattern.pat(1)
+    if pattern.pat(base) == OPCODE_LITERAL:
+        return literal_search(ctx, pattern, base)
     if charset:
-        return charset_search(ctx, base)
-    return regular_search(ctx, base)
+        return charset_search(ctx, pattern, base)
+    return regular_search(ctx, pattern, base)
 
 install_jitdriver('RegularSearch',
-                  greens=['base', 'ctx.pattern'],
+                  greens=['base', 'pattern'],
                   reds=['start', 'ctx'],
                   debugprint=(1, 0))
 
-def regular_search(ctx, base):
+def regular_search(ctx, pattern, base):
     start = ctx.match_start
     while start <= ctx.end:
         ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, start=start,
-                                                    base=base)
-        if sre_match(ctx, base, start, None) is not None:
+                                                    base=base, pattern=pattern)
+        if sre_match(ctx, pattern, base, start, None) is not None:
             ctx.match_start = start
             return True
         start += 1
     return False
 
 install_jitdriver_spec("LiteralSearch",
-                       greens=['base', 'character', 'ctx.pattern'],
+                       greens=['base', 'character', 'pattern'],
                        reds=['start', 'ctx'],
                        debugprint=(2, 0, 1))
 @specializectx
-def literal_search(ctx, base):
+def literal_search(ctx, pattern, base):
     # pattern starts with a literal character.  this is used
     # for short prefixes, and if fast search is disabled
-    character = ctx.pat(base + 1)
+    character = pattern.pat(base + 1)
     base += 2
     start = ctx.match_start
     while start < ctx.end:
         ctx.jitdriver_LiteralSearch.jit_merge_point(ctx=ctx, start=start,
-                                          base=base, character=character)
+                                          base=base, character=character, pattern=pattern)
         if ctx.str(start) == character:
-            if sre_match(ctx, base, start + 1, None) is not None:
+            if sre_match(ctx, pattern, base, start + 1, None) is not None:
                 ctx.match_start = start
                 return True
         start += 1
     return False
 
 install_jitdriver_spec("CharsetSearch",
-                       greens=['base', 'ctx.pattern'],
+                       greens=['base', 'pattern'],
                        reds=['start', 'ctx'],
                        debugprint=(1, 0))
 @specializectx
-def charset_search(ctx, base):
+def charset_search(ctx, pattern, base):
     # pattern starts with a character from a known set
     start = ctx.match_start
     while start < ctx.end:
         ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start,
-                                                    base=base)
-        if rsre_char.check_charset(ctx, 5, ctx.str(start)):
-            if sre_match(ctx, base, start, None) is not None:
+                                                    base=base, pattern=pattern)
+        if rsre_char.check_charset(ctx, pattern, 5, ctx.str(start)):
+            if sre_match(ctx, pattern, base, start, None) is not None:
                 ctx.match_start = start
                 return True
         start += 1
     return False
 
 install_jitdriver_spec('FastSearch',
-                       greens=['i', 'prefix_len', 'ctx.pattern'],
+                       greens=['i', 'prefix_len', 'pattern'],
                        reds=['string_position', 'ctx'],
                        debugprint=(2, 0))
 @specializectx
-def fast_search(ctx):
+def fast_search(ctx, pattern):
     # skips forward in a string as fast as possible using information from
     # an optimization info block
     #  <1=skip> <2=flags> <3=min> <4=...>
@@ -1155,17 +1170,18 @@
     string_position = ctx.match_start
     if string_position >= ctx.end:
         return False
-    prefix_len = ctx.pat(5)
+    prefix_len = pattern.pat(5)
     assert prefix_len >= 0
     i = 0
     while True:
         ctx.jitdriver_FastSearch.jit_merge_point(ctx=ctx,
-                string_position=string_position, i=i, prefix_len=prefix_len)
+                string_position=string_position, i=i, prefix_len=prefix_len,
+                pattern=pattern)
         char_ord = ctx.str(string_position)
-        if char_ord != ctx.pat(7 + i):
+        if char_ord != pattern.pat(7 + i):
             if i > 0:
                 overlap_offset = prefix_len + (7 - 1)
-                i = ctx.pat(overlap_offset + i)
+                i = pattern.pat(overlap_offset + i)
                 continue
         else:
             i += 1
@@ -1173,22 +1189,22 @@
                 # found a potential match
                 start = string_position + 1 - prefix_len
                 assert start >= 0
-                prefix_skip = ctx.pat(6)
+                prefix_skip = pattern.pat(6)
                 ptr = start + prefix_skip
-                #flags = ctx.pat(2)
+                #flags = pattern.pat(2)
                 #if flags & rsre_char.SRE_INFO_LITERAL:
                 #    # matched all of pure literal pattern
                 #    ctx.match_start = start
                 #    ctx.match_end = ptr
                 #    ctx.match_marks = None
                 #    return True
-                pattern_offset = ctx.pat(1) + 1
+                pattern_offset = pattern.pat(1) + 1
                 ppos_start = pattern_offset + 2 * prefix_skip
-                if sre_match(ctx, ppos_start, ptr, None) is not None:
+                if sre_match(ctx, pattern, ppos_start, ptr, None) is not None:
                     ctx.match_start = start
                     return True
                 overlap_offset = prefix_len + (7 - 1)
-                i = ctx.pat(overlap_offset + i)
+                i = pattern.pat(overlap_offset + i)
         string_position += 1
         if string_position >= ctx.end:
             return False
diff --git a/rpython/rlib/rsre/test/test_char.py b/rpython/rlib/rsre/test/test_char.py
--- a/rpython/rlib/rsre/test/test_char.py
+++ b/rpython/rlib/rsre/test/test_char.py
@@ -1,10 +1,16 @@
-from rpython.rlib.rsre import rsre_char
+from rpython.rlib.rsre import rsre_char, rsre_core
 from rpython.rlib.rsre.rsre_char import SRE_FLAG_LOCALE, SRE_FLAG_UNICODE
 
 def setup_module(mod):
     from rpython.rlib.unicodedata import unicodedb
     rsre_char.set_unicode_db(unicodedb)
 
+
+def check_charset(pattern, idx, char):
+    p = rsre_core.CompiledPattern(pattern)
+    return rsre_char.check_charset(Ctx(p), p, idx, char)
+
+
 UPPER_PI = 0x3a0
 LOWER_PI = 0x3c0
 INDIAN_DIGIT = 0x966
@@ -157,12 +163,12 @@
         pat_neg = [70, ord(cat) | 0x80, 0]
         for c in positive:
             assert unicodedb.category(ord(c)).startswith(cat)
-            assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
-            assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
+            assert check_charset(pat_pos, 0, ord(c))
+            assert not check_charset(pat_neg, 0, ord(c))
         for c in negative:
             assert not unicodedb.category(ord(c)).startswith(cat)
-            assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
-            assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
+            assert not check_charset(pat_pos, 0, ord(c))
+            assert check_charset(pat_neg, 0, ord(c))
 
     def cat2num(cat):
         return ord(cat[0]) | (ord(cat[1]) << 8)
@@ -173,17 +179,16 @@
         pat_neg = [70, cat2num(cat) | 0x80, 0]
         for c in positive:
             assert unicodedb.category(ord(c)) == cat
-            assert rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
-            assert not rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
+            assert check_charset(pat_pos, 0, ord(c))
+            assert not check_charset(pat_neg, 0, ord(c))
         for c in negative:
             assert unicodedb.category(ord(c)) != cat
-            assert not rsre_char.check_charset(Ctx(pat_pos), 0, ord(c))
-            assert rsre_char.check_charset(Ctx(pat_neg), 0, ord(c))
+            assert not check_charset(pat_pos, 0, ord(c))
+            assert check_charset(pat_neg, 0, ord(c))
 
     # test for how the common 'L&' pattern might be compiled
     pat = [70, cat2num('Lu'), 70, cat2num('Ll'), 70, cat2num('Lt'), 0]
-    assert rsre_char.check_charset(Ctx(pat), 0, 65)    # Lu
-    assert rsre_char.check_charset(Ctx(pat), 0, 99)    # Ll
-    assert rsre_char.check_charset(Ctx(pat), 0, 453)   # Lt
-    assert not rsre_char.check_charset(Ctx(pat), 0, 688)    # Lm
-    assert not rsre_char.check_charset(Ctx(pat), 0, 5870)   # Nl
+    assert check_charset(pat, 0, 65)    # Lu
+    assert check_charset(pat, 0, 99)    # Lcheck_charset(pat, 0, 453)   # Lt
+    assert not check_charset(pat, 0, 688)    # Lm
+    assert not check_charset(pat, 0, 5870)   # Nl
diff --git a/rpython/rlib/rsre/test/test_ext_opcode.py b/rpython/rlib/rsre/test/test_ext_opcode.py
--- a/rpython/rlib/rsre/test/test_ext_opcode.py
+++ b/rpython/rlib/rsre/test/test_ext_opcode.py
@@ -17,10 +17,10 @@
     # it's a valid optimization because \1 is always one character long
     r = [MARK, 0, ANY, MARK, 1, REPEAT_ONE, 6, 0, MAXREPEAT, 
          GROUPREF, 0, SUCCESS, SUCCESS]
-    assert rsre_core.match(r, "aaa").match_end == 3
+    assert rsre_core.match(rsre_core.CompiledPattern(r), "aaa").match_end == 3
 
 def test_min_repeat_one_with_backref():
     # Python 3.5 compiles "(.)\1*?b" using MIN_REPEAT_ONE
     r = [MARK, 0, ANY, MARK, 1, MIN_REPEAT_ONE, 6, 0, MAXREPEAT,
          GROUPREF, 0, SUCCESS, LITERAL, 98, SUCCESS]
-    assert rsre_core.match(r, "aaab").match_end == 4
+    assert rsre_core.match(rsre_core.CompiledPattern(r), "aaab").match_end == 4
diff --git a/rpython/rlib/rsre/test/test_match.py b/rpython/rlib/rsre/test/test_match.py
--- a/rpython/rlib/rsre/test/test_match.py
+++ b/rpython/rlib/rsre/test/test_match.py
@@ -9,7 +9,7 @@
 def test_get_code_repetition():
     c1 = get_code(r"a+")
     c2 = get_code(r"a+")
-    assert c1 == c2
+    assert c1.pattern == c2.pattern
 
 
 class TestMatch:
@@ -305,6 +305,6 @@
         rsre_char.set_unicode_db(unicodedb)
         #
         r = get_code(u"[\U00010428-\U0001044f]", re.I)
-        assert r.count(27) == 1       # OPCODE_RANGE
-        r[r.index(27)] = 32           # => OPCODE_RANGE_IGNORE
+        assert r.pattern.count(27) == 1       # OPCODE_RANGE
+        r.pattern[r.pattern.index(27)] = 32   # => OPCODE_RANGE_IGNORE
         assert rsre_core.match(r, u"\U00010428")
diff --git a/rpython/rlib/rsre/test/test_re.py b/rpython/rlib/rsre/test/test_re.py
--- a/rpython/rlib/rsre/test/test_re.py
+++ b/rpython/rlib/rsre/test/test_re.py
@@ -426,31 +426,6 @@
         assert pat.match(p) is not None
         assert pat.match(p).span() == (0,256)
 
-    def test_pickling(self):
-        import pickle
-        self.pickle_test(pickle)
-        import cPickle
-        self.pickle_test(cPickle)
-        # old pickles expect the _compile() reconstructor in sre module
-        import warnings
-        original_filters = warnings.filters[:]
-        try:
-            warnings.filterwarnings("ignore", "The sre module is deprecated",
-                                    DeprecationWarning)
-            from sre import _compile
-        finally:
-            warnings.filters = original_filters
-
-    def pickle_test(self, pickle):
-        oldpat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)')
-        s = pickle.dumps(oldpat)
-        newpat = pickle.loads(s)
-        # Not using object identity for _sre.py, since some Python builds do
-        # not seem to preserve that in all cases (observed on an UCS-4 build
-        # of 2.4.1).
-        #self.assertEqual(oldpat, newpat)
-        assert oldpat.__dict__ == newpat.__dict__
-
     def test_constants(self):
         assert re.I == re.IGNORECASE
         assert re.L == re.LOCALE
diff --git a/rpython/rlib/rsre/test/test_zinterp.py b/rpython/rlib/rsre/test/test_zinterp.py
--- a/rpython/rlib/rsre/test/test_zinterp.py
+++ b/rpython/rlib/rsre/test/test_zinterp.py
@@ -11,6 +11,7 @@
     rsre_core.search(pattern, string)
     #
     unicodestr = unichr(n) * n
+    pattern = rsre_core.CompiledPattern(pattern)
     ctx = rsre_core.UnicodeMatchContext(pattern, unicodestr,
                                         0, len(unicodestr), 0)
     rsre_core.search_context(ctx)
diff --git a/rpython/rlib/rsre/test/test_zjit.py b/rpython/rlib/rsre/test/test_zjit.py
--- a/rpython/rlib/rsre/test/test_zjit.py
+++ b/rpython/rlib/rsre/test/test_zjit.py
@@ -6,18 +6,20 @@
 from rpython.rtyper.annlowlevel import llstr, hlstr
 
 def entrypoint1(r, string, repeat):
-    r = array2list(r)
+    r = rsre_core.CompiledPattern(array2list(r))
     string = hlstr(string)
     match = None
     for i in range(repeat):
         match = rsre_core.match(r, string)
+        if match is None:
+            return -1
     if match is None:
         return -1
     else:
         return match.match_end
 
 def entrypoint2(r, string, repeat):
-    r = array2list(r)
+    r = rsre_core.CompiledPattern(array2list(r))
     string = hlstr(string)
     match = None
     for i in range(repeat):
@@ -48,13 +50,13 @@
 
     def meta_interp_match(self, pattern, string, repeat=1):
         r = get_code(pattern)
-        return self.meta_interp(entrypoint1, [list2array(r), llstr(string),
+        return self.meta_interp(entrypoint1, [list2array(r.pattern), llstr(string),
                                               repeat],
                                 listcomp=True, backendopt=True)
 
     def meta_interp_search(self, pattern, string, repeat=1):
         r = get_code(pattern)
-        return self.meta_interp(entrypoint2, [list2array(r), llstr(string),
+        return self.meta_interp(entrypoint2, [list2array(r.pattern), llstr(string),
                                               repeat],
                                 listcomp=True, backendopt=True)
 
@@ -166,3 +168,9 @@
         res = self.meta_interp_search(r"b+", "a"*30 + "b")
         assert res == 30
         self.check_resops(call=0)
+
+    def test_match_jit_bug(self):
+        pattern = ".a" * 2500
+        text = "a" * 6000
+        res = self.meta_interp_match(pattern, text, repeat=10)
+        assert res != -1
diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py
--- a/rpython/rlib/test/test_jit.py
+++ b/rpython/rlib/test/test_jit.py
@@ -225,8 +225,10 @@
     def test_green_field(self):
         def get_printable_location(xfoo):
             return str(ord(xfoo))   # xfoo must be annotated as a character
-        myjitdriver = JitDriver(greens=['x.foo'], reds=['n', 'x'],
+        # green fields are disabled!
+        pytest.raises(ValueError, JitDriver, greens=['x.foo'], reds=['n', 'x'],
                                 get_printable_location=get_printable_location)
+        return
         class A(object):
             _immutable_fields_ = ['foo']
         def fn(n):

From pypy.commits at gmail.com  Thu Mar 29 06:04:14 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 03:04:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: try to see whether disk I/O is the culprit
Message-ID: <5abcba1e.0a561c0a.157ea.9b17@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r94171:6b57c7b53662
Date: 2018-03-29 11:47 +0200
http://bitbucket.org/pypy/pypy/changeset/6b57c7b53662/

Log:	try to see whether disk I/O is the culprit

diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -29,7 +29,7 @@
 
 @given(data=st_readline(),
        mode=st.sampled_from(['\r', '\n', '\r\n', '']))
- at settings(deadline=None)
+ at settings(deadline=None, database=None)
 def test_readline(space, data, mode):
     txt, limits = data
     w_stream = W_BytesIO(space)

From pypy.commits at gmail.com  Thu Mar 29 06:04:16 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 03:04:16 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: yet another workaround suggested by david
Message-ID: <5abcba20.55a5df0a.96199.e2a2@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r94172:2487a89f1e8c
Date: 2018-03-29 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/2487a89f1e8c/

Log:	yet another workaround suggested by david

diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -7,6 +7,11 @@
 from pypy.module._io.interp_bytesio import W_BytesIO
 from pypy.module._io.interp_textio import W_TextIOWrapper, DecodeBuffer
 
+# workaround suggestion for slowness by David McIver:
+# force hypothesis to initialize some lazy stuff
+# (which takes a lot of time, which trips the timer otherwise)
+st.text().example()
+
 def translate_newlines(text):
     text = text.replace(u'\r\n', u'\n')
     text = text.replace(u'\r', u'\n')

From pypy.commits at gmail.com  Thu Mar 29 14:04:48 2018
From: pypy.commits at gmail.com (mjacob)
Date: Thu, 29 Mar 2018 11:04:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: IN-PROGRESS: Implement
 sys.get_asyncgen_hooks() and sys.get_asyncgen_hooks(). This needs to made
 thread-local eventually.
Message-ID: <5abd2ac0.4fa1df0a.14a28.8a69@mx.google.com>

Author: Manuel Jacob 
Branch: py3.6
Changeset: r94173:aa53932d50e0
Date: 2018-03-22 17:00 +0100
http://bitbucket.org/pypy/pypy/changeset/aa53932d50e0/

Log:	IN-PROGRESS: Implement sys.get_asyncgen_hooks() and
	sys.get_asyncgen_hooks(). This needs to made thread-local
	eventually.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -116,6 +116,8 @@
         'flags'                 : 'app.null_sysflags',
         '_xoptions'             : 'app.null__xoptions',
         'implementation'        : 'app.implementation',
+        'get_asyncgen_hooks'    : 'app.get_asyncgen_hooks',
+        'set_asyncgen_hooks'    : 'app.set_asyncgen_hooks',
 
         # these six attributes are here only during tests;
         # they are removed before translation
diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py
--- a/pypy/module/sys/app.py
+++ b/pypy/module/sys/app.py
@@ -111,6 +111,29 @@
 null__xoptions = {}
 
 
+class asyncgen_hooks(metaclass=structseqtype):
+    name = "asyncgen_hooks"
+
+    firstiter = structseqfield(0)
+    finalizer = structseqfield(1)
+
+# FIXME: make this thread-local
+_current_asyncgen_hooks = asyncgen_hooks((None, None))
+
+def get_asyncgen_hooks():
+    return _current_asyncgen_hooks
+
+_default_arg = object()
+
+def set_asyncgen_hooks(firstiter=_default_arg, finalizer=_default_arg):
+    global _current_asyncgen_hooks
+    if firstiter is _default_arg:
+        firstiter = _current_asyncgen_hooks.firstiter
+    if finalizer is _default_arg:
+        finalizer = _current_asyncgen_hooks.finalizer
+    _current_asyncgen_hooks = asyncgen_hooks((firstiter, finalizer))
+
+
 implementation = SimpleNamespace(
     name='pypy',
     version=sys.version_info,
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -713,6 +713,33 @@
         assert not sys.is_finalizing()
         # xxx should also test when it is True, but maybe not worth the effort
 
+    def test_asyncgen_hooks(self):
+        import sys
+        old = sys.get_asyncgen_hooks()
+        assert old.firstiter is None
+        assert old.finalizer is None
+
+        firstiter = lambda *a: None
+        sys.set_asyncgen_hooks(firstiter=firstiter)
+        hooks = sys.get_asyncgen_hooks()
+        assert hooks.firstiter is firstiter
+        assert hooks[0] is firstiter
+        assert hooks.finalizer is None
+        assert hooks[1] is None
+
+        finalizer = lambda *a: None
+        sys.set_asyncgen_hooks(finalizer=finalizer)
+        hooks = sys.get_asyncgen_hooks()
+        assert hooks.firstiter is firstiter
+        assert hooks[0] is firstiter
+        assert hooks.finalizer is finalizer
+        assert hooks[1] is finalizer
+
+        sys.set_asyncgen_hooks(*old)
+        cur = sys.get_asyncgen_hooks()
+        assert cur.firstiter is None
+        assert cur.finalizer is None
+
 
 class AppTestSysSettracePortedFromCpython(object):
     def test_sys_settrace(self):

From pypy.commits at gmail.com  Thu Mar 29 14:23:31 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 11:23:31 -0700 (PDT)
Subject: [pypy-commit] pypy pyparser-improvements: remove redundant assert
Message-ID: <5abd2f23.09e61c0a.3f9bc.bbeb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: pyparser-improvements
Changeset: r94175:c6a2bdbe2a72
Date: 2018-03-29 12:34 +0200
http://bitbucket.org/pypy/pypy/changeset/c6a2bdbe2a72/

Log:	remove redundant assert

diff --git a/pypy/interpreter/pyparser/parser.py b/pypy/interpreter/pyparser/parser.py
--- a/pypy/interpreter/pyparser/parser.py
+++ b/pypy/interpreter/pyparser/parser.py
@@ -267,7 +267,6 @@
                     while state[1] and not state[0]:
                         self.pop()
                         if self.stack is None:
-                            assert self.stack is None
                             # Parsing is done.
                             return True
                         dfa = self.stack.dfa
@@ -287,7 +286,6 @@
                 if is_accepting:
                     self.pop()
                     if self.stack is None:
-                        assert self.stack is None
                         raise ParseError("too much input", token_type, value,
                                          lineno, column, line)
                 else:

From pypy.commits at gmail.com  Thu Mar 29 14:23:33 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 11:23:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: try to see whether disk I/O is the
 culprit
Message-ID: <5abd2f25.8e6f1c0a.6f61e.3df8@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r94176:ac6363dcdcbb
Date: 2018-03-29 11:47 +0200
http://bitbucket.org/pypy/pypy/changeset/ac6363dcdcbb/

Log:	try to see whether disk I/O is the culprit

diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -29,7 +29,7 @@
 
 @given(data=st_readline(),
        mode=st.sampled_from(['\r', '\n', '\r\n', '']))
- at settings(deadline=None)
+ at settings(deadline=None, database=None)
 def test_readline(space, data, mode):
     txt, limits = data
     w_stream = W_BytesIO(space)

From pypy.commits at gmail.com  Thu Mar 29 14:23:35 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 11:23:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: yet another workaround suggested by
 david
Message-ID: <5abd2f27.0f8d1c0a.7caf9.6012@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r94177:058357bd35eb
Date: 2018-03-29 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/058357bd35eb/

Log:	yet another workaround suggested by david

diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -7,6 +7,11 @@
 from pypy.module._io.interp_bytesio import W_BytesIO
 from pypy.module._io.interp_textio import W_TextIOWrapper, DecodeBuffer
 
+# workaround suggestion for slowness by David McIver:
+# force hypothesis to initialize some lazy stuff
+# (which takes a lot of time, which trips the timer otherwise)
+st.text().example()
+
 def translate_newlines(text):
     text = text.replace(u'\r\n', u'\n')
     text = text.replace(u'\r', u'\n')

From pypy.commits at gmail.com  Fri Mar 30 02:29:32 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 29 Mar 2018 23:29:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: test and fix: in the on_abort hook,
 the descrs of guards are still None as it runs before optimization
Message-ID: <5abdd94c.e683500a.cae27.b357@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r94178:fe40f4eff666
Date: 2018-03-30 08:28 +0200
http://bitbucket.org/pypy/pypy/changeset/fe40f4eff666/

Log:	test and fix: in the on_abort hook, the descrs of guards are still
	None as it runs before optimization

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -113,22 +113,27 @@
             ofs = ops_offset.get(op, 0)
         num = op.getopnum()
         name = op.getopname()
+        repr = logops.repr_of_resop(op)
         if num == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
             greenkey = op.getarglist()[3:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, name,
-                                       logops.repr_of_resop(op),
+                                       repr,
                                        jd_sd.jitdriver.name,
                                        op.getarg(1).getint(),
                                        op.getarg(2).getint(),
                                        w_greenkey))
         elif op.is_guard():
-            l_w.append(GuardOp(name, ofs, logops.repr_of_resop(op),
-                op.getdescr().get_jitcounter_hash()))
+            descr = op.getdescr()
+            if descr is not None: # can be none in on_abort!
+                hash = op.getdescr().get_jitcounter_hash()
+            else:
+                hash = -1
+            l_w.append(GuardOp(name, ofs, repr, hash))
         else:
-            l_w.append(WrappedOp(name, ofs, logops.repr_of_resop(op)))
+            l_w.append(WrappedOp(name, ofs, repr))
     return l_w
 
 @unwrap_spec(offset=int, repr='text', name='text')
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -65,6 +65,17 @@
             if i != 1:
                 offset[op] = i
 
+        oplist_no_descrs = parse("""
+        [i1, i2, p2]
+        i3 = int_add(i1, i2)
+        debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0))
+        guard_nonnull(p2) []
+        guard_true(i3) []
+        """, namespace={'ptr0': code_gcref}).operations
+        for op in oplist_no_descrs:
+            if op.is_guard():
+                op.setdescr(None)
+
         class FailDescr(BasicFailDescr):
             def get_jitcounter_hash(self):
                 from rpython.rlib.rarithmetic import r_uint
@@ -101,7 +112,8 @@
         def interp_on_abort():
             if pypy_hooks.are_hooks_enabled():
                 pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver,
-                                    greenkey, 'blah', Logger(MockSD), [])
+                                    greenkey, 'blah', Logger(MockSD),
+                                    cls.oplist_no_descrs)
 
         space = cls.space
         cls.w_on_compile = space.wrap(interp2app(interp_on_compile))
@@ -111,10 +123,12 @@
         cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
+        cls.orig_oplist_no_descrs = oplist_no_descrs
         cls.w_sorted_keys = space.wrap(sorted(Counters.counter_names))
 
     def setup_method(self, meth):
         self.__class__.oplist = self.orig_oplist[:]
+        self.__class__.oplist_no_descrs = self.orig_oplist_no_descrs[:]
 
     def test_on_compile(self):
         import pypyjit
@@ -223,7 +237,11 @@
 
         pypyjit.set_abort_hook(hook)
         self.on_abort()
-        assert l == [('pypyjit', 'ABORT_TOO_LONG', [])]
+        assert len(l) == 1
+        name, reason, ops = l[0]
+        assert name == 'pypyjit'
+        assert reason == 'ABORT_TOO_LONG'
+        assert len(ops) == 4
 
     def test_creation(self):
         from pypyjit import ResOperation

From pypy.commits at gmail.com  Fri Mar 30 04:48:37 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 30 Mar 2018 01:48:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge default
Message-ID: <5abdf9e5.03b01c0a.232e0.a084@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: py3.5
Changeset: r94179:47bcad155e44
Date: 2018-03-29 20:23 +0200
http://bitbucket.org/pypy/pypy/changeset/47bcad155e44/

Log:	merge default

diff --git a/pypy/module/pypyjit/hooks.py b/pypy/module/pypyjit/hooks.py
--- a/pypy/module/pypyjit/hooks.py
+++ b/pypy/module/pypyjit/hooks.py
@@ -7,12 +7,20 @@
     WrappedOp, W_JitLoopInfo, wrap_oplist)
 
 class PyPyJitIface(JitHookInterface):
+    def are_hooks_enabled(self):
+        space = self.space
+        cache = space.fromcache(Cache)
+        return (cache.w_compile_hook is not None or
+                cache.w_abort_hook is not None or
+                cache.w_trace_too_long_hook is not None)
+
+
     def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations):
         space = self.space
         cache = space.fromcache(Cache)
         if cache.in_recursion:
             return
-        if space.is_true(cache.w_abort_hook):
+        if cache.w_abort_hook is not None:
             cache.in_recursion = True
             oplist_w = wrap_oplist(space, logops, operations)
             try:
@@ -33,7 +41,7 @@
         cache = space.fromcache(Cache)
         if cache.in_recursion:
             return
-        if space.is_true(cache.w_trace_too_long_hook):
+        if cache.w_trace_too_long_hook is not None:
             cache.in_recursion = True
             try:
                 try:
@@ -62,7 +70,7 @@
         cache = space.fromcache(Cache)
         if cache.in_recursion:
             return
-        if space.is_true(cache.w_compile_hook):
+        if cache.w_compile_hook is not None:
             w_debug_info = W_JitLoopInfo(space, debug_info, is_bridge,
                                          cache.compile_hook_with_ops)
             cache.in_recursion = True
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -21,9 +21,10 @@
     no = 0
 
     def __init__(self, space):
-        self.w_compile_hook = space.w_None
-        self.w_abort_hook = space.w_None
-        self.w_trace_too_long_hook = space.w_None
+        self.w_compile_hook = None
+        self.w_abort_hook = None
+        self.w_trace_too_long_hook = None
+        self.compile_hook_with_ops = False
 
     def getno(self):
         self.no += 1
@@ -58,7 +59,8 @@
     jit hook won't be called for that.
     """
     cache = space.fromcache(Cache)
-    assert w_hook is not None
+    if space.is_w(w_hook, space.w_None):
+        w_hook = None
     cache.w_compile_hook = w_hook
     cache.compile_hook_with_ops = bool(operations)
     cache.in_recursion = NonConstant(False)
@@ -77,7 +79,8 @@
     as attributes on JitLoopInfo object.
     """
     cache = space.fromcache(Cache)
-    assert w_hook is not None
+    if space.is_w(w_hook, space.w_None):
+        w_hook = None
     cache.w_abort_hook = w_hook
     cache.in_recursion = NonConstant(False)
 
@@ -92,14 +95,15 @@
         hook(jitdriver_name, greenkey)
     """
     cache = space.fromcache(Cache)
-    assert w_hook is not None
+    if space.is_w(w_hook, space.w_None):
+        w_hook = None
     cache.w_trace_too_long_hook = w_hook
     cache.in_recursion = NonConstant(False)
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     # this function is called from the JIT
     from rpython.jit.metainterp.resoperation import rop
-    
+
     l_w = []
     jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -86,18 +86,22 @@
 
         def interp_on_compile():
             di_loop.oplist = cls.oplist
-            pypy_hooks.after_compile(di_loop)
+            if pypy_hooks.are_hooks_enabled():
+                pypy_hooks.after_compile(di_loop)
 
         def interp_on_compile_bridge():
-            pypy_hooks.after_compile_bridge(di_bridge)
+            if pypy_hooks.are_hooks_enabled():
+                pypy_hooks.after_compile_bridge(di_bridge)
 
         def interp_on_optimize():
-            di_loop_optimize.oplist = cls.oplist
-            pypy_hooks.before_compile(di_loop_optimize)
+            if pypy_hooks.are_hooks_enabled():
+                di_loop_optimize.oplist = cls.oplist
+                pypy_hooks.before_compile(di_loop_optimize)
 
         def interp_on_abort():
-            pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver,
-                                greenkey, 'blah', Logger(MockSD), [])
+            if pypy_hooks.are_hooks_enabled():
+                pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver,
+                                    greenkey, 'blah', Logger(MockSD), [])
 
         space = cls.space
         cls.w_on_compile = space.wrap(interp2app(interp_on_compile))
diff --git a/rpython/jit/codewriter/policy.py b/rpython/jit/codewriter/policy.py
--- a/rpython/jit/codewriter/policy.py
+++ b/rpython/jit/codewriter/policy.py
@@ -11,9 +11,6 @@
         self.supports_floats = False
         self.supports_longlong = False
         self.supports_singlefloats = False
-        if jithookiface is None:
-            from rpython.rlib.jit import JitHookInterface
-            jithookiface = JitHookInterface()
         self.jithookiface = jithookiface
 
     def set_supports_floats(self, flag):
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -545,15 +545,17 @@
         show_procedures(metainterp_sd, loop)
         loop.check_consistency()
 
+    debug_info = None
+    hooks = None
     if metainterp_sd.warmrunnerdesc is not None:
         hooks = metainterp_sd.warmrunnerdesc.hooks
-        debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
-                                  original_jitcell_token, loop.operations,
-                                  type, greenkey)
-        hooks.before_compile(debug_info)
-    else:
-        debug_info = None
-        hooks = None
+        if hooks.are_hooks_enabled():
+            debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+                                      original_jitcell_token, loop.operations,
+                                      type, greenkey)
+            hooks.before_compile(debug_info)
+        else:
+            hooks = None
     operations = get_deep_immutable_oplist(loop.operations)
     metainterp_sd.profiler.start_backend()
     debug_start("jit-backend")
@@ -597,15 +599,17 @@
         show_procedures(metainterp_sd)
         seen = dict.fromkeys(inputargs)
         TreeLoop.check_consistency_of_branch(operations, seen)
+    debug_info = None
+    hooks = None
     if metainterp_sd.warmrunnerdesc is not None:
         hooks = metainterp_sd.warmrunnerdesc.hooks
-        debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
-                                  original_loop_token, operations, 'bridge',
-                                  fail_descr=faildescr)
-        hooks.before_compile_bridge(debug_info)
-    else:
-        hooks = None
-        debug_info = None
+        if hooks.are_hooks_enabled():
+            debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+                                      original_loop_token, operations, 'bridge',
+                                      fail_descr=faildescr)
+            hooks.before_compile_bridge(debug_info)
+        else:
+            hooks = None
     operations = get_deep_immutable_oplist(operations)
     metainterp_sd.profiler.start_backend()
     debug_start("jit-backend")
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2365,7 +2365,9 @@
             greenkey = None # we're in the bridge
         else:
             greenkey = self.current_merge_points[0][0][:jd_sd.num_green_args]
-            self.staticdata.warmrunnerdesc.hooks.on_abort(reason,
+            hooks = self.staticdata.warmrunnerdesc.hooks
+            if hooks.are_hooks_enabled():
+                hooks.on_abort(reason,
                     jd_sd.jitdriver, greenkey,
                     jd_sd.warmstate.get_location_str(greenkey),
                     self.staticdata.logger_ops._make_log_operations(
@@ -2374,9 +2376,10 @@
             if self.aborted_tracing_jitdriver is not None:
                 jd_sd = self.aborted_tracing_jitdriver
                 greenkey = self.aborted_tracing_greenkey
-                self.staticdata.warmrunnerdesc.hooks.on_trace_too_long(
-                    jd_sd.jitdriver, greenkey,
-                    jd_sd.warmstate.get_location_str(greenkey))
+                if hooks.are_hooks_enabled():
+                    hooks.on_trace_too_long(
+                        jd_sd.jitdriver, greenkey,
+                        jd_sd.warmstate.get_location_str(greenkey))
                 # no ops for now
                 self.aborted_tracing_jitdriver = None
                 self.aborted_tracing_greenkey = None
diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py
--- a/rpython/jit/metainterp/test/test_jitiface.py
+++ b/rpython/jit/metainterp/test/test_jitiface.py
@@ -238,7 +238,7 @@
 
         hashes = Hashes()
 
-        class Hooks(object):
+        class Hooks(JitHookInterface):
             def before_compile(self, debug_info):
                 pass
 
@@ -279,6 +279,44 @@
         self.meta_interp(main, [1, 1], policy=JitPolicy(hooks))
         assert len(hashes.t) == 1
 
+
+    def test_are_hooks_enabled(self):
+        reasons = []
+
+        class MyJitIface(JitHookInterface):
+            def are_hooks_enabled(self):
+                return False
+
+            def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, ops):
+                reasons.append(reason)
+
+        iface = MyJitIface()
+
+        myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'],
+                                get_printable_location=lambda *args: 'blah')
+
+        class Foo:
+            _immutable_fields_ = ['a?']
+
+            def __init__(self, a):
+                self.a = a
+
+        def f(a, x):
+            foo = Foo(a)
+            total = 0
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x, total=total)
+                total += foo.a
+                foo.a += 1
+                x -= 1
+            return total
+        #
+        assert f(100, 7) == 721
+        res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface))
+        assert res == 721
+        assert reasons == []
+
+
 class LLJitHookInterfaceTests(JitHookInterfaceTests):
     # use this for any backend, instead of the super class
     
@@ -320,7 +358,6 @@
         # this so far does not work because of the way setup_once is done,
         # but fine, it's only about untranslated version anyway
         #self.meta_interp(main, [False], ProfilerClass=Profiler)
-        
 
 class TestJitHookInterface(JitHookInterfaceTests, LLJitMixin):
     pass
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -220,6 +220,15 @@
     stats.check_consistency()
 
 # ____________________________________________________________
+# always disabled hooks interface
+
+from rpython.rlib.jit import JitHookInterface
+
+class NoHooksInterface(JitHookInterface):
+    def are_hooks_enabled(self):
+        return False
+
+# ____________________________________________________________
 
 class WarmRunnerDesc(object):
 
@@ -259,7 +268,7 @@
         else:
             self.jitcounter = counter.DeterministicJitCounter()
         #
-        self.hooks = policy.jithookiface
+        self.make_hooks(policy.jithookiface)
         self.make_virtualizable_infos()
         self.make_driverhook_graphs()
         self.make_enter_functions()
@@ -498,6 +507,12 @@
             self.metainterp_sd.opencoder_model = Model
         self.stats.metainterp_sd = self.metainterp_sd
 
+    def make_hooks(self, hooks):
+        if hooks is None:
+            # interface not overridden, use a special one that is never enabled
+            hooks = NoHooksInterface()
+        self.hooks = hooks
+
     def make_virtualizable_infos(self):
         vinfos = {}
         for jd in self.jitdrivers_sd:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,7 +1087,8 @@
     """ This is the main connector between the JIT and the interpreter.
     Several methods on this class will be invoked at various stages
     of JIT running like JIT loops compiled, aborts etc.
-    An instance of this class will be available as policy.jithookiface.
+    An instance of this class has to be passed into the JitPolicy constructor
+    (and will then be available as policy.jithookiface).
     """
     # WARNING: You should make a single prebuilt instance of a subclass
     # of this class.  You can, before translation, initialize some
@@ -1097,6 +1098,13 @@
     # of the program!  A line like ``pypy_hooks.foo = ...`` must not
     # appear inside your interpreter's RPython code.
 
+    def are_hooks_enabled(self):
+        """ A hook that is called to check whether the interpreter's hooks are
+        enabled at all. Only if this function returns True, are the other hooks
+        called. Otherwise, nothing happens. This is done because constructing
+        some of the hooks' arguments is expensive, so we'd rather not do it."""
+        return True
+
     def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations):
         """ A hook called each time a loop is aborted with jitdriver and
         greenkey where it started, reason is a string why it got aborted

From pypy.commits at gmail.com  Fri Mar 30 04:48:39 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 30 Mar 2018 01:48:39 -0700 (PDT)
Subject: [pypy-commit] pypy pyparser-improvements: adapt to new error message
Message-ID: <5abdf9e7.e18ddf0a.9e51.2f6e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: pyparser-improvements
Changeset: r94180:dd9a56219993
Date: 2018-03-30 10:29 +0200
http://bitbucket.org/pypy/pypy/changeset/dd9a56219993/

Log:	adapt to new error message

diff --git a/lib-python/2.7/test/test_genexps.py b/lib-python/2.7/test/test_genexps.py
--- a/lib-python/2.7/test/test_genexps.py
+++ b/lib-python/2.7/test/test_genexps.py
@@ -87,7 +87,7 @@
     >>> dict(a = i for i in xrange(10))
     Traceback (most recent call last):
        ...
-    SyntaxError: invalid syntax
+    SyntaxError: invalid syntax (expected ')')
 
 Verify that parenthesis are required when used as a keyword argument value
 

From pypy.commits at gmail.com  Fri Mar 30 05:29:00 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:00 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: start a branch in which to implement
 app-level hooks to signal major (and possibly minor?) collections
Message-ID: <5abe035c.93e51c0a.488f7.2f4d@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94181:ee3944e10c6c
Date: 2018-03-28 11:58 +0200
http://bitbucket.org/pypy/pypy/changeset/ee3944e10c6c/

Log:	start a branch in which to implement app-level hooks to signal major
	(and possibly minor?) collections


From pypy.commits at gmail.com  Fri Mar 30 05:29:02 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:02 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: add a hook interface to GC,
 which is implemented by a prebuilt object. Implement the hook for
 incminimark gc-minor
Message-ID: <5abe035e.01b9df0a.78523.d0b1@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94182:7b2f669acfa4
Date: 2018-03-28 23:14 +0200
http://bitbucket.org/pypy/pypy/changeset/7b2f669acfa4/

Log:	add a hook interface to GC, which is implemented by a prebuilt
	object. Implement the hook for incminimark gc-minor

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -5,6 +5,7 @@
 from rpython.memory.support import DEFAULT_CHUNK_SIZE
 from rpython.memory.support import get_address_stack, get_address_deque
 from rpython.memory.support import AddressDict, null_address_dict
+from rpython.memory.gc.hook import GcHooks
 from rpython.rtyper.lltypesystem.llmemory import NULL, raw_malloc_usage
 from rpython.rtyper.annlowlevel import cast_adr_to_nongc_instance
 
@@ -25,7 +26,7 @@
     _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
-                 translated_to_c=True):
+                 translated_to_c=True, hooks=None):
         self.gcheaderbuilder = GCHeaderBuilder(self.HDR)
         self.AddressStack = get_address_stack(chunk_size)
         self.AddressDeque = get_address_deque(chunk_size)
@@ -34,6 +35,9 @@
         self.config = config
         assert isinstance(translated_to_c, bool)
         self.translated_to_c = translated_to_c
+        if hooks is None:
+            hooks = GcHooks() # the default hooks are empty
+        self.hooks = hooks
 
     def setup(self):
         # all runtime mutable values' setup should happen here
diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/hook.py
@@ -0,0 +1,6 @@
+class GcHooks(object):
+
+    def on_gc_minor(self, total_memory_used, pinned_objects):
+        """
+        Called after a minor collection
+        """
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1828,8 +1828,8 @@
         # from the nursery that we just moved out.
         self.size_objects_made_old += r_uint(self.nursery_surviving_size)
         #
-        debug_print("minor collect, total memory used:",
-                    self.get_total_memory_used())
+        total_memory_used = self.get_total_memory_used()
+        debug_print("minor collect, total memory used:", total_memory_used)
         debug_print("number of pinned objects:",
                     self.pinned_objects_in_nursery)
         if self.DEBUG >= 2:
@@ -1838,6 +1838,8 @@
         self.root_walker.finished_minor_collection()
         #
         debug_stop("gc-minor")
+        self.hooks.on_gc_minor(total_memory_used=total_memory_used,
+                               pinned_objects=self.pinned_objects_in_nursery)
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
         ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
diff --git a/rpython/memory/gc/test/test_direct.py b/rpython/memory/gc/test/test_direct.py
--- a/rpython/memory/gc/test/test_direct.py
+++ b/rpython/memory/gc/test/test_direct.py
@@ -70,6 +70,9 @@
 class BaseDirectGCTest(object):
     GC_PARAMS = {}
 
+    def get_extra_gc_params(self):
+        return {}
+
     def setup_method(self, meth):
         from rpython.config.translationoption import get_combined_translation_config
         config = get_combined_translation_config(translating=True).translation
@@ -78,6 +81,7 @@
         if hasattr(meth, 'GC_PARAMS'):
             GC_PARAMS.update(meth.GC_PARAMS)
         GC_PARAMS['translated_to_c'] = False
+        GC_PARAMS.update(self.get_extra_gc_params())
         self.gc = self.GCClass(config, **GC_PARAMS)
         self.gc.DEBUG = True
         self.rootwalker = DirectRootWalker(self)
diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
new file mode 100644
--- /dev/null
+++ b/rpython/memory/gc/test/test_hook.py
@@ -0,0 +1,39 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.memory.gc.hook import GcHooks
+from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
+
+class MyGcHooks(GcHooks):
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.minors = []
+
+    def on_gc_minor(self, **kwds):
+        self.minors.append(kwds)
+
+
+class TestIncMiniMarkHooks(BaseDirectGCTest):
+    from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
+
+    def get_extra_gc_params(self):
+        return {'hooks': MyGcHooks()}
+
+    def test_on_gc_minor(self):
+        self.malloc(S)
+        self.gc._minor_collection()
+        assert self.gc.hooks.minors == [
+            {'total_memory_used': 0, 'pinned_objects': 0}
+            ]
+        self.gc.hooks.reset()
+        #
+        # these objects survive, so the total_memory_used is > 0
+        self.stackroots.append(self.malloc(S))
+        self.stackroots.append(self.malloc(S))
+        size = llmemory.sizeof(S) + self.gc.gcheaderbuilder.size_gc_header
+        rawsize = llmemory.raw_malloc_usage(size)
+        self.gc._minor_collection()
+        assert self.gc.hooks.minors == [
+            {'total_memory_used': rawsize*2, 'pinned_objects': 0}
+            ]

From pypy.commits at gmail.com  Fri Mar 30 05:29:04 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:04 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: add a hook for gc-collect-done
Message-ID: <5abe0360.55a5df0a.96199.17fb@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94183:5fbf8b32c70c
Date: 2018-03-29 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/5fbf8b32c70c/

Log:	add a hook for gc-collect-done

diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
--- a/rpython/memory/gc/hook.py
+++ b/rpython/memory/gc/hook.py
@@ -4,3 +4,10 @@
         """
         Called after a minor collection
         """
+
+    def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
+                      arenas_bytes, rawmalloc_bytes_before,
+                      rawmalloc_bytes_after):
+        """
+        Called after a major collection is fully done
+        """
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2422,6 +2422,13 @@
                             self.stat_rawmalloced_total_size, " => ",
                             self.rawmalloced_total_size)
                 debug_stop("gc-collect-done")
+                self.hooks.on_gc_collect(
+                    count=self.num_major_collects,
+                    arenas_count_before=self.stat_ac_arenas_count,
+                    arenas_count_after=self.ac.arenas_count,
+                    arenas_bytes=self.ac.total_memory_used,
+                    rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
+                    rawmalloc_bytes_after=self.rawmalloced_total_size)
                 #
                 # Set the threshold for the next major collection to be when we
                 # have allocated 'major_collection_threshold' times more than
diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -9,10 +9,14 @@
 
     def reset(self):
         self.minors = []
+        self.collects = []
 
     def on_gc_minor(self, **kwds):
         self.minors.append(kwds)
 
+    def on_gc_collect(self, **kwds):
+        self.collects.append(kwds)
+
 
 class TestIncMiniMarkHooks(BaseDirectGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
@@ -20,6 +24,11 @@
     def get_extra_gc_params(self):
         return {'hooks': MyGcHooks()}
 
+    def setup_method(self, m):
+        BaseDirectGCTest.setup_method(self, m)
+        size = llmemory.sizeof(S) + self.gc.gcheaderbuilder.size_gc_header
+        self.size_of_S = llmemory.raw_malloc_usage(size)
+
     def test_on_gc_minor(self):
         self.malloc(S)
         self.gc._minor_collection()
@@ -31,9 +40,31 @@
         # these objects survive, so the total_memory_used is > 0
         self.stackroots.append(self.malloc(S))
         self.stackroots.append(self.malloc(S))
-        size = llmemory.sizeof(S) + self.gc.gcheaderbuilder.size_gc_header
-        rawsize = llmemory.raw_malloc_usage(size)
         self.gc._minor_collection()
         assert self.gc.hooks.minors == [
-            {'total_memory_used': rawsize*2, 'pinned_objects': 0}
+            {'total_memory_used': self.size_of_S*2, 'pinned_objects': 0}
             ]
+
+    def test_on_gc_collect(self):
+        self.malloc(S)
+        self.gc.collect()
+        assert self.gc.hooks.collects == [
+            {'count': 1,
+             'arenas_count_before': 0,
+             'arenas_count_after': 0,
+             'arenas_bytes': 0,
+             'rawmalloc_bytes_after': 0,
+             'rawmalloc_bytes_before': 0}
+            ]
+        self.gc.hooks.reset()
+        #
+        self.stackroots.append(self.malloc(S))
+        self.gc.collect()
+        assert self.gc.hooks.collects == [
+            {'count': 2,
+             'arenas_count_before': 1,
+             'arenas_count_after': 1,
+             'arenas_bytes': self.size_of_S,
+             'rawmalloc_bytes_after': 0,
+             'rawmalloc_bytes_before': 0}
+            ]

From pypy.commits at gmail.com  Fri Mar 30 05:29:06 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:06 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: add a hook for gc-collect-step
Message-ID: <5abe0362.06031c0a.4360a.c588@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94184:f16482adff0b
Date: 2018-03-29 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/f16482adff0b/

Log:	add a hook for gc-collect-step

diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
--- a/rpython/memory/gc/hook.py
+++ b/rpython/memory/gc/hook.py
@@ -5,6 +5,16 @@
         Called after a minor collection
         """
 
+    def on_gc_collect_step(self, oldstate, newstate):
+        """
+        Called after each individual step of a major collection, in case the GC is
+        incremental.
+
+        ``oldstate`` and ``newstate`` are integers which indicate the GC
+        state; for incminimark, see incminimark.STATE_* and
+        incminimark.GC_STATES.
+        """
+
     def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
                       arenas_bytes, rawmalloc_bytes_before,
                       rawmalloc_bytes_after):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2243,6 +2243,7 @@
     # is done before every major collection step
     def major_collection_step(self, reserving_size=0):
         debug_start("gc-collect-step")
+        oldstate = self.gc_state
         debug_print("starting gc state: ", GC_STATES[self.gc_state])
         # Debugging checks
         if self.pinned_objects_in_nursery == 0:
@@ -2480,6 +2481,8 @@
 
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
+        self.hooks.on_gc_collect_step(oldstate=oldstate,
+                                      newstate=self.gc_state)
 
     def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):
         if self.header(obj).tid & GCFLAG_VISITED:
diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -2,6 +2,7 @@
 from rpython.memory.gc.hook import GcHooks
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
 
+
 class MyGcHooks(GcHooks):
 
     def __init__(self):
@@ -9,11 +10,15 @@
 
     def reset(self):
         self.minors = []
+        self.steps = []
         self.collects = []
 
     def on_gc_minor(self, **kwds):
         self.minors.append(kwds)
 
+    def on_gc_collect_step(self, **kwds):
+        self.steps.append(kwds)
+
     def on_gc_collect(self, **kwds):
         self.collects.append(kwds)
 
@@ -46,8 +51,15 @@
             ]
 
     def test_on_gc_collect(self):
+        from rpython.memory.gc import incminimark as m
         self.malloc(S)
         self.gc.collect()
+        assert self.gc.hooks.steps == [
+            {'oldstate': m.STATE_SCANNING, 'newstate': m.STATE_MARKING},
+            {'oldstate': m.STATE_MARKING, 'newstate': m.STATE_SWEEPING},
+            {'oldstate': m.STATE_SWEEPING, 'newstate': m.STATE_FINALIZING},
+            {'oldstate': m.STATE_FINALIZING, 'newstate': m.STATE_SCANNING}
+        ]
         assert self.gc.hooks.collects == [
             {'count': 1,
              'arenas_count_before': 0,

From pypy.commits at gmail.com  Fri Mar 30 05:29:08 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:08 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: WIP: hack around until we can pass a
 prebuilt instance of gchooks to the GC transformer, which will then forward
 it to the GCClass;
 start to test the behavior inside test_transformed_gc, but for now some
 tests fail because our testing hooks allocate too much, causing infinite
 recursion
Message-ID: <5abe0364.55a5df0a.96199.17fd@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94185:9bb5a3a3e292
Date: 2018-03-29 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/9bb5a3a3e292/

Log:	WIP: hack around until we can pass a prebuilt instance of gchooks to
	the GC transformer, which will then forward it to the GCClass; start
	to test the behavior inside test_transformed_gc, but for now some
	tests fail because our testing hooks allocate too much, causing
	infinite recursion

diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -1,8 +1,11 @@
+from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.memory.gc.hook import GcHooks
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
 
 
+# The following class is used also by test_transformed_gc and so it needs to
+# be RPython, that's why we have to use intmask to get consistent types
 class MyGcHooks(GcHooks):
 
     def __init__(self):
@@ -13,14 +16,26 @@
         self.steps = []
         self.collects = []
 
-    def on_gc_minor(self, **kwds):
-        self.minors.append(kwds)
+    def on_gc_minor(self, total_memory_used, pinned_objects):
+        self.minors.append({
+            'total_memory_used': intmask(total_memory_used),
+            'pinned_objects': pinned_objects})
 
-    def on_gc_collect_step(self, **kwds):
-        self.steps.append(kwds)
+    def on_gc_collect_step(self, oldstate, newstate):
+        self.steps.append({
+            'oldstate': oldstate,
+            'newstate': newstate})
 
-    def on_gc_collect(self, **kwds):
-        self.collects.append(kwds)
+    def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
+                      arenas_bytes, rawmalloc_bytes_before,
+                      rawmalloc_bytes_after):
+        self.collects.append({
+            'count': count,
+            'arenas_count_before': arenas_count_before,
+            'arenas_count_after': arenas_count_after,
+            'arenas_bytes': intmask(arenas_bytes),
+            'rawmalloc_bytes_before': intmask(rawmalloc_bytes_before),
+            'rawmalloc_bytes_after': intmask(rawmalloc_bytes_after)})
 
 
 class TestIncMiniMarkHooks(BaseDirectGCTest):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -116,7 +116,7 @@
 class BaseFrameworkGCTransformer(GCTransformer):
     root_stack_depth = None    # for tests to override
 
-    def __init__(self, translator):
+    def __init__(self, translator, gchooks=None):
         from rpython.memory.gc.base import choose_gc_from_config
 
         super(BaseFrameworkGCTransformer, self).__init__(translator,
@@ -162,7 +162,8 @@
         self.finalizer_queue_indexes = {}
         self.finalizer_handlers = []
 
-        gcdata.gc = GCClass(translator.config.translation, **GC_PARAMS)
+        gcdata.gc = GCClass(translator.config.translation, hooks=gchooks,
+                            **GC_PARAMS)
         root_walker = self.build_root_walker()
         root_walker.finished_minor_collection_func = finished_minor_collection
         self.root_walker = root_walker
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -15,6 +15,7 @@
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.rtyper.rtyper import llinterp_backend
+from rpython.memory.gc.test.test_hook import MyGcHooks
 
 
 WORD = LONG_BIT // 8
@@ -48,6 +49,7 @@
     gcpolicy = None
     GC_CAN_MOVE = False
     taggedpointers = False
+    gchooks = None
 
     def setup_class(cls):
         cls.marker = lltype.malloc(rffi.CArray(lltype.Signed), 1,
@@ -112,7 +114,8 @@
                 fixup(t)
 
         cbuild = CStandaloneBuilder(t, entrypoint, config=t.config,
-                                    gcpolicy=cls.gcpolicy)
+                                    gcpolicy=cls.gcpolicy,
+                                    gchooks=cls.gchooks)
         cbuild.make_entrypoint_wrapper = False
         db = cbuild.build_database()
         entrypointptr = cbuild.getentrypointptr()
@@ -1405,6 +1408,11 @@
                          }
             root_stack_depth = 200
 
+    gchooks = MyGcHooks()
+
+    def setup_method(self, m):
+        self.gchooks.reset()
+
     def define_malloc_array_of_gcptr(self):
         S = lltype.GcStruct('S', ('x', lltype.Signed))
         A = lltype.GcArray(lltype.Ptr(S))
diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py
--- a/rpython/translator/c/database.py
+++ b/rpython/translator/c/database.py
@@ -29,6 +29,7 @@
 
     def __init__(self, translator=None, standalone=False,
                  gcpolicyclass=None,
+                 gchooks=None,
                  exctransformer=None,
                  thread_enabled=False,
                  sandbox=False):
@@ -56,7 +57,7 @@
         self.namespace = CNameManager()
 
         if translator is not None:
-            self.gctransformer = self.gcpolicy.gettransformer(translator)
+            self.gctransformer = self.gcpolicy.gettransformer(translator, gchooks)
         self.completed = False
 
         self.instrument_ncounter = 0
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -302,9 +302,9 @@
 
 class BasicFrameworkGcPolicy(BasicGcPolicy):
 
-    def gettransformer(self, translator):
+    def gettransformer(self, translator, gchooks):
         if hasattr(self, 'transformerclass'):    # for rpython/memory tests
-            return self.transformerclass(translator)
+            return self.transformerclass(translator, gchooks=gchooks)
         raise NotImplementedError
 
     def struct_setup(self, structdefnode, rtti):
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -64,13 +64,14 @@
     split = False
 
     def __init__(self, translator, entrypoint, config, gcpolicy=None,
-            secondary_entrypoints=()):
+                 gchooks=None, secondary_entrypoints=()):
         self.translator = translator
         self.entrypoint = entrypoint
         self.entrypoint_name = getattr(self.entrypoint, 'func_name', None)
         self.originalentrypoint = entrypoint
         self.config = config
         self.gcpolicy = gcpolicy    # for tests only, e.g. rpython/memory/
+        self.gchooks = gchooks
         self.eci = self.get_eci()
         self.secondary_entrypoints = secondary_entrypoints
 
@@ -91,6 +92,7 @@
         exctransformer = translator.getexceptiontransformer()
         db = LowLevelDatabase(translator, standalone=self.standalone,
                               gcpolicyclass=gcpolicyclass,
+                              gchooks=self.gchooks,
                               exctransformer=exctransformer,
                               thread_enabled=self.config.translation.thread,
                               sandbox=self.config.translation.sandbox)

From pypy.commits at gmail.com  Fri Mar 30 05:29:10 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:10 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: make it possible to disable the gc
 hooks
Message-ID: <5abe0366.77a9df0a.5b00d.ec30@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94186:e6a1a204ce7a
Date: 2018-03-29 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/e6a1a204ce7a/

Log:	make it possible to disable the gc hooks

diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
--- a/rpython/memory/gc/hook.py
+++ b/rpython/memory/gc/hook.py
@@ -1,5 +1,10 @@
 class GcHooks(object):
 
+    def __init__(self):
+        self.gc_minor_enabled = False
+        self.gc_collect_step_enabled = False
+        self.gc_collect_enabled = False
+
     def on_gc_minor(self, total_memory_used, pinned_objects):
         """
         Called after a minor collection
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1838,8 +1838,9 @@
         self.root_walker.finished_minor_collection()
         #
         debug_stop("gc-minor")
-        self.hooks.on_gc_minor(total_memory_used=total_memory_used,
-                               pinned_objects=self.pinned_objects_in_nursery)
+        if self.hooks.gc_minor_enabled:
+            self.hooks.on_gc_minor(total_memory_used=total_memory_used,
+                                   pinned_objects=self.pinned_objects_in_nursery)
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
         ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
@@ -2423,13 +2424,14 @@
                             self.stat_rawmalloced_total_size, " => ",
                             self.rawmalloced_total_size)
                 debug_stop("gc-collect-done")
-                self.hooks.on_gc_collect(
-                    count=self.num_major_collects,
-                    arenas_count_before=self.stat_ac_arenas_count,
-                    arenas_count_after=self.ac.arenas_count,
-                    arenas_bytes=self.ac.total_memory_used,
-                    rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
-                    rawmalloc_bytes_after=self.rawmalloced_total_size)
+                if self.hooks.gc_collect_enabled:
+                    self.hooks.on_gc_collect(
+                        count=self.num_major_collects,
+                        arenas_count_before=self.stat_ac_arenas_count,
+                        arenas_count_after=self.ac.arenas_count,
+                        arenas_bytes=self.ac.total_memory_used,
+                        rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
+                        rawmalloc_bytes_after=self.rawmalloced_total_size)
                 #
                 # Set the threshold for the next major collection to be when we
                 # have allocated 'major_collection_threshold' times more than
@@ -2481,8 +2483,9 @@
 
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
-        self.hooks.on_gc_collect_step(oldstate=oldstate,
-                                      newstate=self.gc_state)
+        if self.hooks.gc_collect_step_enabled:
+            self.hooks.on_gc_collect_step(oldstate=oldstate,
+                                          newstate=self.gc_state)
 
     def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):
         if self.header(obj).tid & GCFLAG_VISITED:
diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -9,6 +9,7 @@
 class MyGcHooks(GcHooks):
 
     def __init__(self):
+        GcHooks.__init__(self)
         self.reset()
 
     def reset(self):
@@ -50,6 +51,7 @@
         self.size_of_S = llmemory.raw_malloc_usage(size)
 
     def test_on_gc_minor(self):
+        self.gc.hooks.gc_minor_enabled = True
         self.malloc(S)
         self.gc._minor_collection()
         assert self.gc.hooks.minors == [
@@ -67,6 +69,8 @@
 
     def test_on_gc_collect(self):
         from rpython.memory.gc import incminimark as m
+        self.gc.hooks.gc_collect_step_enabled = True
+        self.gc.hooks.gc_collect_enabled = True
         self.malloc(S)
         self.gc.collect()
         assert self.gc.hooks.steps == [
@@ -95,3 +99,10 @@
              'rawmalloc_bytes_after': 0,
              'rawmalloc_bytes_before': 0}
             ]
+
+    def test_hook_disabled(self):
+        self.gc._minor_collection()
+        self.gc.collect()
+        assert self.gc.hooks.minors == []
+        assert self.gc.hooks.steps == []
+        assert self.gc.hooks.collects == []

From pypy.commits at gmail.com  Fri Mar 30 05:29:12 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 02:29:12 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: add a translated test for gc hooks
Message-ID: <5abe0368.45931c0a.6efe.5a9c@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94187:5cb980c10b8c
Date: 2018-03-30 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/5cb980c10b8c/

Log:	add a translated test for gc hooks

diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -1,11 +1,8 @@
-from rpython.rlib.rarithmetic import intmask
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.memory.gc.hook import GcHooks
 from rpython.memory.gc.test.test_direct import BaseDirectGCTest, S
 
 
-# The following class is used also by test_transformed_gc and so it needs to
-# be RPython, that's why we have to use intmask to get consistent types
 class MyGcHooks(GcHooks):
 
     def __init__(self):
@@ -19,7 +16,7 @@
 
     def on_gc_minor(self, total_memory_used, pinned_objects):
         self.minors.append({
-            'total_memory_used': intmask(total_memory_used),
+            'total_memory_used': total_memory_used,
             'pinned_objects': pinned_objects})
 
     def on_gc_collect_step(self, oldstate, newstate):
@@ -34,9 +31,9 @@
             'count': count,
             'arenas_count_before': arenas_count_before,
             'arenas_count_after': arenas_count_after,
-            'arenas_bytes': intmask(arenas_bytes),
-            'rawmalloc_bytes_before': intmask(rawmalloc_bytes_before),
-            'rawmalloc_bytes_after': intmask(rawmalloc_bytes_after)})
+            'arenas_bytes': arenas_bytes,
+            'rawmalloc_bytes_before': rawmalloc_bytes_before,
+            'rawmalloc_bytes_after': rawmalloc_bytes_after})
 
 
 class TestIncMiniMarkHooks(BaseDirectGCTest):
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -14,8 +14,9 @@
 from rpython.conftest import option
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import LONG_BIT
+from rpython.rlib.nonconst import NonConstant
 from rpython.rtyper.rtyper import llinterp_backend
-from rpython.memory.gc.test.test_hook import MyGcHooks
+from rpython.memory.gc.hook import GcHooks
 
 
 WORD = LONG_BIT // 8
@@ -1391,6 +1392,32 @@
         assert res([]) == 0
 
 
+class GcHooksStats(object):
+    minors = 0
+    steps = 0
+    collects = 0
+
+GC_HOOKS_STATS = GcHooksStats()
+
+class MyGcHooks(GcHooks):
+
+    def __init__(self):
+        self.gc_minor_enabled = True
+        self.gc_collect_step_enabled = True
+        self.gc_collect_enabled = True
+
+    def on_gc_minor(self, total_memory_used, pinned_objects):
+        GC_HOOKS_STATS.minors += 1
+
+    def on_gc_collect_step(self, oldstate, newstate):
+        GC_HOOKS_STATS.steps += 1
+        
+    def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
+                      arenas_bytes, rawmalloc_bytes_before,
+                      rawmalloc_bytes_after):
+        GC_HOOKS_STATS.collects += 1
+
+
 class TestIncrementalMiniMarkGC(TestMiniMarkGC):
     gcname = "incminimark"
 
@@ -1410,9 +1437,6 @@
 
     gchooks = MyGcHooks()
 
-    def setup_method(self, m):
-        self.gchooks.reset()
-
     def define_malloc_array_of_gcptr(self):
         S = lltype.GcStruct('S', ('x', lltype.Signed))
         A = lltype.GcArray(lltype.Ptr(S))
@@ -1446,6 +1470,38 @@
         res = run([])
         assert res
 
+    def define_gc_hooks(cls):
+        gchooks = cls.gchooks
+        def f():
+            if NonConstant(False):
+                # this is needed to "fix" the annotation of GcHooksStats
+                # early; else, we change the annotation during the GC
+                # transform, when it's too late
+                GC_HOOKS_STATS.collects += 42
+                GC_HOOKS_STATS.steps += 42
+                GC_HOOKS_STATS.minors += 42
+
+            # trigger two major collections
+            llop.gc__collect(lltype.Void)
+            llop.gc__collect(lltype.Void)
+            return (10000 * GC_HOOKS_STATS.collects +
+                      100 * GC_HOOKS_STATS.steps +
+                        1 * GC_HOOKS_STATS.minors)
+        return f
+
+    def test_gc_hooks(self):
+        run = self.runner("gc_hooks")
+        count = run([])
+        collects, count = divmod(count, 10000)
+        steps, minors = divmod(count, 100)
+        #
+        # note: the following asserts are slightly fragile, as they assume
+        # that we do NOT run any minor collection apart the ones triggered by
+        # major_collection_step
+        assert collects == 2           # 2 collections, manually triggered
+        assert steps == 4 * collects   # 4 steps for each major collection
+        assert minors == steps         # one minor collection for each step
+
 # ________________________________________________________________
 # tagged pointers
 

From pypy.commits at gmail.com  Fri Mar 30 09:00:42 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 06:00:42 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: call the hooks through helpers which
 are marked as @rgc.no_collect,
 to ensure that we cannot allocate anything from within
Message-ID: <5abe34fa.c8c51c0a.db96c.25f3@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94188:3d2b6d04026f
Date: 2018-03-30 14:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3d2b6d04026f/

Log:	call the hooks through helpers which are marked as @rgc.no_collect,
	to ensure that we cannot allocate anything from within

diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
--- a/rpython/memory/gc/hook.py
+++ b/rpython/memory/gc/hook.py
@@ -1,10 +1,21 @@
+from rpython.rlib import rgc
+
 class GcHooks(object):
+    """
+    Base class to write your own GC hooks.
+
+    Subclasses are expected to override the on_* methods. Note that such
+    methods can do only simple stuff such as updating statistics and/or
+    setting a flag: in particular, they cannot do anything which can possibly
+    trigger a GC collection.
+    """
 
     def __init__(self):
         self.gc_minor_enabled = False
         self.gc_collect_step_enabled = False
         self.gc_collect_enabled = False
 
+
     def on_gc_minor(self, total_memory_used, pinned_objects):
         """
         Called after a minor collection
@@ -20,9 +31,29 @@
         incminimark.GC_STATES.
         """
 
+
     def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
                       arenas_bytes, rawmalloc_bytes_before,
                       rawmalloc_bytes_after):
         """
         Called after a major collection is fully done
         """
+
+    # the fire_* methods are meant to be called from the GC are should NOT be
+    # overridden
+
+    @rgc.no_collect
+    def fire_gc_minor(self, total_memory_used, pinned_objects):
+        self.on_gc_minor(total_memory_used, pinned_objects)
+
+    @rgc.no_collect
+    def fire_gc_collect_step(self, oldstate, newstate):
+        self.on_gc_collect_step(oldstate, newstate)
+
+    @rgc.no_collect
+    def fire_gc_collect(self, count, arenas_count_before, arenas_count_after,
+                        arenas_bytes, rawmalloc_bytes_before,
+                        rawmalloc_bytes_after):
+        self.on_gc_collect(count, arenas_count_before, arenas_count_after,
+                           arenas_bytes, rawmalloc_bytes_before,
+                           rawmalloc_bytes_after)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1839,7 +1839,7 @@
         #
         debug_stop("gc-minor")
         if self.hooks.gc_minor_enabled:
-            self.hooks.on_gc_minor(total_memory_used=total_memory_used,
+            self.hooks.fire_gc_minor(total_memory_used=total_memory_used,
                                    pinned_objects=self.pinned_objects_in_nursery)
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
@@ -2425,7 +2425,7 @@
                             self.rawmalloced_total_size)
                 debug_stop("gc-collect-done")
                 if self.hooks.gc_collect_enabled:
-                    self.hooks.on_gc_collect(
+                    self.hooks.fire_gc_collect(
                         count=self.num_major_collects,
                         arenas_count_before=self.stat_ac_arenas_count,
                         arenas_count_after=self.ac.arenas_count,
@@ -2484,7 +2484,7 @@
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
         if self.hooks.gc_collect_step_enabled:
-            self.hooks.on_gc_collect_step(oldstate=oldstate,
+            self.hooks.fire_gc_collect_step(oldstate=oldstate,
                                           newstate=self.gc_state)
 
     def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):

From pypy.commits at gmail.com  Fri Mar 30 09:00:44 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 06:00:44 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: make it possible to define gchooks in
 the target,
 and use the new interface to print some stats inside targetgcbench. I ran
 benchmarks on targetgcbench,
 and it shows no significative difference between having or not having the
 hooks
Message-ID: <5abe34fc.e380df0a.59961.c583@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94189:696e7ef11214
Date: 2018-03-30 14:59 +0200
http://bitbucket.org/pypy/pypy/changeset/696e7ef11214/

Log:	make it possible to define gchooks in the target, and use the new
	interface to print some stats inside targetgcbench. I ran benchmarks
	on targetgcbench, and it shows no significative difference between
	having or not having the hooks

diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1397,6 +1397,16 @@
     steps = 0
     collects = 0
 
+    @staticmethod
+    def fix_annotation():
+        # this is needed to "fix" the annotation of GcHooksStats early, and
+        # must be called from the "main" program. Else, we change the
+        # annotation during the GC transform, when it's too late
+        if NonConstant(False):
+            GC_HOOKS_STATS.collects += 42
+            GC_HOOKS_STATS.steps += 42
+            GC_HOOKS_STATS.minors += 42
+
 GC_HOOKS_STATS = GcHooksStats()
 
 class MyGcHooks(GcHooks):
@@ -1473,14 +1483,7 @@
     def define_gc_hooks(cls):
         gchooks = cls.gchooks
         def f():
-            if NonConstant(False):
-                # this is needed to "fix" the annotation of GcHooksStats
-                # early; else, we change the annotation during the GC
-                # transform, when it's too late
-                GC_HOOKS_STATS.collects += 42
-                GC_HOOKS_STATS.steps += 42
-                GC_HOOKS_STATS.minors += 42
-
+            GC_HOOKS_STATS.fix_annotation()
             # trigger two major collections
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py
--- a/rpython/translator/c/gc.py
+++ b/rpython/translator/c/gc.py
@@ -94,7 +94,7 @@
 
 class RefcountingGcPolicy(BasicGcPolicy):
 
-    def gettransformer(self, translator):
+    def gettransformer(self, translator, gchooks):
         from rpython.memory.gctransform import refcounting
         return refcounting.RefcountingGCTransformer(translator)
 
@@ -175,7 +175,7 @@
 
 class BoehmGcPolicy(BasicGcPolicy):
 
-    def gettransformer(self, translator):
+    def gettransformer(self, translator, gchooks):
         from rpython.memory.gctransform import boehm
         return boehm.BoehmGCTransformer(translator)
 
@@ -439,9 +439,9 @@
 
 class ShadowStackFrameworkGcPolicy(BasicFrameworkGcPolicy):
 
-    def gettransformer(self, translator):
+    def gettransformer(self, translator, gchooks):
         from rpython.memory.gctransform import shadowstack
-        return shadowstack.ShadowStackFrameworkGCTransformer(translator)
+        return shadowstack.ShadowStackFrameworkGCTransformer(translator, gchooks)
 
     def enter_roots_frame(self, funcgen, (c_gcdata, c_numcolors)):
         numcolors = c_numcolors.value
@@ -484,9 +484,9 @@
 
 class AsmGcRootFrameworkGcPolicy(BasicFrameworkGcPolicy):
 
-    def gettransformer(self, translator):
+    def gettransformer(self, translator, gchooks):
         from rpython.memory.gctransform import asmgcroot
-        return asmgcroot.AsmGcRootFrameworkGCTransformer(translator)
+        return asmgcroot.AsmGcRootFrameworkGCTransformer(translator, gchooks)
 
     def GC_KEEPALIVE(self, funcgen, v):
         return 'pypy_asm_keepalive(%s);' % funcgen.expr(v)
diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py
--- a/rpython/translator/driver.py
+++ b/rpython/translator/driver.py
@@ -413,11 +413,12 @@
             translator.frozen = True
 
         standalone = self.standalone
+        gchooks = self.extra.get('gchooks', None)
 
         if standalone:
             from rpython.translator.c.genc import CStandaloneBuilder
             cbuilder = CStandaloneBuilder(self.translator, self.entry_point,
-                                          config=self.config,
+                                          config=self.config, gchooks=gchooks,
                       secondary_entrypoints=
                       self.secondary_entrypoints + annotated_jit_entrypoints)
         else:
@@ -426,7 +427,8 @@
             cbuilder = CLibraryBuilder(self.translator, self.entry_point,
                                        functions=functions,
                                        name='libtesting',
-                                       config=self.config)
+                                       config=self.config,
+                                       gchooks=gchooks)
         if not standalone:     # xxx more messy
             cbuilder.modulename = self.extmod_name
         database = cbuilder.build_database()
diff --git a/rpython/translator/goal/targetgcbench.py b/rpython/translator/goal/targetgcbench.py
--- a/rpython/translator/goal/targetgcbench.py
+++ b/rpython/translator/goal/targetgcbench.py
@@ -1,16 +1,23 @@
 from rpython.translator.goal import gcbench
+from rpython.memory.test.test_transformed_gc import MyGcHooks, GC_HOOKS_STATS
 
 # _____ Define and setup target ___
 
+def entry_point(argv):
+    GC_HOOKS_STATS.fix_annotation()
+    ret = gcbench.entry_point(argv)
+    minors = GC_HOOKS_STATS.minors
+    steps = GC_HOOKS_STATS.steps
+    collects = GC_HOOKS_STATS.collects
+    print 'GC hooks statistics'
+    print '    gc-minor:        ', minors
+    print '    gc-collect-step: ', steps
+    print '    gc-collect:      ', collects
+    return ret
+
+gchooks = MyGcHooks()
+
 def target(*args):
     gcbench.ENABLE_THREADS = False    # not RPython
-    return gcbench.entry_point, None
+    return entry_point, None
 
-"""
-Why is this a stand-alone target?
-
-The above target specifies None as the argument types list.
-This is a case treated specially in the driver.py . If the list
-of input types is empty, it is meant to be a list of strings,
-actually implementing argv of the executable.
-"""

From pypy.commits at gmail.com  Fri Mar 30 13:18:50 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 10:18:50 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: reset the counters at the beginning of
 this test, else we count also the colletions occurred in the previous tests
Message-ID: <5abe717a.e18ddf0a.9e51.7375@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94190:2492c826dee0
Date: 2018-03-30 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/2492c826dee0/

Log:	reset the counters at the beginning of this test, else we count also
	the colletions occurred in the previous tests

diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1397,6 +1397,11 @@
     steps = 0
     collects = 0
 
+    def reset(self):
+        self.minors = 0
+        self.steps = 0
+        self.collects = 0
+
     @staticmethod
     def fix_annotation():
         # this is needed to "fix" the annotation of GcHooksStats early, and
@@ -1484,6 +1489,7 @@
         gchooks = cls.gchooks
         def f():
             GC_HOOKS_STATS.fix_annotation()
+            GC_HOOKS_STATS.reset()
             # trigger two major collections
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)

From pypy.commits at gmail.com  Fri Mar 30 13:18:52 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 10:18:52 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: hg merge default
Message-ID: <5abe717c.89af1c0a.e65fd.9d91@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94191:9edf064fc152
Date: 2018-03-30 18:31 +0200
http://bitbucket.org/pypy/pypy/changeset/9edf064fc152/

Log:	hg merge default

diff too long, truncating to 2000 out of 2274 lines

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -68,3 +68,14 @@
 Optimize `Py*_Check` for `Bool`, `Float`, `Set`. Also refactor and simplify
 `W_PyCWrapperObject` which is used to call slots from the C-API, greatly
 improving microbenchmarks in https://github.com/antocuni/cpyext-benchmarks
+
+
+.. branch: fix-sre-problems
+
+Fix two (unrelated) JIT bugs manifesting in the re module:
+
+- green fields are broken and were thus disabled, plus their usage removed from
+  the _sre implementation
+
+- in rare "trace is too long" situations, the JIT could break behaviour
+  arbitrarily.
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -232,7 +232,9 @@
                 "different from the 'ffi.h' file seen at compile-time)")
 
     def py_invoke(self, ll_res, ll_args):
+        key_pycode = self.key_pycode
         jitdriver1.jit_merge_point(callback=self,
+                                   key_pycode=key_pycode,
                                    ll_res=ll_res,
                                    ll_args=ll_args)
         self.do_invoke(ll_res, ll_args)
@@ -294,7 +296,7 @@
     return 'cffi_callback ' + key_pycode.get_repr()
 
 jitdriver1 = jit.JitDriver(name='cffi_callback',
-                           greens=['callback.key_pycode'],
+                           greens=['key_pycode'],
                            reds=['ll_res', 'll_args', 'callback'],
                            get_printable_location=get_printable_location1)
 
diff --git a/pypy/module/_io/test/test_interp_textio.py b/pypy/module/_io/test/test_interp_textio.py
--- a/pypy/module/_io/test/test_interp_textio.py
+++ b/pypy/module/_io/test/test_interp_textio.py
@@ -7,6 +7,11 @@
 from pypy.module._io.interp_bytesio import W_BytesIO
 from pypy.module._io.interp_textio import W_TextIOWrapper, DecodeBuffer
 
+# workaround suggestion for slowness by David McIver:
+# force hypothesis to initialize some lazy stuff
+# (which takes a lot of time, which trips the timer otherwise)
+st.text().example()
+
 def translate_newlines(text):
     text = text.replace(u'\r\n', u'\n')
     text = text.replace(u'\r', u'\n')
@@ -29,7 +34,7 @@
 
 @given(data=st_readline(),
        mode=st.sampled_from(['\r', '\n', '\r\n', '']))
- at settings(deadline=None)
+ at settings(deadline=None, database=None)
 def test_readline(space, data, mode):
     txt, limits = data
     w_stream = W_BytesIO(space)
diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -77,15 +77,15 @@
     w_import = space.getattr(w_builtin, space.newtext("__import__"))
     return space.call_function(w_import, space.newtext("re"))
 
-def matchcontext(space, ctx):
+def matchcontext(space, ctx, pattern):
     try:
-        return rsre_core.match_context(ctx)
+        return rsre_core.match_context(ctx, pattern)
     except rsre_core.Error as e:
         raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
-def searchcontext(space, ctx):
+def searchcontext(space, ctx, pattern):
     try:
-        return rsre_core.search_context(ctx)
+        return rsre_core.search_context(ctx, pattern)
     except rsre_core.Error as e:
         raise OperationError(space.w_RuntimeError, space.newtext(e.msg))
 
@@ -114,7 +114,7 @@
                 pos = len(unicodestr)
             if endpos > len(unicodestr):
                 endpos = len(unicodestr)
-            return rsre_core.UnicodeMatchContext(self.code, unicodestr,
+            return rsre_core.UnicodeMatchContext(unicodestr,
                                                  pos, endpos, self.flags)
         elif space.isinstance_w(w_string, space.w_bytes):
             str = space.bytes_w(w_string)
@@ -122,7 +122,7 @@
                 pos = len(str)
             if endpos > len(str):
                 endpos = len(str)
-            return rsre_core.StrMatchContext(self.code, str,
+            return rsre_core.StrMatchContext(str,
                                              pos, endpos, self.flags)
         else:
             buf = space.readbuf_w(w_string)
@@ -132,7 +132,7 @@
                 pos = size
             if endpos > size:
                 endpos = size
-            return rsre_core.BufMatchContext(self.code, buf,
+            return rsre_core.BufMatchContext(buf,
                                              pos, endpos, self.flags)
 
     def getmatch(self, ctx, found):
@@ -144,12 +144,12 @@
     @unwrap_spec(pos=int, endpos=int)
     def match_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
-        return self.getmatch(ctx, matchcontext(self.space, ctx))
+        return self.getmatch(ctx, matchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def search_w(self, w_string, pos=0, endpos=sys.maxint):
         ctx = self.make_ctx(w_string, pos, endpos)
-        return self.getmatch(ctx, searchcontext(self.space, ctx))
+        return self.getmatch(ctx, searchcontext(self.space, ctx, self.code))
 
     @unwrap_spec(pos=int, endpos=int)
     def findall_w(self, w_string, pos=0, endpos=sys.maxint):
@@ -157,7 +157,7 @@
         matchlist_w = []
         ctx = self.make_ctx(w_string, pos, endpos)
         while ctx.match_start <= ctx.end:
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, self.code):
                 break
             num_groups = self.num_groups
             w_emptystr = space.newtext("")
@@ -182,7 +182,7 @@
         # this also works as the implementation of the undocumented
         # scanner() method.
         ctx = self.make_ctx(w_string, pos, endpos)
-        scanner = W_SRE_Scanner(self, ctx)
+        scanner = W_SRE_Scanner(self, ctx, self.code)
         return scanner
 
     @unwrap_spec(maxsplit=int)
@@ -193,7 +193,7 @@
         last = 0
         ctx = self.make_ctx(w_string)
         while not maxsplit or n < maxsplit:
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, self.code):
                 break
             if ctx.match_start == ctx.match_end:     # zero-width match
                 if ctx.match_start == ctx.end:       # or end of string
@@ -274,8 +274,8 @@
         else:
             sublist_w = []
         n = last_pos = 0
+        pattern = self.code
         while not count or n < count:
-            pattern = ctx.pattern
             sub_jitdriver.jit_merge_point(
                 self=self,
                 use_builder=use_builder,
@@ -292,7 +292,7 @@
                 n=n, last_pos=last_pos, sublist_w=sublist_w
                 )
             space = self.space
-            if not searchcontext(space, ctx):
+            if not searchcontext(space, ctx, pattern):
                 break
             if last_pos < ctx.match_start:
                 _sub_append_slice(
@@ -388,7 +388,11 @@
     srepat.space = space
     srepat.w_pattern = w_pattern      # the original uncompiled pattern
     srepat.flags = flags
-    srepat.code = code
+    # note: we assume that the app-level is caching SRE_Pattern objects,
+    # so that we don't need to do it here.  Creating new SRE_Pattern
+    # objects all the time would be bad for the JIT, which relies on the
+    # identity of the CompiledPattern() object.
+    srepat.code = rsre_core.CompiledPattern(code)
     srepat.num_groups = groups
     srepat.w_groupindex = w_groupindex
     srepat.w_indexgroup = w_indexgroup
@@ -611,10 +615,11 @@
 # Our version is also directly iterable, to make finditer() easier.
 
 class W_SRE_Scanner(W_Root):
-    def __init__(self, pattern, ctx):
+    def __init__(self, pattern, ctx, code):
         self.space = pattern.space
         self.srepat = pattern
         self.ctx = ctx
+        self.code = code
         # 'self.ctx' is always a fresh context in which no searching
         # or matching succeeded so far.
 
@@ -624,19 +629,19 @@
     def next_w(self):
         if self.ctx.match_start > self.ctx.end:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        if not searchcontext(self.space, self.ctx):
+        if not searchcontext(self.space, self.ctx, self.code):
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
         return self.getmatch(True)
 
     def match_w(self):
         if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
-        return self.getmatch(matchcontext(self.space, self.ctx))
+        return self.getmatch(matchcontext(self.space, self.ctx, self.code))
 
     def search_w(self):
         if self.ctx.match_start > self.ctx.end:
             return self.space.w_None
-        return self.getmatch(searchcontext(self.space, self.ctx))
+        return self.getmatch(searchcontext(self.space, self.ctx, self.code))
 
     def getmatch(self, found):
         if found:
diff --git a/pypy/module/pypyjit/hooks.py b/pypy/module/pypyjit/hooks.py
--- a/pypy/module/pypyjit/hooks.py
+++ b/pypy/module/pypyjit/hooks.py
@@ -7,12 +7,20 @@
     WrappedOp, W_JitLoopInfo, wrap_oplist)
 
 class PyPyJitIface(JitHookInterface):
+    def are_hooks_enabled(self):
+        space = self.space
+        cache = space.fromcache(Cache)
+        return (cache.w_compile_hook is not None or
+                cache.w_abort_hook is not None or
+                cache.w_trace_too_long_hook is not None)
+
+
     def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations):
         space = self.space
         cache = space.fromcache(Cache)
         if cache.in_recursion:
             return
-        if space.is_true(cache.w_abort_hook):
+        if cache.w_abort_hook is not None:
             cache.in_recursion = True
             oplist_w = wrap_oplist(space, logops, operations)
             try:
@@ -33,7 +41,7 @@
         cache = space.fromcache(Cache)
         if cache.in_recursion:
             return
-        if space.is_true(cache.w_trace_too_long_hook):
+        if cache.w_trace_too_long_hook is not None:
             cache.in_recursion = True
             try:
                 try:
@@ -62,7 +70,7 @@
         cache = space.fromcache(Cache)
         if cache.in_recursion:
             return
-        if space.is_true(cache.w_compile_hook):
+        if cache.w_compile_hook is not None:
             w_debug_info = W_JitLoopInfo(space, debug_info, is_bridge,
                                          cache.compile_hook_with_ops)
             cache.in_recursion = True
diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -21,9 +21,10 @@
     no = 0
 
     def __init__(self, space):
-        self.w_compile_hook = space.w_None
-        self.w_abort_hook = space.w_None
-        self.w_trace_too_long_hook = space.w_None
+        self.w_compile_hook = None
+        self.w_abort_hook = None
+        self.w_trace_too_long_hook = None
+        self.compile_hook_with_ops = False
 
     def getno(self):
         self.no += 1
@@ -58,7 +59,8 @@
     jit hook won't be called for that.
     """
     cache = space.fromcache(Cache)
-    assert w_hook is not None
+    if space.is_w(w_hook, space.w_None):
+        w_hook = None
     cache.w_compile_hook = w_hook
     cache.compile_hook_with_ops = operations
     cache.in_recursion = NonConstant(False)
@@ -77,7 +79,8 @@
     as attributes on JitLoopInfo object.
     """
     cache = space.fromcache(Cache)
-    assert w_hook is not None
+    if space.is_w(w_hook, space.w_None):
+        w_hook = None
     cache.w_abort_hook = w_hook
     cache.in_recursion = NonConstant(False)
 
@@ -92,14 +95,15 @@
         hook(jitdriver_name, greenkey)
     """
     cache = space.fromcache(Cache)
-    assert w_hook is not None
+    if space.is_w(w_hook, space.w_None):
+        w_hook = None
     cache.w_trace_too_long_hook = w_hook
     cache.in_recursion = NonConstant(False)
 
 def wrap_oplist(space, logops, operations, ops_offset=None):
     # this function is called from the JIT
     from rpython.jit.metainterp.resoperation import rop
-    
+
     l_w = []
     jitdrivers_sd = logops.metainterp_sd.jitdrivers_sd
     for op in operations:
@@ -109,22 +113,27 @@
             ofs = ops_offset.get(op, 0)
         num = op.getopnum()
         name = op.getopname()
+        repr = logops.repr_of_resop(op)
         if num == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
             greenkey = op.getarglist()[3:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, name,
-                                       logops.repr_of_resop(op),
+                                       repr,
                                        jd_sd.jitdriver.name,
                                        op.getarg(1).getint(),
                                        op.getarg(2).getint(),
                                        w_greenkey))
         elif op.is_guard():
-            l_w.append(GuardOp(name, ofs, logops.repr_of_resop(op),
-                op.getdescr().get_jitcounter_hash()))
+            descr = op.getdescr()
+            if descr is not None: # can be none in on_abort!
+                hash = op.getdescr().get_jitcounter_hash()
+            else:
+                hash = -1
+            l_w.append(GuardOp(name, ofs, repr, hash))
         else:
-            l_w.append(WrappedOp(name, ofs, logops.repr_of_resop(op)))
+            l_w.append(WrappedOp(name, ofs, repr))
     return l_w
 
 @unwrap_spec(offset=int, repr='text', name='text')
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -65,6 +65,17 @@
             if i != 1:
                 offset[op] = i
 
+        oplist_no_descrs = parse("""
+        [i1, i2, p2]
+        i3 = int_add(i1, i2)
+        debug_merge_point(0, 0, 0, 0, 0, ConstPtr(ptr0))
+        guard_nonnull(p2) []
+        guard_true(i3) []
+        """, namespace={'ptr0': code_gcref}).operations
+        for op in oplist_no_descrs:
+            if op.is_guard():
+                op.setdescr(None)
+
         class FailDescr(BasicFailDescr):
             def get_jitcounter_hash(self):
                 from rpython.rlib.rarithmetic import r_uint
@@ -86,18 +97,23 @@
 
         def interp_on_compile():
             di_loop.oplist = cls.oplist
-            pypy_hooks.after_compile(di_loop)
+            if pypy_hooks.are_hooks_enabled():
+                pypy_hooks.after_compile(di_loop)
 
         def interp_on_compile_bridge():
-            pypy_hooks.after_compile_bridge(di_bridge)
+            if pypy_hooks.are_hooks_enabled():
+                pypy_hooks.after_compile_bridge(di_bridge)
 
         def interp_on_optimize():
-            di_loop_optimize.oplist = cls.oplist
-            pypy_hooks.before_compile(di_loop_optimize)
+            if pypy_hooks.are_hooks_enabled():
+                di_loop_optimize.oplist = cls.oplist
+                pypy_hooks.before_compile(di_loop_optimize)
 
         def interp_on_abort():
-            pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver,
-                                greenkey, 'blah', Logger(MockSD), [])
+            if pypy_hooks.are_hooks_enabled():
+                pypy_hooks.on_abort(Counters.ABORT_TOO_LONG, pypyjitdriver,
+                                    greenkey, 'blah', Logger(MockSD),
+                                    cls.oplist_no_descrs)
 
         space = cls.space
         cls.w_on_compile = space.wrap(interp2app(interp_on_compile))
@@ -107,10 +123,12 @@
         cls.w_dmp_num = space.wrap(rop.DEBUG_MERGE_POINT)
         cls.w_on_optimize = space.wrap(interp2app(interp_on_optimize))
         cls.orig_oplist = oplist
+        cls.orig_oplist_no_descrs = oplist_no_descrs
         cls.w_sorted_keys = space.wrap(sorted(Counters.counter_names))
 
     def setup_method(self, meth):
         self.__class__.oplist = self.orig_oplist[:]
+        self.__class__.oplist_no_descrs = self.orig_oplist_no_descrs[:]
 
     def test_on_compile(self):
         import pypyjit
@@ -219,7 +237,11 @@
 
         pypyjit.set_abort_hook(hook)
         self.on_abort()
-        assert l == [('pypyjit', 'ABORT_TOO_LONG', [])]
+        assert len(l) == 1
+        name, reason, ops = l[0]
+        assert name == 'pypyjit'
+        assert reason == 'ABORT_TOO_LONG'
+        assert len(ops) == 4
 
     def test_creation(self):
         from pypyjit import ResOperation
diff --git a/rpython/jit/codewriter/policy.py b/rpython/jit/codewriter/policy.py
--- a/rpython/jit/codewriter/policy.py
+++ b/rpython/jit/codewriter/policy.py
@@ -11,9 +11,6 @@
         self.supports_floats = False
         self.supports_longlong = False
         self.supports_singlefloats = False
-        if jithookiface is None:
-            from rpython.rlib.jit import JitHookInterface
-            jithookiface = JitHookInterface()
         self.jithookiface = jithookiface
 
     def set_supports_floats(self, flag):
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -545,15 +545,17 @@
         show_procedures(metainterp_sd, loop)
         loop.check_consistency()
 
+    debug_info = None
+    hooks = None
     if metainterp_sd.warmrunnerdesc is not None:
         hooks = metainterp_sd.warmrunnerdesc.hooks
-        debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
-                                  original_jitcell_token, loop.operations,
-                                  type, greenkey)
-        hooks.before_compile(debug_info)
-    else:
-        debug_info = None
-        hooks = None
+        if hooks.are_hooks_enabled():
+            debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+                                      original_jitcell_token, loop.operations,
+                                      type, greenkey)
+            hooks.before_compile(debug_info)
+        else:
+            hooks = None
     operations = get_deep_immutable_oplist(loop.operations)
     metainterp_sd.profiler.start_backend()
     debug_start("jit-backend")
@@ -597,15 +599,17 @@
         show_procedures(metainterp_sd)
         seen = dict.fromkeys(inputargs)
         TreeLoop.check_consistency_of_branch(operations, seen)
+    debug_info = None
+    hooks = None
     if metainterp_sd.warmrunnerdesc is not None:
         hooks = metainterp_sd.warmrunnerdesc.hooks
-        debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
-                                  original_loop_token, operations, 'bridge',
-                                  fail_descr=faildescr)
-        hooks.before_compile_bridge(debug_info)
-    else:
-        hooks = None
-        debug_info = None
+        if hooks.are_hooks_enabled():
+            debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops,
+                                      original_loop_token, operations, 'bridge',
+                                      fail_descr=faildescr)
+            hooks.before_compile_bridge(debug_info)
+        else:
+            hooks = None
     operations = get_deep_immutable_oplist(operations)
     metainterp_sd.profiler.start_backend()
     debug_start("jit-backend")
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -701,6 +701,9 @@
     def length(self):
         return self.trace._count - len(self.trace.inputargs)
 
+    def trace_tag_overflow(self):
+        return self.trace.tag_overflow
+
     def get_trace_position(self):
         return self.trace.cut_point()
 
diff --git a/rpython/jit/metainterp/opencoder.py b/rpython/jit/metainterp/opencoder.py
--- a/rpython/jit/metainterp/opencoder.py
+++ b/rpython/jit/metainterp/opencoder.py
@@ -49,13 +49,6 @@
     way up to lltype.Signed for indexes everywhere
     """
 
-def frontend_tag_overflow():
-    # Minor abstraction leak: raise directly the right exception
-    # expected by the rest of the machinery
-    from rpython.jit.metainterp import history
-    from rpython.rlib.jit import Counters
-    raise history.SwitchToBlackhole(Counters.ABORT_TOO_LONG)
-
 class BaseTrace(object):
     pass
 
@@ -293,6 +286,7 @@
         self._start = len(inputargs)
         self._pos = self._start
         self.inputargs = inputargs
+        self.tag_overflow = False
 
     def append(self, v):
         model = get_model(self)
@@ -300,12 +294,14 @@
             # grow by 2X
             self._ops = self._ops + [rffi.cast(model.STORAGE_TP, 0)] * len(self._ops)
         if not model.MIN_VALUE <= v <= model.MAX_VALUE:
-            raise frontend_tag_overflow()
+            v = 0 # broken value, but that's fine, tracing will stop soon
+            self.tag_overflow = True
         self._ops[self._pos] = rffi.cast(model.STORAGE_TP, v)
         self._pos += 1
 
-    def done(self):
+    def tracing_done(self):
         from rpython.rlib.debug import debug_start, debug_stop, debug_print
+        assert not self.tag_overflow
 
         self._bigints_dict = {}
         self._refs_dict = llhelper.new_ref_dict_3()
@@ -317,8 +313,6 @@
         debug_print(" ref consts: " + str(self._consts_ptr) + " " + str(len(self._refs)))
         debug_print(" descrs: " + str(len(self._descrs)))
         debug_stop("jit-trace-done")
-        return 0 # completely different than TraceIter.done, but we have to
-        # share the base class
 
     def length(self):
         return self._pos
@@ -379,6 +373,7 @@
 
     def record_op(self, opnum, argboxes, descr=None):
         pos = self._index
+        old_pos = self._pos
         self.append(opnum)
         expected_arity = oparity[opnum]
         if expected_arity == -1:
@@ -397,6 +392,10 @@
         self._count += 1
         if opclasses[opnum].type != 'v':
             self._index += 1
+        if self.tag_overflow:
+            # potentially a broken op is left behind
+            # clean it up
+            self._pos = old_pos
         return pos
 
     def _encode_descr(self, descr):
@@ -424,10 +423,11 @@
         vref_array = self._list_of_boxes(vref_boxes)
         s = TopSnapshot(combine_uint(jitcode.index, pc), array, vable_array,
                         vref_array)
-        assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
         # guards have no descr
         self._snapshots.append(s)
-        self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
+        if not self.tag_overflow: # otherwise we're broken anyway
+            assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
+            self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
         return s
 
     def create_empty_top_snapshot(self, vable_boxes, vref_boxes):
@@ -436,10 +436,11 @@
         vref_array = self._list_of_boxes(vref_boxes)
         s = TopSnapshot(combine_uint(2**16 - 1, 0), [], vable_array,
                         vref_array)
-        assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
         # guards have no descr
         self._snapshots.append(s)
-        self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
+        if not self.tag_overflow: # otherwise we're broken anyway
+            assert rffi.cast(lltype.Signed, self._ops[self._pos - 1]) == 0
+            self._ops[self._pos - 1] = rffi.cast(get_model(self).STORAGE_TP, len(self._snapshots) - 1)
         return s
 
     def create_snapshot(self, jitcode, pc, frame, flag):
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2365,7 +2365,9 @@
             greenkey = None # we're in the bridge
         else:
             greenkey = self.current_merge_points[0][0][:jd_sd.num_green_args]
-            self.staticdata.warmrunnerdesc.hooks.on_abort(reason,
+            hooks = self.staticdata.warmrunnerdesc.hooks
+            if hooks.are_hooks_enabled():
+                hooks.on_abort(reason,
                     jd_sd.jitdriver, greenkey,
                     jd_sd.warmstate.get_location_str(greenkey),
                     self.staticdata.logger_ops._make_log_operations(
@@ -2374,9 +2376,10 @@
             if self.aborted_tracing_jitdriver is not None:
                 jd_sd = self.aborted_tracing_jitdriver
                 greenkey = self.aborted_tracing_greenkey
-                self.staticdata.warmrunnerdesc.hooks.on_trace_too_long(
-                    jd_sd.jitdriver, greenkey,
-                    jd_sd.warmstate.get_location_str(greenkey))
+                if hooks.are_hooks_enabled():
+                    hooks.on_trace_too_long(
+                        jd_sd.jitdriver, greenkey,
+                        jd_sd.warmstate.get_location_str(greenkey))
                 # no ops for now
                 self.aborted_tracing_jitdriver = None
                 self.aborted_tracing_greenkey = None
@@ -2384,9 +2387,9 @@
 
     def blackhole_if_trace_too_long(self):
         warmrunnerstate = self.jitdriver_sd.warmstate
-        if self.history.length() > warmrunnerstate.trace_limit:
+        if (self.history.length() > warmrunnerstate.trace_limit or
+                self.history.trace_tag_overflow()):
             jd_sd, greenkey_of_huge_function = self.find_biggest_function()
-            self.history.trace.done()
             self.staticdata.stats.record_aborted(greenkey_of_huge_function)
             self.portal_trace_positions = None
             if greenkey_of_huge_function is not None:
@@ -2689,7 +2692,9 @@
                      try_disabling_unroll=False, exported_state=None):
         num_green_args = self.jitdriver_sd.num_green_args
         greenkey = original_boxes[:num_green_args]
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         if not self.partial_trace:
             ptoken = self.get_procedure_token(greenkey)
             if ptoken is not None and ptoken.target_tokens is not None:
@@ -2742,7 +2747,9 @@
         self.history.record(rop.JUMP, live_arg_boxes[num_green_args:], None,
                             descr=target_jitcell_token)
         self.history.ends_with_jump = True
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         try:
             target_token = compile.compile_trace(self, self.resumekey,
                 live_arg_boxes[num_green_args:])
@@ -2776,7 +2783,9 @@
             assert False
         # FIXME: can we call compile_trace?
         self.history.record(rop.FINISH, exits, None, descr=token)
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         target_token = compile.compile_trace(self, self.resumekey, exits)
         if target_token is not token:
             compile.giveup()
@@ -2802,7 +2811,9 @@
         sd = self.staticdata
         token = sd.exit_frame_with_exception_descr_ref
         self.history.record(rop.FINISH, [valuebox], None, descr=token)
-        self.history.trace.done()
+        if self.history.trace_tag_overflow():
+            raise SwitchToBlackhole(Counters.ABORT_TOO_LONG)
+        self.history.trace.tracing_done()
         target_token = compile.compile_trace(self, self.resumekey, [valuebox])
         if target_token is not token:
             compile.giveup()
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -4661,3 +4661,36 @@
 
         f() # finishes
         self.meta_interp(f, [])
+
+    def test_trace_too_long_bug(self):
+        driver = JitDriver(greens=[], reds=['i'])
+        @unroll_safe
+        def match(s):
+            l = len(s)
+            p = 0
+            for i in range(2500): # produces too long trace
+                c = s[p]
+                if c != 'a':
+                    return False
+                p += 1
+                if p >= l:
+                    return True
+                c = s[p]
+                if c != '\n':
+                    p += 1
+                    if p >= l:
+                        return True
+                else:
+                    return False
+            return True
+
+        def f(i):
+            while i > 0:
+                driver.jit_merge_point(i=i)
+                match('a' * (500 * i))
+                i -= 1
+            return i
+
+        res = self.meta_interp(f, [10])
+        assert res == f(10)
+
diff --git a/rpython/jit/metainterp/test/test_greenfield.py b/rpython/jit/metainterp/test/test_greenfield.py
--- a/rpython/jit/metainterp/test/test_greenfield.py
+++ b/rpython/jit/metainterp/test/test_greenfield.py
@@ -1,6 +1,17 @@
+import pytest
 from rpython.jit.metainterp.test.support import LLJitMixin
 from rpython.rlib.jit import JitDriver, assert_green
 
+pytest.skip("this feature is disabled at the moment!")
+
+# note why it is disabled: before d721da4573ad
+# there was a failing assert when inlining python -> sre -> python:
+# https://bitbucket.org/pypy/pypy/issues/2775/
+# this shows, that the interaction of greenfields and virtualizables is broken,
+# because greenfields use MetaInterp.virtualizable_boxes, which confuses
+# MetaInterp._nonstandard_virtualizable somehow (and makes no sense
+# conceptually anyway). to fix greenfields, the two mechanisms would have to be
+# disentangled.
 
 class GreenFieldsTests:
 
diff --git a/rpython/jit/metainterp/test/test_jitiface.py b/rpython/jit/metainterp/test/test_jitiface.py
--- a/rpython/jit/metainterp/test/test_jitiface.py
+++ b/rpython/jit/metainterp/test/test_jitiface.py
@@ -238,7 +238,7 @@
 
         hashes = Hashes()
 
-        class Hooks(object):
+        class Hooks(JitHookInterface):
             def before_compile(self, debug_info):
                 pass
 
@@ -279,6 +279,44 @@
         self.meta_interp(main, [1, 1], policy=JitPolicy(hooks))
         assert len(hashes.t) == 1
 
+
+    def test_are_hooks_enabled(self):
+        reasons = []
+
+        class MyJitIface(JitHookInterface):
+            def are_hooks_enabled(self):
+                return False
+
+            def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, ops):
+                reasons.append(reason)
+
+        iface = MyJitIface()
+
+        myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'],
+                                get_printable_location=lambda *args: 'blah')
+
+        class Foo:
+            _immutable_fields_ = ['a?']
+
+            def __init__(self, a):
+                self.a = a
+
+        def f(a, x):
+            foo = Foo(a)
+            total = 0
+            while x > 0:
+                myjitdriver.jit_merge_point(foo=foo, x=x, total=total)
+                total += foo.a
+                foo.a += 1
+                x -= 1
+            return total
+        #
+        assert f(100, 7) == 721
+        res = self.meta_interp(f, [100, 7], policy=JitPolicy(iface))
+        assert res == 721
+        assert reasons == []
+
+
 class LLJitHookInterfaceTests(JitHookInterfaceTests):
     # use this for any backend, instead of the super class
     
@@ -320,7 +358,6 @@
         # this so far does not work because of the way setup_once is done,
         # but fine, it's only about untranslated version anyway
         #self.meta_interp(main, [False], ProfilerClass=Profiler)
-        
 
 class TestJitHookInterface(JitHookInterfaceTests, LLJitMixin):
     pass
diff --git a/rpython/jit/metainterp/test/test_opencoder.py b/rpython/jit/metainterp/test/test_opencoder.py
--- a/rpython/jit/metainterp/test/test_opencoder.py
+++ b/rpython/jit/metainterp/test/test_opencoder.py
@@ -209,5 +209,8 @@
     def test_tag_overflow(self):
         t = Trace([], metainterp_sd)
         i0 = FakeOp(100000)
-        py.test.raises(SwitchToBlackhole, t.record_op, rop.FINISH, [i0])
-        assert t.unpack() == ([], [])
+        # if we overflow, we can keep recording
+        for i in range(10):
+            t.record_op(rop.FINISH, [i0])
+            assert t.unpack() == ([], [])
+        assert t.tag_overflow
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -220,6 +220,15 @@
     stats.check_consistency()
 
 # ____________________________________________________________
+# always disabled hooks interface
+
+from rpython.rlib.jit import JitHookInterface
+
+class NoHooksInterface(JitHookInterface):
+    def are_hooks_enabled(self):
+        return False
+
+# ____________________________________________________________
 
 class WarmRunnerDesc(object):
 
@@ -259,7 +268,7 @@
         else:
             self.jitcounter = counter.DeterministicJitCounter()
         #
-        self.hooks = policy.jithookiface
+        self.make_hooks(policy.jithookiface)
         self.make_virtualizable_infos()
         self.make_driverhook_graphs()
         self.make_enter_functions()
@@ -498,6 +507,12 @@
             self.metainterp_sd.opencoder_model = Model
         self.stats.metainterp_sd = self.metainterp_sd
 
+    def make_hooks(self, hooks):
+        if hooks is None:
+            # interface not overridden, use a special one that is never enabled
+            hooks = NoHooksInterface()
+        self.hooks = hooks
+
     def make_virtualizable_infos(self):
         vinfos = {}
         for jd in self.jitdrivers_sd:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -653,6 +653,9 @@
         self._make_extregistryentries()
         assert get_jitcell_at is None, "get_jitcell_at no longer used"
         assert set_jitcell_at is None, "set_jitcell_at no longer used"
+        for green in self.greens:
+            if "." in green:
+                raise ValueError("green fields are buggy! if you need them fixed, please talk to us")
         self.get_printable_location = get_printable_location
         self.get_location = get_location
         self.has_unique_id = (get_unique_id is not None)
@@ -1084,7 +1087,8 @@
     """ This is the main connector between the JIT and the interpreter.
     Several methods on this class will be invoked at various stages
     of JIT running like JIT loops compiled, aborts etc.
-    An instance of this class will be available as policy.jithookiface.
+    An instance of this class has to be passed into the JitPolicy constructor
+    (and will then be available as policy.jithookiface).
     """
     # WARNING: You should make a single prebuilt instance of a subclass
     # of this class.  You can, before translation, initialize some
@@ -1094,6 +1098,13 @@
     # of the program!  A line like ``pypy_hooks.foo = ...`` must not
     # appear inside your interpreter's RPython code.
 
+    def are_hooks_enabled(self):
+        """ A hook that is called to check whether the interpreter's hooks are
+        enabled at all. Only if this function returns True, are the other hooks
+        called. Otherwise, nothing happens. This is done because constructing
+        some of the hooks' arguments is expensive, so we'd rather not do it."""
+        return True
+
     def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations):
         """ A hook called each time a loop is aborted with jitdriver and
         greenkey where it started, reason is a string why it got aborted
diff --git a/rpython/rlib/rsre/rpy/_sre.py b/rpython/rlib/rsre/rpy/_sre.py
--- a/rpython/rlib/rsre/rpy/_sre.py
+++ b/rpython/rlib/rsre/rpy/_sre.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rsre import rsre_char
+from rpython.rlib.rsre import rsre_char, rsre_core
 from rpython.rlib.rarithmetic import intmask
 
 VERSION = "2.7.6"
@@ -12,7 +12,7 @@
     pass
 
 def compile(pattern, flags, code, *args):
-    raise GotIt([intmask(i) for i in code], flags, args)
+    raise GotIt(rsre_core.CompiledPattern([intmask(i) for i in code]), flags, args)
 
 
 def get_code(regexp, flags=0, allargs=False):
diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -152,17 +152,16 @@
 ##### Charset evaluation
 
 @jit.unroll_safe
-def check_charset(ctx, ppos, char_code):
+def check_charset(ctx, pattern, ppos, char_code):
     """Checks whether a character matches set of arbitrary length.
     The set starts at pattern[ppos]."""
     negated = False
     result = False
-    pattern = ctx.pattern
     while True:
-        opcode = pattern[ppos]
+        opcode = pattern.pattern[ppos]
         for i, function in set_dispatch_unroll:
             if opcode == i:
-                newresult, ppos = function(ctx, ppos, char_code)
+                newresult, ppos = function(ctx, pattern, ppos, char_code)
                 result |= newresult
                 break
         else:
@@ -177,50 +176,44 @@
         return not result
     return result
 
-def set_literal(ctx, index, char_code):
+def set_literal(ctx, pattern, index, char_code):
     #  
-    pat = ctx.pattern
-    match = pat[index+1] == char_code
+    match = pattern.pattern[index+1] == char_code
     return match, index + 2
 
-def set_category(ctx, index, char_code):
+def set_category(ctx, pattern, index, char_code):
     #  
-    pat = ctx.pattern
-    match = category_dispatch(pat[index+1], char_code)
+    match = category_dispatch(pattern.pattern[index+1], char_code)
     return match, index + 2
 
-def set_charset(ctx, index, char_code):
+def set_charset(ctx, pattern, index, char_code):
     #   (16 bits per code word)
-    pat = ctx.pattern
     if CODESIZE == 2:
         match = char_code < 256 and \
-                (pat[index+1+(char_code >> 4)] & (1 << (char_code & 15)))
+                (pattern.pattern[index+1+(char_code >> 4)] & (1 << (char_code & 15)))
         return match, index + 17  # skip bitmap
     else:
         match = char_code < 256 and \
-                (pat[index+1+(char_code >> 5)] & (1 << (char_code & 31)))
+                (pattern.pattern[index+1+(char_code >> 5)] & (1 << (char_code & 31)))
         return match, index + 9   # skip bitmap
 
-def set_range(ctx, index, char_code):
+def set_range(ctx, pattern, index, char_code):
     #   
-    pat = ctx.pattern
-    match = int_between(pat[index+1], char_code, pat[index+2] + 1)
+    match = int_between(pattern.pattern[index+1], char_code, pattern.pattern[index+2] + 1)
     return match, index + 3
 
-def set_range_ignore(ctx, index, char_code):
+def set_range_ignore(ctx, pattern, index, char_code):
     #   
     # the char_code is already lower cased
-    pat = ctx.pattern
-    lower = pat[index + 1]
-    upper = pat[index + 2]
+    lower = pattern.pattern[index + 1]
+    upper = pattern.pattern[index + 2]
     match1 = int_between(lower, char_code, upper + 1)
     match2 = int_between(lower, getupper(char_code, ctx.flags), upper + 1)
     return match1 | match2, index + 3
 
-def set_bigcharset(ctx, index, char_code):
+def set_bigcharset(ctx, pattern, index, char_code):
     #   <256 blockindices> 
-    pat = ctx.pattern
-    count = pat[index+1]
+    count = pattern.pattern[index+1]
     index += 2
 
     if CODESIZE == 2:
@@ -238,7 +231,7 @@
             return False, index
         shift = 5
 
-    block = pat[index + (char_code >> (shift + 5))]
+    block = pattern.pattern[index + (char_code >> (shift + 5))]
 
     block_shift = char_code >> 5
     if BIG_ENDIAN:
@@ -247,23 +240,22 @@
     block = (block >> block_shift) & 0xFF
 
     index += 256 / CODESIZE
-    block_value = pat[index+(block * (32 / CODESIZE)
+    block_value = pattern.pattern[index+(block * (32 / CODESIZE)
                              + ((char_code & 255) >> shift))]
     match = (block_value & (1 << (char_code & ((8 * CODESIZE) - 1))))
     index += count * (32 / CODESIZE)  # skip blocks
     return match, index
 
-def set_unicode_general_category(ctx, index, char_code):
+def set_unicode_general_category(ctx, pattern, index, char_code):
     # Unicode "General category property code" (not used by Python).
-    # A general category is two letters.  'pat[index+1]' contains both
+    # A general category is two letters.  'pattern.pattern[index+1]' contains both
     # the first character, and the second character shifted by 8.
     # http://en.wikipedia.org/wiki/Unicode_character_property#General_Category
     # Also supports single-character categories, if the second character is 0.
     # Negative matches are triggered by bit number 7.
     assert unicodedb is not None
     cat = unicodedb.category(char_code)
-    pat = ctx.pattern
-    category_code = pat[index + 1]
+    category_code = pattern.pattern[index + 1]
     first_character = category_code & 0x7F
     second_character = (category_code >> 8) & 0x7F
     negative_match = category_code & 0x80
diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py
--- a/rpython/rlib/rsre/rsre_core.py
+++ b/rpython/rlib/rsre/rsre_core.py
@@ -83,35 +83,19 @@
     def __init__(self, msg):
         self.msg = msg
 
-class AbstractMatchContext(object):
-    """Abstract base class"""
-    _immutable_fields_ = ['pattern[*]', 'flags', 'end']
-    match_start = 0
-    match_end = 0
-    match_marks = None
-    match_marks_flat = None
-    fullmatch_only = False
 
-    def __init__(self, pattern, match_start, end, flags):
-        # 'match_start' and 'end' must be known to be non-negative
-        # and they must not be more than len(string).
-        check_nonneg(match_start)
-        check_nonneg(end)
+class CompiledPattern(object):
+    _immutable_fields_ = ['pattern[*]']
+
+    def __init__(self, pattern):
         self.pattern = pattern
-        self.match_start = match_start
-        self.end = end
-        self.flags = flags
         # check we don't get the old value of MAXREPEAT
         # during the untranslated tests
         if not we_are_translated():
             assert 65535 not in pattern
 
-    def reset(self, start):
-        self.match_start = start
-        self.match_marks = None
-        self.match_marks_flat = None
-
     def pat(self, index):
+        jit.promote(self)
         check_nonneg(index)
         result = self.pattern[index]
         # Check that we only return non-negative integers from this helper.
@@ -121,6 +105,29 @@
         assert result >= 0
         return result
 
+class AbstractMatchContext(object):
+    """Abstract base class"""
+    _immutable_fields_ = ['flags', 'end']
+    match_start = 0
+    match_end = 0
+    match_marks = None
+    match_marks_flat = None
+    fullmatch_only = False
+
+    def __init__(self, match_start, end, flags):
+        # 'match_start' and 'end' must be known to be non-negative
+        # and they must not be more than len(string).
+        check_nonneg(match_start)
+        check_nonneg(end)
+        self.match_start = match_start
+        self.end = end
+        self.flags = flags
+
+    def reset(self, start):
+        self.match_start = start
+        self.match_marks = None
+        self.match_marks_flat = None
+
     @not_rpython
     def str(self, index):
         """Must be overridden in a concrete subclass.
@@ -183,8 +190,8 @@
 
     _immutable_fields_ = ["_buffer"]
 
-    def __init__(self, pattern, buf, match_start, end, flags):
-        AbstractMatchContext.__init__(self, pattern, match_start, end, flags)
+    def __init__(self, buf, match_start, end, flags):
+        AbstractMatchContext.__init__(self, match_start, end, flags)
         self._buffer = buf
 
     def str(self, index):
@@ -196,7 +203,7 @@
         return rsre_char.getlower(c, self.flags)
 
     def fresh_copy(self, start):
-        return BufMatchContext(self.pattern, self._buffer, start,
+        return BufMatchContext(self._buffer, start,
                                self.end, self.flags)
 
 class StrMatchContext(AbstractMatchContext):
@@ -204,8 +211,8 @@
 
     _immutable_fields_ = ["_string"]
 
-    def __init__(self, pattern, string, match_start, end, flags):
-        AbstractMatchContext.__init__(self, pattern, match_start, end, flags)
+    def __init__(self, string, match_start, end, flags):
+        AbstractMatchContext.__init__(self, match_start, end, flags)
         self._string = string
         if not we_are_translated() and isinstance(string, unicode):
             self.flags |= rsre_char.SRE_FLAG_UNICODE   # for rsre_re.py
@@ -219,7 +226,7 @@
         return rsre_char.getlower(c, self.flags)
 
     def fresh_copy(self, start):
-        return StrMatchContext(self.pattern, self._string, start,
+        return StrMatchContext(self._string, start,
                                self.end, self.flags)
 
 class UnicodeMatchContext(AbstractMatchContext):
@@ -227,8 +234,8 @@
 
     _immutable_fields_ = ["_unicodestr"]
 
-    def __init__(self, pattern, unicodestr, match_start, end, flags):
-        AbstractMatchContext.__init__(self, pattern, match_start, end, flags)
+    def __init__(self, unicodestr, match_start, end, flags):
+        AbstractMatchContext.__init__(self, match_start, end, flags)
         self._unicodestr = unicodestr
 
     def str(self, index):
@@ -240,7 +247,7 @@
         return rsre_char.getlower(c, self.flags)
 
     def fresh_copy(self, start):
-        return UnicodeMatchContext(self.pattern, self._unicodestr, start,
+        return UnicodeMatchContext(self._unicodestr, start,
                                    self.end, self.flags)
 
 # ____________________________________________________________
@@ -265,16 +272,16 @@
 class MatchResult(object):
     subresult = None
 
-    def move_to_next_result(self, ctx):
+    def move_to_next_result(self, ctx, pattern):
         # returns either 'self' or None
         result = self.subresult
         if result is None:
             return
-        if result.move_to_next_result(ctx):
+        if result.move_to_next_result(ctx, pattern):
             return self
-        return self.find_next_result(ctx)
+        return self.find_next_result(ctx, pattern)
 
-    def find_next_result(self, ctx):
+    def find_next_result(self, ctx, pattern):
         raise NotImplementedError
 
 MATCHED_OK = MatchResult()
@@ -287,11 +294,11 @@
         self.start_marks = marks
 
     @jit.unroll_safe
-    def find_first_result(self, ctx):
+    def find_first_result(self, ctx, pattern):
         ppos = jit.hint(self.ppos, promote=True)
-        while ctx.pat(ppos):
-            result = sre_match(ctx, ppos + 1, self.start_ptr, self.start_marks)
-            ppos += ctx.pat(ppos)
+        while pattern.pat(ppos):
+            result = sre_match(ctx, pattern, ppos + 1, self.start_ptr, self.start_marks)
+            ppos += pattern.pat(ppos)
             if result is not None:
                 self.subresult = result
                 self.ppos = ppos
@@ -300,7 +307,7 @@
 
 class RepeatOneMatchResult(MatchResult):
     install_jitdriver('RepeatOne',
-                      greens=['nextppos', 'ctx.pattern'],
+                      greens=['nextppos', 'pattern'],
                       reds=['ptr', 'self', 'ctx'],
                       debugprint=(1, 0))   # indices in 'greens'
 
@@ -310,13 +317,14 @@
         self.start_ptr = ptr
         self.start_marks = marks
 
-    def find_first_result(self, ctx):
+    def find_first_result(self, ctx, pattern):
         ptr = self.start_ptr
         nextppos = self.nextppos
         while ptr >= self.minptr:
             ctx.jitdriver_RepeatOne.jit_merge_point(
-                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos)
-            result = sre_match(ctx, nextppos, ptr, self.start_marks)
+                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos,
+                pattern=pattern)
+            result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks)
             ptr -= 1
             if result is not None:
                 self.subresult = result
@@ -327,7 +335,7 @@
 
 class MinRepeatOneMatchResult(MatchResult):
     install_jitdriver('MinRepeatOne',
-                      greens=['nextppos', 'ppos3', 'ctx.pattern'],
+                      greens=['nextppos', 'ppos3', 'pattern'],
                       reds=['ptr', 'self', 'ctx'],
                       debugprint=(2, 0))   # indices in 'greens'
 
@@ -338,39 +346,40 @@
         self.start_ptr = ptr
         self.start_marks = marks
 
-    def find_first_result(self, ctx):
+    def find_first_result(self, ctx, pattern):
         ptr = self.start_ptr
         nextppos = self.nextppos
         ppos3 = self.ppos3
         while ptr <= self.maxptr:
             ctx.jitdriver_MinRepeatOne.jit_merge_point(
-                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3)
-            result = sre_match(ctx, nextppos, ptr, self.start_marks)
+                self=self, ptr=ptr, ctx=ctx, nextppos=nextppos, ppos3=ppos3,
+                pattern=pattern)
+            result = sre_match(ctx, pattern, nextppos, ptr, self.start_marks)
             if result is not None:
                 self.subresult = result
                 self.start_ptr = ptr
                 return self
-            if not self.next_char_ok(ctx, ptr, ppos3):
+            if not self.next_char_ok(ctx, pattern, ptr, ppos3):
                 break
             ptr += 1
 
-    def find_next_result(self, ctx):
+    def find_next_result(self, ctx, pattern):
         ptr = self.start_ptr
-        if not self.next_char_ok(ctx, ptr, self.ppos3):
+        if not self.next_char_ok(ctx, pattern, ptr, self.ppos3):
             return
         self.start_ptr = ptr + 1
-        return self.find_first_result(ctx)
+        return self.find_first_result(ctx, pattern)
 
-    def next_char_ok(self, ctx, ptr, ppos):
+    def next_char_ok(self, ctx, pattern, ptr, ppos):
         if ptr == ctx.end:
             return False
-        op = ctx.pat(ppos)
+        op = pattern.pat(ppos)
         for op1, checkerfn in unroll_char_checker:
             if op1 == op:
-                return checkerfn(ctx, ptr, ppos)
+                return checkerfn(ctx, pattern, ptr, ppos)
         # obscure case: it should be a single char pattern, but isn't
         # one of the opcodes in unroll_char_checker (see test_ext_opcode)
-        return sre_match(ctx, ppos, ptr, self.start_marks) is not None
+        return sre_match(ctx, pattern, ppos, ptr, self.start_marks) is not None
 
 class AbstractUntilMatchResult(MatchResult):
 
@@ -391,17 +400,17 @@
 
 class MaxUntilMatchResult(AbstractUntilMatchResult):
     install_jitdriver('MaxUntil',
-                      greens=['ppos', 'tailppos', 'match_more', 'ctx.pattern'],
+                      greens=['ppos', 'tailppos', 'match_more', 'pattern'],
                       reds=['ptr', 'marks', 'self', 'ctx'],
                       debugprint=(3, 0, 2))
 
-    def find_first_result(self, ctx):
-        return self.search_next(ctx, match_more=True)
+    def find_first_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, match_more=True)
 
-    def find_next_result(self, ctx):
-        return self.search_next(ctx, match_more=False)
+    def find_next_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, match_more=False)
 
-    def search_next(self, ctx, match_more):
+    def search_next(self, ctx, pattern, match_more):
         ppos = self.ppos
         tailppos = self.tailppos
         ptr = self.cur_ptr
@@ -409,12 +418,13 @@
         while True:
             ctx.jitdriver_MaxUntil.jit_merge_point(
                 ppos=ppos, tailppos=tailppos, match_more=match_more,
-                ptr=ptr, marks=marks, self=self, ctx=ctx)
+                ptr=ptr, marks=marks, self=self, ctx=ctx,
+                pattern=pattern)
             if match_more:
-                max = ctx.pat(ppos+2)
+                max = pattern.pat(ppos+2)
                 if max == rsre_char.MAXREPEAT or self.num_pending < max:
                     # try to match one more 'item'
-                    enum = sre_match(ctx, ppos + 3, ptr, marks)
+                    enum = sre_match(ctx, pattern, ppos + 3, ptr, marks)
                 else:
                     enum = None    # 'max' reached, no more matches
             else:
@@ -425,9 +435,9 @@
                 self.num_pending -= 1
                 ptr = p.ptr
                 marks = p.marks
-                enum = p.enum.move_to_next_result(ctx)
+                enum = p.enum.move_to_next_result(ctx, pattern)
             #
-            min = ctx.pat(ppos+1)
+            min = pattern.pat(ppos+1)
             if enum is not None:
                 # matched one more 'item'.  record it and continue.
                 last_match_length = ctx.match_end - ptr
@@ -447,7 +457,7 @@
             # 'item' no longer matches.
             if self.num_pending >= min:
                 # try to match 'tail' if we have enough 'item'
-                result = sre_match(ctx, tailppos, ptr, marks)
+                result = sre_match(ctx, pattern, tailppos, ptr, marks)
                 if result is not None:
                     self.subresult = result
                     self.cur_ptr = ptr
@@ -457,23 +467,23 @@
 
 class MinUntilMatchResult(AbstractUntilMatchResult):
 
-    def find_first_result(self, ctx):
-        return self.search_next(ctx, resume=False)
+    def find_first_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, resume=False)
 
-    def find_next_result(self, ctx):
-        return self.search_next(ctx, resume=True)
+    def find_next_result(self, ctx, pattern):
+        return self.search_next(ctx, pattern, resume=True)
 
-    def search_next(self, ctx, resume):
+    def search_next(self, ctx, pattern, resume):
         # XXX missing jit support here
         ppos = self.ppos
-        min = ctx.pat(ppos+1)
-        max = ctx.pat(ppos+2)
+        min = pattern.pat(ppos+1)
+        max = pattern.pat(ppos+2)
         ptr = self.cur_ptr
         marks = self.cur_marks
         while True:
             # try to match 'tail' if we have enough 'item'
             if not resume and self.num_pending >= min:
-                result = sre_match(ctx, self.tailppos, ptr, marks)
+                result = sre_match(ctx, pattern, self.tailppos, ptr, marks)
                 if result is not None:
                     self.subresult = result
                     self.cur_ptr = ptr
@@ -483,12 +493,12 @@
 
             if max == rsre_char.MAXREPEAT or self.num_pending < max:
                 # try to match one more 'item'
-                enum = sre_match(ctx, ppos + 3, ptr, marks)
+                enum = sre_match(ctx, pattern, ppos + 3, ptr, marks)
                 #
                 # zero-width match protection
                 if self.num_pending >= min:
                     while enum is not None and ptr == ctx.match_end:
-                        enum = enum.move_to_next_result(ctx)
+                        enum = enum.move_to_next_result(ctx, pattern)
             else:
                 enum = None    # 'max' reached, no more matches
 
@@ -502,7 +512,7 @@
                 self.num_pending -= 1
                 ptr = p.ptr
                 marks = p.marks
-                enum = p.enum.move_to_next_result(ctx)
+                enum = p.enum.move_to_next_result(ctx, pattern)
 
             # matched one more 'item'.  record it and continue
             self.pending = Pending(ptr, marks, enum, self.pending)
@@ -514,13 +524,13 @@
 
 @specializectx
 @jit.unroll_safe
-def sre_match(ctx, ppos, ptr, marks):
+def sre_match(ctx, pattern, ppos, ptr, marks):
     """Returns either None or a MatchResult object.  Usually we only need
     the first result, but there is the case of REPEAT...UNTIL where we
     need all results; in that case we use the method move_to_next_result()
     of the MatchResult."""
     while True:
-        op = ctx.pat(ppos)
+        op = pattern.pat(ppos)
         ppos += 1
 
         #jit.jit_debug("sre_match", op, ppos, ptr)
@@ -563,33 +573,33 @@
         elif op == OPCODE_ASSERT:
             # assert subpattern
             #  <0=skip> <1=back> 
-            ptr1 = ptr - ctx.pat(ppos+1)
+            ptr1 = ptr - pattern.pat(ppos+1)
             saved = ctx.fullmatch_only
             ctx.fullmatch_only = False
-            stop = ptr1 < 0 or sre_match(ctx, ppos + 2, ptr1, marks) is None
+            stop = ptr1 < 0 or sre_match(ctx, pattern, ppos + 2, ptr1, marks) is None
             ctx.fullmatch_only = saved
             if stop:
                 return
             marks = ctx.match_marks
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_ASSERT_NOT:
             # assert not subpattern
             #  <0=skip> <1=back> 
-            ptr1 = ptr - ctx.pat(ppos+1)
+            ptr1 = ptr - pattern.pat(ppos+1)
             saved = ctx.fullmatch_only
             ctx.fullmatch_only = False
-            stop = (ptr1 >= 0 and sre_match(ctx, ppos + 2, ptr1, marks)
+            stop = (ptr1 >= 0 and sre_match(ctx, pattern, ppos + 2, ptr1, marks)
                                       is not None)
             ctx.fullmatch_only = saved
             if stop:
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_AT:
             # match at given position (e.g. at beginning, at boundary, etc.)
             #  
-            if not sre_at(ctx, ctx.pat(ppos), ptr):
+            if not sre_at(ctx, pattern.pat(ppos), ptr):
                 return
             ppos += 1
 
@@ -597,14 +607,14 @@
             # alternation
             #  <0=skip> code  ... 
             result = BranchMatchResult(ppos, ptr, marks)
-            return result.find_first_result(ctx)
+            return result.find_first_result(ctx, pattern)
 
         elif op == OPCODE_CATEGORY:
             # seems to be never produced, but used by some tests from
             # pypy/module/_sre/test
             #  
             if (ptr == ctx.end or
-                not rsre_char.category_dispatch(ctx.pat(ppos), ctx.str(ptr))):
+                not rsre_char.category_dispatch(pattern.pat(ppos), ctx.str(ptr))):
                 return
             ptr += 1
             ppos += 1
@@ -612,7 +622,7 @@
         elif op == OPCODE_GROUPREF:
             # match backreference
             #  
-            startptr, length = get_group_ref(marks, ctx.pat(ppos))
+            startptr, length = get_group_ref(marks, pattern.pat(ppos))
             if length < 0:
                 return     # group was not previously defined
             if not match_repeated(ctx, ptr, startptr, length):
@@ -623,7 +633,7 @@
         elif op == OPCODE_GROUPREF_IGNORE:
             # match backreference
             #  
-            startptr, length = get_group_ref(marks, ctx.pat(ppos))
+            startptr, length = get_group_ref(marks, pattern.pat(ppos))
             if length < 0:
                 return     # group was not previously defined
             if not match_repeated_ignore(ctx, ptr, startptr, length):
@@ -634,44 +644,44 @@
         elif op == OPCODE_GROUPREF_EXISTS:
             # conditional match depending on the existence of a group
             #    codeyes  codeno ...
-            _, length = get_group_ref(marks, ctx.pat(ppos))
+            _, length = get_group_ref(marks, pattern.pat(ppos))
             if length >= 0:
                 ppos += 2                  # jump to 'codeyes'
             else:
-                ppos += ctx.pat(ppos+1)    # jump to 'codeno'
+                ppos += pattern.pat(ppos+1)    # jump to 'codeno'
 
         elif op == OPCODE_IN:
             # match set member (or non_member)
             #   
-            if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1,
+            if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1,
                                                              ctx.str(ptr)):
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
             ptr += 1
 
         elif op == OPCODE_IN_IGNORE:
             # match set member (or non_member), ignoring case
             #   
-            if ptr >= ctx.end or not rsre_char.check_charset(ctx, ppos+1,
+            if ptr >= ctx.end or not rsre_char.check_charset(ctx, pattern, ppos+1,
                                                              ctx.lowstr(ptr)):
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
             ptr += 1
 
         elif op == OPCODE_INFO:
             # optimization info block
             #  <0=skip> <1=flags> <2=min> ...
-            if (ctx.end - ptr) < ctx.pat(ppos+2):
+            if (ctx.end - ptr) < pattern.pat(ppos+2):
                 return
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_JUMP:
-            ppos += ctx.pat(ppos)
+            ppos += pattern.pat(ppos)
 
         elif op == OPCODE_LITERAL:
             # match literal string
             #  
-            if ptr >= ctx.end or ctx.str(ptr) != ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.str(ptr) != pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -679,7 +689,7 @@
         elif op == OPCODE_LITERAL_IGNORE:
             # match literal string, ignoring case
             #  
-            if ptr >= ctx.end or ctx.lowstr(ptr) != ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.lowstr(ptr) != pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -687,14 +697,14 @@
         elif op == OPCODE_MARK:
             # set mark
             #  
-            gid = ctx.pat(ppos)
+            gid = pattern.pat(ppos)
             marks = Mark(gid, ptr, marks)
             ppos += 1
 
         elif op == OPCODE_NOT_LITERAL:
             # match if it's not a literal string
             #  
-            if ptr >= ctx.end or ctx.str(ptr) == ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.str(ptr) == pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -702,7 +712,7 @@
         elif op == OPCODE_NOT_LITERAL_IGNORE:
             # match if it's not a literal string, ignoring case
             #  
-            if ptr >= ctx.end or ctx.lowstr(ptr) == ctx.pat(ppos):
+            if ptr >= ctx.end or ctx.lowstr(ptr) == pattern.pat(ppos):
                 return
             ppos += 1
             ptr += 1
@@ -715,22 +725,22 @@
 
             # decode the later UNTIL operator to see if it is actually
             # a MAX_UNTIL or MIN_UNTIL
-            untilppos = ppos + ctx.pat(ppos)
+            untilppos = ppos + pattern.pat(ppos)
             tailppos = untilppos + 1
-            op = ctx.pat(untilppos)
+            op = pattern.pat(untilppos)
             if op == OPCODE_MAX_UNTIL:
                 # the hard case: we have to match as many repetitions as
                 # possible, followed by the 'tail'.  we do this by
                 # remembering each state for each possible number of
                 # 'item' matching.
                 result = MaxUntilMatchResult(ppos, tailppos, ptr, marks)
-                return result.find_first_result(ctx)
+                return result.find_first_result(ctx, pattern)
 
             elif op == OPCODE_MIN_UNTIL:
                 # first try to match the 'tail', and if it fails, try
                 # to match one more 'item' and try again
                 result = MinUntilMatchResult(ppos, tailppos, ptr, marks)
-                return result.find_first_result(ctx)
+                return result.find_first_result(ctx, pattern)
 
             else:
                 raise Error("missing UNTIL after REPEAT")
@@ -743,17 +753,18 @@
             # use the MAX_REPEAT operator.
             #   <1=min> <2=max> item  tail
             start = ptr
-            minptr = start + ctx.pat(ppos+1)
+            minptr = start + pattern.pat(ppos+1)
             if minptr > ctx.end:
                 return    # cannot match
-            ptr = find_repetition_end(ctx, ppos+3, start, ctx.pat(ppos+2),
+            ptr = find_repetition_end(ctx, pattern, ppos+3, start,
+                                      pattern.pat(ppos+2),
                                       marks)
             # when we arrive here, ptr points to the tail of the target
             # string.  check if the rest of the pattern matches,
             # and backtrack if not.
-            nextppos = ppos + ctx.pat(ppos)
+            nextppos = ppos + pattern.pat(ppos)
             result = RepeatOneMatchResult(nextppos, minptr, ptr, marks)
-            return result.find_first_result(ctx)
+            return result.find_first_result(ctx, pattern)
 
         elif op == OPCODE_MIN_REPEAT_ONE:
             # match repeated sequence (minimizing regexp).
@@ -763,26 +774,26 @@
             # use the MIN_REPEAT operator.
             #   <1=min> <2=max> item  tail
             start = ptr
-            min = ctx.pat(ppos+1)
+            min = pattern.pat(ppos+1)
             if min > 0:
                 minptr = ptr + min
                 if minptr > ctx.end:
                     return   # cannot match
                 # count using pattern min as the maximum
-                ptr = find_repetition_end(ctx, ppos+3, ptr, min, marks)
+                ptr = find_repetition_end(ctx, pattern, ppos+3, ptr, min, marks)
                 if ptr < minptr:
                     return   # did not match minimum number of times
 
             maxptr = ctx.end
-            max = ctx.pat(ppos+2)
+            max = pattern.pat(ppos+2)
             if max != rsre_char.MAXREPEAT:
                 maxptr1 = start + max
                 if maxptr1 <= maxptr:
                     maxptr = maxptr1
-            nextppos = ppos + ctx.pat(ppos)
+            nextppos = ppos + pattern.pat(ppos)
             result = MinRepeatOneMatchResult(nextppos, ppos+3, maxptr,
                                              ptr, marks)
-            return result.find_first_result(ctx)
+            return result.find_first_result(ctx, pattern)
 
         else:
             raise Error("bad pattern code %d" % op)
@@ -816,7 +827,7 @@
     return True
 
 @specializectx
-def find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+def find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks):
     end = ctx.end
     ptrp1 = ptr + 1
     # First get rid of the cases where we don't have room for any match.
@@ -826,16 +837,16 @@
     # The idea is to be fast for cases like re.search("b+"), where we expect
     # the common case to be a non-match.  It's much faster with the JIT to
     # have the non-match inlined here rather than detect it in the fre() call.
-    op = ctx.pat(ppos)
+    op = pattern.pat(ppos)
     for op1, checkerfn in unroll_char_checker:
         if op1 == op:
-            if checkerfn(ctx, ptr, ppos):
+            if checkerfn(ctx, pattern, ptr, ppos):
                 break
             return ptr
     else:
         # obscure case: it should be a single char pattern, but isn't
         # one of the opcodes in unroll_char_checker (see test_ext_opcode)
-        return general_find_repetition_end(ctx, ppos, ptr, maxcount, marks)
+        return general_find_repetition_end(ctx, pattern, ppos, ptr, maxcount, marks)
     # It matches at least once.  If maxcount == 1 (relatively common),
     # then we are done.
     if maxcount == 1:
@@ -846,14 +857,14 @@
         end1 = ptr + maxcount
         if end1 <= end:
             end = end1
-    op = ctx.pat(ppos)
+    op = pattern.pat(ppos)
     for op1, fre in unroll_fre_checker:
         if op1 == op:
-            return fre(ctx, ptrp1, end, ppos)
+            return fre(ctx, pattern, ptrp1, end, ppos)
     raise Error("rsre.find_repetition_end[%d]" % op)
 
 @specializectx
-def general_find_repetition_end(ctx, ppos, ptr, maxcount, marks):
+def general_find_repetition_end(ctx, patern, ppos, ptr, maxcount, marks):
     # moved into its own JIT-opaque function
     end = ctx.end
     if maxcount != rsre_char.MAXREPEAT:
@@ -861,63 +872,65 @@
         end1 = ptr + maxcount
         if end1 <= end:
             end = end1
-    while ptr < end and sre_match(ctx, ppos, ptr, marks) is not None:
+    while ptr < end and sre_match(ctx, patern, ppos, ptr, marks) is not None:
         ptr += 1
     return ptr
 
 @specializectx
-def match_ANY(ctx, ptr, ppos):   # dot wildcard.
+def match_ANY(ctx, pattern, ptr, ppos):   # dot wildcard.
     return not rsre_char.is_linebreak(ctx.str(ptr))
-def match_ANY_ALL(ctx, ptr, ppos):
+def match_ANY_ALL(ctx, pattern, ptr, ppos):
     return True    # match anything (including a newline)
 @specializectx
-def match_IN(ctx, ptr, ppos):
-    return rsre_char.check_charset(ctx, ppos+2, ctx.str(ptr))
+def match_IN(ctx, pattern, ptr, ppos):
+    return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.str(ptr))
 @specializectx
-def match_IN_IGNORE(ctx, ptr, ppos):
-    return rsre_char.check_charset(ctx, ppos+2, ctx.lowstr(ptr))
+def match_IN_IGNORE(ctx, pattern, ptr, ppos):
+    return rsre_char.check_charset(ctx, pattern, ppos+2, ctx.lowstr(ptr))
 @specializectx
-def match_LITERAL(ctx, ptr, ppos):
-    return ctx.str(ptr) == ctx.pat(ppos+1)
+def match_LITERAL(ctx, pattern, ptr, ppos):
+    return ctx.str(ptr) == pattern.pat(ppos+1)
 @specializectx
-def match_LITERAL_IGNORE(ctx, ptr, ppos):
-    return ctx.lowstr(ptr) == ctx.pat(ppos+1)
+def match_LITERAL_IGNORE(ctx, pattern, ptr, ppos):
+    return ctx.lowstr(ptr) == pattern.pat(ppos+1)
 @specializectx
-def match_NOT_LITERAL(ctx, ptr, ppos):
-    return ctx.str(ptr) != ctx.pat(ppos+1)
+def match_NOT_LITERAL(ctx, pattern, ptr, ppos):
+    return ctx.str(ptr) != pattern.pat(ppos+1)
 @specializectx
-def match_NOT_LITERAL_IGNORE(ctx, ptr, ppos):
-    return ctx.lowstr(ptr) != ctx.pat(ppos+1)
+def match_NOT_LITERAL_IGNORE(ctx, pattern, ptr, ppos):
+    return ctx.lowstr(ptr) != pattern.pat(ppos+1)
 
 def _make_fre(checkerfn):
     if checkerfn == match_ANY_ALL:
-        def fre(ctx, ptr, end, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
             return end
     elif checkerfn == match_IN:
         install_jitdriver_spec('MatchIn',
-                               greens=['ppos', 'ctx.pattern'],
+                               greens=['ppos', 'pattern'],
                                reds=['ptr', 'end', 'ctx'],
                                debugprint=(1, 0))
         @specializectx
-        def fre(ctx, ptr, end, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
             while True:
                 ctx.jitdriver_MatchIn.jit_merge_point(ctx=ctx, ptr=ptr,
-                                                      end=end, ppos=ppos)
-                if ptr < end and checkerfn(ctx, ptr, ppos):
+                                                      end=end, ppos=ppos,
+                                                      pattern=pattern)
+                if ptr < end and checkerfn(ctx, pattern, ptr, ppos):
                     ptr += 1
                 else:
                     return ptr
     elif checkerfn == match_IN_IGNORE:
         install_jitdriver_spec('MatchInIgnore',
-                               greens=['ppos', 'ctx.pattern'],
+                               greens=['ppos', 'pattern'],
                                reds=['ptr', 'end', 'ctx'],
                                debugprint=(1, 0))
         @specializectx
-        def fre(ctx, ptr, end, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
             while True:
                 ctx.jitdriver_MatchInIgnore.jit_merge_point(ctx=ctx, ptr=ptr,
-                                                            end=end, ppos=ppos)
-                if ptr < end and checkerfn(ctx, ptr, ppos):
+                                                            end=end, ppos=ppos,
+                                                            pattern=pattern)
+                if ptr < end and checkerfn(ctx, pattern, ptr, ppos):
                     ptr += 1
                 else:
                     return ptr
@@ -925,8 +938,8 @@
         # in the other cases, the fre() function is not JITted at all
         # and is present as a residual call.
         @specializectx
-        def fre(ctx, ptr, end, ppos):
-            while ptr < end and checkerfn(ctx, ptr, ppos):
+        def fre(ctx, pattern, ptr, end, ppos):
+            while ptr < end and checkerfn(ctx, pattern, ptr, ppos):
                 ptr += 1
             return ptr
     fre = func_with_new_name(fre, 'fre_' + checkerfn.__name__)
@@ -1037,10 +1050,11 @@
     return start, end
 
 def match(pattern, string, start=0, end=sys.maxint, flags=0, fullmatch=False):
+    assert isinstance(pattern, CompiledPattern)
     start, end = _adjust(start, end, len(string))
-    ctx = StrMatchContext(pattern, string, start, end, flags)
+    ctx = StrMatchContext(string, start, end, flags)
     ctx.fullmatch_only = fullmatch
-    if match_context(ctx):
+    if match_context(ctx, pattern):
         return ctx
     else:
         return None
@@ -1049,105 +1063,106 @@
     return match(pattern, string, start, end, flags, fullmatch=True)
 
 def search(pattern, string, start=0, end=sys.maxint, flags=0):
+    assert isinstance(pattern, CompiledPattern)
     start, end = _adjust(start, end, len(string))
-    ctx = StrMatchContext(pattern, string, start, end, flags)
-    if search_context(ctx):
+    ctx = StrMatchContext(string, start, end, flags)
+    if search_context(ctx, pattern):
         return ctx
     else:
         return None
 
 install_jitdriver('Match',
-                  greens=['ctx.pattern'], reds=['ctx'],
+                  greens=['pattern'], reds=['ctx'],
                   debugprint=(0,))
 
-def match_context(ctx):
+def match_context(ctx, pattern):
     ctx.original_pos = ctx.match_start
     if ctx.end < ctx.match_start:
         return False
-    ctx.jitdriver_Match.jit_merge_point(ctx=ctx)
-    return sre_match(ctx, 0, ctx.match_start, None) is not None
+    ctx.jitdriver_Match.jit_merge_point(ctx=ctx, pattern=pattern)
+    return sre_match(ctx, pattern, 0, ctx.match_start, None) is not None
 
-def search_context(ctx):
+def search_context(ctx, pattern):
     ctx.original_pos = ctx.match_start
     if ctx.end < ctx.match_start:
         return False
     base = 0
     charset = False
-    if ctx.pat(base) == OPCODE_INFO:
-        flags = ctx.pat(2)
+    if pattern.pat(base) == OPCODE_INFO:
+        flags = pattern.pat(2)
         if flags & rsre_char.SRE_INFO_PREFIX:
-            if ctx.pat(5) > 1:
-                return fast_search(ctx)
+            if pattern.pat(5) > 1:
+                return fast_search(ctx, pattern)
         else:
             charset = (flags & rsre_char.SRE_INFO_CHARSET)
-        base += 1 + ctx.pat(1)
-    if ctx.pat(base) == OPCODE_LITERAL:
-        return literal_search(ctx, base)
+        base += 1 + pattern.pat(1)
+    if pattern.pat(base) == OPCODE_LITERAL:
+        return literal_search(ctx, pattern, base)
     if charset:
-        return charset_search(ctx, base)
-    return regular_search(ctx, base)
+        return charset_search(ctx, pattern, base)
+    return regular_search(ctx, pattern, base)
 
 install_jitdriver('RegularSearch',
-                  greens=['base', 'ctx.pattern'],
+                  greens=['base', 'pattern'],
                   reds=['start', 'ctx'],
                   debugprint=(1, 0))
 
-def regular_search(ctx, base):
+def regular_search(ctx, pattern, base):
     start = ctx.match_start
     while start <= ctx.end:
         ctx.jitdriver_RegularSearch.jit_merge_point(ctx=ctx, start=start,
-                                                    base=base)
-        if sre_match(ctx, base, start, None) is not None:
+                                                    base=base, pattern=pattern)
+        if sre_match(ctx, pattern, base, start, None) is not None:
             ctx.match_start = start
             return True
         start += 1
     return False
 
 install_jitdriver_spec("LiteralSearch",
-                       greens=['base', 'character', 'ctx.pattern'],
+                       greens=['base', 'character', 'pattern'],
                        reds=['start', 'ctx'],
                        debugprint=(2, 0, 1))
 @specializectx
-def literal_search(ctx, base):
+def literal_search(ctx, pattern, base):
     # pattern starts with a literal character.  this is used
     # for short prefixes, and if fast search is disabled
-    character = ctx.pat(base + 1)
+    character = pattern.pat(base + 1)
     base += 2
     start = ctx.match_start
     while start < ctx.end:
         ctx.jitdriver_LiteralSearch.jit_merge_point(ctx=ctx, start=start,
-                                          base=base, character=character)
+                                          base=base, character=character, pattern=pattern)
         if ctx.str(start) == character:
-            if sre_match(ctx, base, start + 1, None) is not None:
+            if sre_match(ctx, pattern, base, start + 1, None) is not None:
                 ctx.match_start = start
                 return True
         start += 1
     return False
 
 install_jitdriver_spec("CharsetSearch",
-                       greens=['base', 'ctx.pattern'],
+                       greens=['base', 'pattern'],
                        reds=['start', 'ctx'],
                        debugprint=(1, 0))
 @specializectx
-def charset_search(ctx, base):
+def charset_search(ctx, pattern, base):
     # pattern starts with a character from a known set
     start = ctx.match_start
     while start < ctx.end:
         ctx.jitdriver_CharsetSearch.jit_merge_point(ctx=ctx, start=start,
-                                                    base=base)
-        if rsre_char.check_charset(ctx, 5, ctx.str(start)):
-            if sre_match(ctx, base, start, None) is not None:
+                                                    base=base, pattern=pattern)
+        if rsre_char.check_charset(ctx, pattern, 5, ctx.str(start)):
+            if sre_match(ctx, pattern, base, start, None) is not None:
                 ctx.match_start = start
                 return True
         start += 1
     return False
 
 install_jitdriver_spec('FastSearch',
-                       greens=['i', 'prefix_len', 'ctx.pattern'],
+                       greens=['i', 'prefix_len', 'pattern'],
                        reds=['string_position', 'ctx'],
                        debugprint=(2, 0))
 @specializectx
-def fast_search(ctx):
+def fast_search(ctx, pattern):
     # skips forward in a string as fast as possible using information from
     # an optimization info block
     #  <1=skip> <2=flags> <3=min> <4=...>
@@ -1155,17 +1170,18 @@
     string_position = ctx.match_start
     if string_position >= ctx.end:
         return False

From pypy.commits at gmail.com  Fri Mar 30 19:59:57 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 16:59:57 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: simplify MyGcHooks: instead of using a
 global GC_HOOK_STATS,
 we store the stats as an attribute and we ensure that we don't see gchooks
 from the main rpython program. This is closer to the approach that we will
 use for the real applevel hooks
Message-ID: <5abecf7d.deafdf0a.f7918.fb63@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94192:ecb86e9ba2d6
Date: 2018-03-31 01:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ecb86e9ba2d6/

Log:	simplify MyGcHooks: instead of using a global GC_HOOK_STATS, we
	store the stats as an attribute and we ensure that we don't see
	gchooks from the main rpython program. This is closer to the
	approach that we will use for the real applevel hooks

diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1398,21 +1398,14 @@
     collects = 0
 
     def reset(self):
-        self.minors = 0
-        self.steps = 0
-        self.collects = 0
+        # the NonConstant are needed so that the annotator annotates the
+        # fields as a generic SomeInteger(), instead of a constant 0. A call
+        # to this method MUST be seen during normal annotation, else the class
+        # is annotated only during GC transform, when it's too late
+        self.minors = NonConstant(0)
+        self.steps = NonConstant(0)
+        self.collects = NonConstant(0)
 
-    @staticmethod
-    def fix_annotation():
-        # this is needed to "fix" the annotation of GcHooksStats early, and
-        # must be called from the "main" program. Else, we change the
-        # annotation during the GC transform, when it's too late
-        if NonConstant(False):
-            GC_HOOKS_STATS.collects += 42
-            GC_HOOKS_STATS.steps += 42
-            GC_HOOKS_STATS.minors += 42
-
-GC_HOOKS_STATS = GcHooksStats()
 
 class MyGcHooks(GcHooks):
 
@@ -1420,17 +1413,18 @@
         self.gc_minor_enabled = True
         self.gc_collect_step_enabled = True
         self.gc_collect_enabled = True
+        self.stats = GcHooksStats()
 
     def on_gc_minor(self, total_memory_used, pinned_objects):
-        GC_HOOKS_STATS.minors += 1
+        self.stats.minors += 1
 
     def on_gc_collect_step(self, oldstate, newstate):
-        GC_HOOKS_STATS.steps += 1
+        self.stats.steps += 1
         
     def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
                       arenas_bytes, rawmalloc_bytes_before,
                       rawmalloc_bytes_after):
-        GC_HOOKS_STATS.collects += 1
+        self.stats.collects += 1
 
 
 class TestIncrementalMiniMarkGC(TestMiniMarkGC):
@@ -1487,15 +1481,17 @@
 
     def define_gc_hooks(cls):
         gchooks = cls.gchooks
+        # it is important that we fish .stats OUTSIDE f(); we cannot see
+        # gchooks from within RPython code
+        stats = gchooks.stats
         def f():
-            GC_HOOKS_STATS.fix_annotation()
-            GC_HOOKS_STATS.reset()
+            stats.reset()
             # trigger two major collections
             llop.gc__collect(lltype.Void)
             llop.gc__collect(lltype.Void)
-            return (10000 * GC_HOOKS_STATS.collects +
-                      100 * GC_HOOKS_STATS.steps +
-                        1 * GC_HOOKS_STATS.minors)
+            return (10000 * stats.collects +
+                      100 * stats.steps +
+                        1 * stats.minors)
         return f
 
     def test_gc_hooks(self):
diff --git a/rpython/translator/goal/targetgcbench.py b/rpython/translator/goal/targetgcbench.py
--- a/rpython/translator/goal/targetgcbench.py
+++ b/rpython/translator/goal/targetgcbench.py
@@ -1,10 +1,10 @@
 from rpython.translator.goal import gcbench
-from rpython.memory.test.test_transformed_gc import MyGcHooks, GC_HOOKS_STATS
+from rpython.memory.test.test_transformed_gc import MyGcHooks
 
 # _____ Define and setup target ___
 
 def entry_point(argv):
-    GC_HOOKS_STATS.fix_annotation()
+    GC_HOOKS_STATS.reset()
     ret = gcbench.entry_point(argv)
     minors = GC_HOOKS_STATS.minors
     steps = GC_HOOKS_STATS.steps
@@ -16,6 +16,7 @@
     return ret
 
 gchooks = MyGcHooks()
+GC_HOOKS_STATS = gchooks.stats
 
 def target(*args):
     gcbench.ENABLE_THREADS = False    # not RPython

From pypy.commits at gmail.com  Fri Mar 30 20:00:00 2018
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 30 Mar 2018 17:00:00 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: start to add app-level gc hooks;
 the test pass, but we still need to polish stuff a bit
Message-ID: <5abecf80.4a061c0a.74ae9.fc3d@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94193:ba293bc808ee
Date: 2018-03-31 01:59 +0200
http://bitbucket.org/pypy/pypy/changeset/ba293bc808ee/

Log:	start to add app-level gc hooks; the test pass, but we still need to
	polish stuff a bit

diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -34,5 +34,10 @@
                 'get_typeids_z': 'referents.get_typeids_z',
                 'get_typeids_list': 'referents.get_typeids_list',
                 'GcRef': 'referents.W_GcRef',
+                'set_hooks': 'hook.set_hooks',
                 })
         MixedModule.__init__(self, space, w_name)
+
+    def setup_after_space_initialization(self):
+        from pypy.module.gc.hook import gchooks
+        gchooks.setspace(self.space)
diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/gc/hook.py
@@ -0,0 +1,54 @@
+from rpython.memory.gc.hook import GcHooks
+from pypy.interpreter.baseobjspace import ObjSpace
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.executioncontext import AsyncAction
+
+class LowLevelGcHooks(GcHooks):
+
+    def setspace(self, space):
+        self.space = space
+        self.hooks = space.fromcache(AppLevelHooks)
+
+    def on_gc_minor(self, total_memory_used, pinned_objects):
+        action = self.hooks.gc_minor
+        action.total_memory_used = total_memory_used
+        action.pinned_objects = pinned_objects
+        action.fire()
+
+    def on_gc_collect_step(self, oldstate, newstate):
+        pass
+
+    def on_gc_collect(self, count, arenas_count_before, arenas_count_after,
+                      arenas_bytes, rawmalloc_bytes_before,
+                      rawmalloc_bytes_after):
+        pass
+
+
+gchooks = LowLevelGcHooks()
+
+class AppLevelHooks(object):
+
+    def __init__(self, space):
+        self.space = space
+        self.gc_minor = GcMinorHookAction(space)
+
+    def set_hooks(self, space, w_on_gc_minor):
+        # XXX: check for None and enable/disable accordingly
+        self.gc_minor.w_callable = w_on_gc_minor
+
+
+class GcMinorHookAction(AsyncAction):
+    w_callable = None
+    total_memory_used = 0
+    pinned_objects = 0
+
+    def perform(self, ec, frame):
+        self.space.call_function(self.w_callable,
+                                 self.space.wrap(self.total_memory_used),
+                                 self.space.wrap(self.pinned_objects))
+
+
+
+
+def set_hooks(space, w_on_gc_minor):
+    space.fromcache(AppLevelHooks).set_hooks(space, w_on_gc_minor)
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/gc/test/test_hook.py
@@ -0,0 +1,26 @@
+from pypy.module.gc.hook import gchooks
+from pypy.interpreter.baseobjspace import ObjSpace
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+
+class AppTestGcHooks(object):
+
+    def setup_class(cls):
+        space = cls.space
+
+        @unwrap_spec(ObjSpace, int, int)
+        def fire_gc_minor(space, total_memory_used, pinned_objects):
+            gchooks.fire_gc_minor(total_memory_used, pinned_objects)
+        cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor))
+
+    def test_on_gc_minor(self):
+        import gc
+        lst = []
+        def on_gc_minor(total_memory_used, pinned_objects):
+            lst.append((total_memory_used, pinned_objects))
+        gc.set_hooks(on_gc_minor=on_gc_minor)
+        self.fire_gc_minor(10, 20)
+        self.fire_gc_minor(30, 40)
+        assert lst == [
+            (10, 20),
+            (30, 40),
+            ]

From pypy.commits at gmail.com  Sat Mar 31 06:07:41 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 03:07:41 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: move the if gc_*_enabled checks inside
 the fire_* methods
Message-ID: <5abf5ded.92181c0a.13e09.0cd0@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94194:def5c83256cc
Date: 2018-03-31 11:49 +0200
http://bitbucket.org/pypy/pypy/changeset/def5c83256cc/

Log:	move the if gc_*_enabled checks inside the fire_* methods

diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
--- a/rpython/memory/gc/hook.py
+++ b/rpython/memory/gc/hook.py
@@ -15,7 +15,6 @@
         self.gc_collect_step_enabled = False
         self.gc_collect_enabled = False
 
-
     def on_gc_minor(self, total_memory_used, pinned_objects):
         """
         Called after a minor collection
@@ -44,16 +43,19 @@
 
     @rgc.no_collect
     def fire_gc_minor(self, total_memory_used, pinned_objects):
-        self.on_gc_minor(total_memory_used, pinned_objects)
+        if self.gc_minor_enabled:
+            self.on_gc_minor(total_memory_used, pinned_objects)
 
     @rgc.no_collect
     def fire_gc_collect_step(self, oldstate, newstate):
-        self.on_gc_collect_step(oldstate, newstate)
+        if self.gc_collect_step_enabled:
+            self.on_gc_collect_step(oldstate, newstate)
 
     @rgc.no_collect
     def fire_gc_collect(self, count, arenas_count_before, arenas_count_after,
                         arenas_bytes, rawmalloc_bytes_before,
                         rawmalloc_bytes_after):
-        self.on_gc_collect(count, arenas_count_before, arenas_count_after,
-                           arenas_bytes, rawmalloc_bytes_before,
-                           rawmalloc_bytes_after)
+        if self.gc_collect_enabled:
+            self.on_gc_collect(count, arenas_count_before, arenas_count_after,
+                               arenas_bytes, rawmalloc_bytes_before,
+                               rawmalloc_bytes_after)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1838,9 +1838,8 @@
         self.root_walker.finished_minor_collection()
         #
         debug_stop("gc-minor")
-        if self.hooks.gc_minor_enabled:
-            self.hooks.fire_gc_minor(total_memory_used=total_memory_used,
-                                   pinned_objects=self.pinned_objects_in_nursery)
+        self.hooks.fire_gc_minor(total_memory_used=total_memory_used,
+                               pinned_objects=self.pinned_objects_in_nursery)
 
     def _reset_flag_old_objects_pointing_to_pinned(self, obj, ignore):
         ll_assert(self.header(obj).tid & GCFLAG_PINNED_OBJECT_PARENT_KNOWN != 0,
@@ -2424,14 +2423,13 @@
                             self.stat_rawmalloced_total_size, " => ",
                             self.rawmalloced_total_size)
                 debug_stop("gc-collect-done")
-                if self.hooks.gc_collect_enabled:
-                    self.hooks.fire_gc_collect(
-                        count=self.num_major_collects,
-                        arenas_count_before=self.stat_ac_arenas_count,
-                        arenas_count_after=self.ac.arenas_count,
-                        arenas_bytes=self.ac.total_memory_used,
-                        rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
-                        rawmalloc_bytes_after=self.rawmalloced_total_size)
+                self.hooks.fire_gc_collect(
+                    count=self.num_major_collects,
+                    arenas_count_before=self.stat_ac_arenas_count,
+                    arenas_count_after=self.ac.arenas_count,
+                    arenas_bytes=self.ac.total_memory_used,
+                    rawmalloc_bytes_before=self.stat_rawmalloced_total_size,
+                    rawmalloc_bytes_after=self.rawmalloced_total_size)
                 #
                 # Set the threshold for the next major collection to be when we
                 # have allocated 'major_collection_threshold' times more than
@@ -2483,9 +2481,8 @@
 
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
-        if self.hooks.gc_collect_step_enabled:
-            self.hooks.fire_gc_collect_step(oldstate=oldstate,
-                                          newstate=self.gc_state)
+        self.hooks.fire_gc_collect_step(oldstate=oldstate,
+                                      newstate=self.gc_state)
 
     def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):
         if self.header(obj).tid & GCFLAG_VISITED:

From pypy.commits at gmail.com  Sat Mar 31 06:07:45 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 03:07:45 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: enable/disable the low-level hook
 according to the value of w_on_gc_minor
Message-ID: <5abf5df1.22b7df0a.8b10c.e530@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94196:91b33ec9b345
Date: 2018-03-31 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/91b33ec9b345/

Log:	enable/disable the low-level hook according to the value of
	w_on_gc_minor

diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -9,6 +9,9 @@
         self.space = space
         self.hooks = space.fromcache(AppLevelHooks)
 
+    def is_gc_minor_enabled(self):
+        return self.hooks.gc_minor_enabled
+
     def on_gc_minor(self, total_memory_used, pinned_objects):
         action = self.hooks.gc_minor
         action.total_memory_used = total_memory_used
@@ -30,10 +33,11 @@
 
     def __init__(self, space):
         self.space = space
+        self.gc_minor_enabled = False
         self.gc_minor = GcMinorHookAction(space)
 
     def set_hooks(self, space, w_on_gc_minor):
-        # XXX: check for None and enable/disable accordingly
+        self.gc_minor_enabled = not space.is_none(w_on_gc_minor)
         self.gc_minor.w_callable = w_on_gc_minor
 
 
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
--- a/pypy/module/gc/test/test_hook.py
+++ b/pypy/module/gc/test/test_hook.py
@@ -24,3 +24,10 @@
             (10, 20),
             (30, 40),
             ]
+        #
+        gc.set_hooks(on_gc_minor=None)
+        self.fire_gc_minor(50, 60)  # won't fire because the hooks is disabled
+        assert lst == [
+            (10, 20),
+            (30, 40),
+            ]

From pypy.commits at gmail.com  Sat Mar 31 06:07:44 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 03:07:44 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: turn gc_*_enabled from attributes into
 overrideable methods. This is needed because the normal rpython code CANNOT
 reference the global gchooks,
 else we have annotations issues: by using methods,
 we can write GcHooks in a way which stores the actual *_enabled flags on
 e.g. the AppLevelGcHooks object
Message-ID: <5abf5df0.ab87df0a.8736b.2211@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94195:e854ddff70bd
Date: 2018-03-31 12:02 +0200
http://bitbucket.org/pypy/pypy/changeset/e854ddff70bd/

Log:	turn gc_*_enabled from attributes into overrideable methods. This is
	needed because the normal rpython code CANNOT reference the global
	gchooks, else we have annotations issues: by using methods, we can
	write GcHooks in a way which stores the actual *_enabled flags on
	e.g. the AppLevelGcHooks object

diff --git a/rpython/memory/gc/hook.py b/rpython/memory/gc/hook.py
--- a/rpython/memory/gc/hook.py
+++ b/rpython/memory/gc/hook.py
@@ -10,10 +10,14 @@
     trigger a GC collection.
     """
 
-    def __init__(self):
-        self.gc_minor_enabled = False
-        self.gc_collect_step_enabled = False
-        self.gc_collect_enabled = False
+    def is_gc_minor_enabled(self):
+        return False
+
+    def is_gc_collect_step_enabled(self):
+        return False
+
+    def is_gc_collect_enabled(self):
+        return False
 
     def on_gc_minor(self, total_memory_used, pinned_objects):
         """
@@ -43,19 +47,19 @@
 
     @rgc.no_collect
     def fire_gc_minor(self, total_memory_used, pinned_objects):
-        if self.gc_minor_enabled:
+        if self.is_gc_minor_enabled():
             self.on_gc_minor(total_memory_used, pinned_objects)
 
     @rgc.no_collect
     def fire_gc_collect_step(self, oldstate, newstate):
-        if self.gc_collect_step_enabled:
+        if self.is_gc_collect_step_enabled():
             self.on_gc_collect_step(oldstate, newstate)
 
     @rgc.no_collect
     def fire_gc_collect(self, count, arenas_count_before, arenas_count_after,
                         arenas_bytes, rawmalloc_bytes_before,
                         rawmalloc_bytes_after):
-        if self.gc_collect_enabled:
+        if self.is_gc_collect_enabled():
             self.on_gc_collect(count, arenas_count_before, arenas_count_after,
                                arenas_bytes, rawmalloc_bytes_before,
                                rawmalloc_bytes_after)
diff --git a/rpython/memory/gc/test/test_hook.py b/rpython/memory/gc/test/test_hook.py
--- a/rpython/memory/gc/test/test_hook.py
+++ b/rpython/memory/gc/test/test_hook.py
@@ -7,8 +7,20 @@
 
     def __init__(self):
         GcHooks.__init__(self)
+        self._gc_minor_enabled = False
+        self._gc_collect_step_enabled = False
+        self._gc_collect_enabled = False
         self.reset()
 
+    def is_gc_minor_enabled(self):
+        return self._gc_minor_enabled
+
+    def is_gc_collect_step_enabled(self):
+        return self._gc_collect_step_enabled
+
+    def is_gc_collect_enabled(self):
+        return self._gc_collect_enabled
+
     def reset(self):
         self.minors = []
         self.steps = []
@@ -48,7 +60,7 @@
         self.size_of_S = llmemory.raw_malloc_usage(size)
 
     def test_on_gc_minor(self):
-        self.gc.hooks.gc_minor_enabled = True
+        self.gc.hooks._gc_minor_enabled = True
         self.malloc(S)
         self.gc._minor_collection()
         assert self.gc.hooks.minors == [
@@ -66,8 +78,8 @@
 
     def test_on_gc_collect(self):
         from rpython.memory.gc import incminimark as m
-        self.gc.hooks.gc_collect_step_enabled = True
-        self.gc.hooks.gc_collect_enabled = True
+        self.gc.hooks._gc_collect_step_enabled = True
+        self.gc.hooks._gc_collect_enabled = True
         self.malloc(S)
         self.gc.collect()
         assert self.gc.hooks.steps == [
diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py
--- a/rpython/memory/test/test_transformed_gc.py
+++ b/rpython/memory/test/test_transformed_gc.py
@@ -1410,11 +1410,17 @@
 class MyGcHooks(GcHooks):
 
     def __init__(self):
-        self.gc_minor_enabled = True
-        self.gc_collect_step_enabled = True
-        self.gc_collect_enabled = True
         self.stats = GcHooksStats()
 
+    def is_gc_minor_enabled(self):
+        return True
+
+    def is_gc_collect_step_enabled(self):
+        return True
+
+    def is_gc_collect_enabled(self):
+        return True
+
     def on_gc_minor(self, total_memory_used, pinned_objects):
         self.stats.minors += 1
 

From pypy.commits at gmail.com  Sat Mar 31 06:58:45 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 31 Mar 2018 03:58:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation, hopefully
Message-ID: <5abf69e5.95921c0a.1a169.51cd@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r94197:6b7d3a98a2ba
Date: 2018-03-31 12:58 +0200
http://bitbucket.org/pypy/pypy/changeset/6b7d3a98a2ba/

Log:	fix translation, hopefully

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -113,14 +113,14 @@
             ofs = ops_offset.get(op, 0)
         num = op.getopnum()
         name = op.getopname()
-        repr = logops.repr_of_resop(op)
+        repr_op = logops.repr_of_resop(op)
         if num == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
             greenkey = op.getarglist()[3:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, name,
-                                       repr,
+                                       repr_op,
                                        jd_sd.jitdriver.name,
                                        op.getarg(1).getint(),
                                        op.getarg(2).getint(),
@@ -131,9 +131,9 @@
                 hash = op.getdescr().get_jitcounter_hash()
             else:
                 hash = -1
-            l_w.append(GuardOp(name, ofs, repr, hash))
+            l_w.append(GuardOp(name, ofs, repr_op, hash))
         else:
-            l_w.append(WrappedOp(name, ofs, repr))
+            l_w.append(WrappedOp(name, ofs, repr_op))
     return l_w
 
 @unwrap_spec(offset=int, repr='text', name='text')

From pypy.commits at gmail.com  Sat Mar 31 08:48:13 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 05:48:13 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: blindly try to use the new gchooks in
 the pypy target
Message-ID: <5abf838d.abaddf0a.cd1de.9fe6@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94198:5bf5fddf26f6
Date: 2018-03-31 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/5bf5fddf26f6/

Log:	blindly try to use the new gchooks in the pypy target

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -364,6 +364,10 @@
         from pypy.module.pypyjit.hooks import pypy_hooks
         return PyPyJitPolicy(pypy_hooks)
 
+    def get_gchooks(self):
+        from pypy.module.gc.hook import gchooks
+        return gchooks
+
     def get_entry_point(self, config):
         space = make_objspace(config)
 
@@ -381,7 +385,7 @@
                      'jitpolicy', 'get_entry_point',
                      'get_additional_config_options']:
             ns[name] = getattr(self, name)
-
+        ns['gchooks'] = self.get_gchooks()
 
 PyPyTarget().interface(globals())
 

From pypy.commits at gmail.com  Sat Mar 31 09:08:59 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 06:08:59 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: space.wrap is no longer rpython
Message-ID: <5abf886b.5b88df0a.be1c9.d96d@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94199:e44493a28aeb
Date: 2018-03-31 15:08 +0200
http://bitbucket.org/pypy/pypy/changeset/e44493a28aeb/

Log:	space.wrap is no longer rpython

diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -48,8 +48,8 @@
 
     def perform(self, ec, frame):
         self.space.call_function(self.w_callable,
-                                 self.space.wrap(self.total_memory_used),
-                                 self.space.wrap(self.pinned_objects))
+                                 self.space.newint(self.total_memory_used),
+                                 self.space.newint(self.pinned_objects))
 
 
 

From pypy.commits at gmail.com  Sat Mar 31 09:42:11 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 06:42:11 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: try to fix annotation
Message-ID: <5abf9033.88c0df0a.bce60.a529@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94201:6d1d852ff79f
Date: 2018-03-31 15:41 +0200
http://bitbucket.org/pypy/pypy/changeset/6d1d852ff79f/

Log:	try to fix annotation

diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -1,4 +1,5 @@
 from rpython.memory.gc.hook import GcHooks
+from rpython.rlib.nonconst import NonConstant
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
@@ -47,7 +48,14 @@
     total_memory_used = 0
     pinned_objects = 0
 
+    def fix_annotation(self):
+        # XXX write comment
+        if NonConstant(False):
+            self.total_memory_used += 42
+            self.pinned_objects += 42
+
     def perform(self, ec, frame):
+        self.fix_annotation()
         w_stats = W_GcMinorStats(self.total_memory_used, self.pinned_objects)
         self.space.call_function(self.w_callable, w_stats)
 

From pypy.commits at gmail.com  Sat Mar 31 09:42:09 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 06:42:09 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: pass a 'stats' object to the hook
 instead of passing all the values individually
Message-ID: <5abf9031.c7a3df0a.65316.3b9a@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94200:ce5745894545
Date: 2018-03-31 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/ce5745894545/

Log:	pass a 'stats' object to the hook instead of passing all the values
	individually

diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -1,6 +1,7 @@
 from rpython.memory.gc.hook import GcHooks
-from pypy.interpreter.baseobjspace import ObjSpace
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.executioncontext import AsyncAction
 
 class LowLevelGcHooks(GcHooks):
@@ -47,11 +48,24 @@
     pinned_objects = 0
 
     def perform(self, ec, frame):
-        self.space.call_function(self.w_callable,
-                                 self.space.newint(self.total_memory_used),
-                                 self.space.newint(self.pinned_objects))
+        w_stats = W_GcMinorStats(self.total_memory_used, self.pinned_objects)
+        self.space.call_function(self.w_callable, w_stats)
 
 
+class W_GcMinorStats(W_Root):
+
+    def __init__(self, total_memory_used, pinned_objects):
+        self.total_memory_used = total_memory_used
+        self.pinned_objects = pinned_objects
+
+
+W_GcMinorStats.typedef = TypeDef(
+    "GcMinorStats",
+    total_memory_used = interp_attrproperty("total_memory_used",
+                                            cls=W_GcMinorStats, wrapfn="newint"),
+    pinned_objects = interp_attrproperty("pinned_objects",
+                                         cls=W_GcMinorStats, wrapfn="newint"),
+    )
 
 
 def set_hooks(space, w_on_gc_minor):
diff --git a/pypy/module/gc/test/test_hook.py b/pypy/module/gc/test/test_hook.py
--- a/pypy/module/gc/test/test_hook.py
+++ b/pypy/module/gc/test/test_hook.py
@@ -1,3 +1,4 @@
+from rpython.rlib.rarithmetic import r_uint
 from pypy.module.gc.hook import gchooks
 from pypy.interpreter.baseobjspace import ObjSpace
 from pypy.interpreter.gateway import interp2app, unwrap_spec
@@ -7,7 +8,7 @@
     def setup_class(cls):
         space = cls.space
 
-        @unwrap_spec(ObjSpace, int, int)
+        @unwrap_spec(ObjSpace, r_uint, int)
         def fire_gc_minor(space, total_memory_used, pinned_objects):
             gchooks.fire_gc_minor(total_memory_used, pinned_objects)
         cls.w_fire_gc_minor = space.wrap(interp2app(fire_gc_minor))
@@ -15,8 +16,8 @@
     def test_on_gc_minor(self):
         import gc
         lst = []
-        def on_gc_minor(total_memory_used, pinned_objects):
-            lst.append((total_memory_used, pinned_objects))
+        def on_gc_minor(stats):
+            lst.append((stats.total_memory_used, stats.pinned_objects))
         gc.set_hooks(on_gc_minor=on_gc_minor)
         self.fire_gc_minor(10, 20)
         self.fire_gc_minor(30, 40)

From pypy.commits at gmail.com  Sat Mar 31 11:05:10 2018
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 31 Mar 2018 08:05:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: like this, I hope
Message-ID: <5abfa3a6.13811c0a.b0189.91be@mx.google.com>

Author: Carl Friedrich Bolz-Tereick 
Branch: 
Changeset: r94202:a145a9a4581b
Date: 2018-03-31 17:04 +0200
http://bitbucket.org/pypy/pypy/changeset/a145a9a4581b/

Log:	like this, I hope

diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py
--- a/pypy/module/pypyjit/interp_resop.py
+++ b/pypy/module/pypyjit/interp_resop.py
@@ -113,14 +113,13 @@
             ofs = ops_offset.get(op, 0)
         num = op.getopnum()
         name = op.getopname()
-        repr_op = logops.repr_of_resop(op)
         if num == rop.DEBUG_MERGE_POINT:
             jd_sd = jitdrivers_sd[op.getarg(0).getint()]
             greenkey = op.getarglist()[3:]
             repr = jd_sd.warmstate.get_location_str(greenkey)
             w_greenkey = wrap_greenkey(space, jd_sd.jitdriver, greenkey, repr)
             l_w.append(DebugMergePoint(space, name,
-                                       repr_op,
+                                       logops.repr_of_resop(op),
                                        jd_sd.jitdriver.name,
                                        op.getarg(1).getint(),
                                        op.getarg(2).getint(),
@@ -130,10 +129,11 @@
             if descr is not None: # can be none in on_abort!
                 hash = op.getdescr().get_jitcounter_hash()
             else:
-                hash = -1
-            l_w.append(GuardOp(name, ofs, repr_op, hash))
+                hash = r_uint(0)
+            l_w.append(GuardOp(name, ofs, logops.repr_of_resop(op),
+                hash))
         else:
-            l_w.append(WrappedOp(name, ofs, repr_op))
+            l_w.append(WrappedOp(name, ofs, logops.repr_of_resop(op)))
     return l_w
 
 @unwrap_spec(offset=int, repr='text', name='text')
diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py
--- a/pypy/module/pypyjit/test/test_jit_hook.py
+++ b/pypy/module/pypyjit/test/test_jit_hook.py
@@ -242,6 +242,7 @@
         assert name == 'pypyjit'
         assert reason == 'ABORT_TOO_LONG'
         assert len(ops) == 4
+        assert ops[0].hash == 0
 
     def test_creation(self):
         from pypyjit import ResOperation

From pypy.commits at gmail.com  Sat Mar 31 14:04:58 2018
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 31 Mar 2018 11:04:58 -0700 (PDT)
Subject: [pypy-commit] pypy gc-hooks: this seems to fix annotation;
 however, it still cannot translated because AsyncAction.fire() can
 allocate, and so we cannot call it directly from the GC hook
Message-ID: <5abfcdca.09c5df0a.96cca.0968@mx.google.com>

Author: Antonio Cuni 
Branch: gc-hooks
Changeset: r94204:f261ec7406c6
Date: 2018-03-31 19:04 +0100
http://bitbucket.org/pypy/pypy/changeset/f261ec7406c6/

Log:	this seems to fix annotation; however, it still cannot translated
	because AsyncAction.fire() can allocate, and so we cannot call it
	directly from the GC hook

diff --git a/pypy/module/gc/hook.py b/pypy/module/gc/hook.py
--- a/pypy/module/gc/hook.py
+++ b/pypy/module/gc/hook.py
@@ -1,5 +1,6 @@
 from rpython.memory.gc.hook import GcHooks
 from rpython.rlib.nonconst import NonConstant
+from rpython.rlib.rarithmetic import r_uint
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
@@ -41,6 +42,7 @@
     def set_hooks(self, space, w_on_gc_minor):
         self.gc_minor_enabled = not space.is_none(w_on_gc_minor)
         self.gc_minor.w_callable = w_on_gc_minor
+        self.gc_minor.fix_annotation()
 
 
 class GcMinorHookAction(AsyncAction):
@@ -49,13 +51,15 @@
     pinned_objects = 0
 
     def fix_annotation(self):
-        # XXX write comment
+        # the annotation of the class and its attributes must be completed
+        # BEFORE we do the gc transform; this makes sure that everything is
+        # annotated with the correct types
         if NonConstant(False):
-            self.total_memory_used += 42
-            self.pinned_objects += 42
+            self.total_memory_used = NonConstant(r_uint(42))
+            self.pinned_objects = NonConstant(-42)
+            self.fire()
 
     def perform(self, ec, frame):
-        self.fix_annotation()
         w_stats = W_GcMinorStats(self.total_memory_used, self.pinned_objects)
         self.space.call_function(self.w_callable, w_stats)
 

From pypy.commits at gmail.com  Sat Mar 31 14:38:17 2018
From: pypy.commits at gmail.com (mjacob)
Date: Sat, 31 Mar 2018 11:38:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: Test and fix async generator with 'await'
 in 'finally' block.
Message-ID: <5abfd599.d0d0df0a.2a4a3.0b73@mx.google.com>

Author: Manuel Jacob 
Branch: py3.6
Changeset: r94205:252f070e9c1a
Date: 2018-03-31 20:17 +0200
http://bitbucket.org/pypy/pypy/changeset/252f070e9c1a/

Log:	Test and fix async generator with 'await' in 'finally' block.

diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -718,7 +718,7 @@
                     # TODO: add equivalent to CPython's o->agt_gen->ag_closed = 1;
                     w_value = self.async_gen.throw(space.w_GeneratorExit,
                                                    None, None)
-                    if w_value is not None:
+                    if w_value is not None and isinstance(w_value, AsyncGenValueWrapper):
                         raise oefmt(space.w_RuntimeError,
                                     "async generator ignored GeneratorExit")
                 else:
diff --git a/pypy/interpreter/test/test_coroutine.py b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -504,6 +504,39 @@
         raises(RuntimeError, run().send, None)
     """
 
+    def test_async_aclose_await_in_finally(self): """
+        import types
+
+        @types.coroutine
+        def coro():
+            yield 'coro'
+
+        state = 0
+        async def ag():
+            nonlocal state
+            try:
+                yield
+            finally:
+                state = 1
+                await coro()
+                state = 2
+
+        async def run():
+            a = ag()
+            async for i in a:
+                break
+            await a.aclose()
+        a = run()
+        assert state == 0
+        assert a.send(None) == 'coro'
+        assert state == 1
+        try:
+            a.send(None)
+        except StopIteration:
+            pass
+        assert state == 2
+    """
+
     def test_async_anext_close(self): """
         async def ag():
             yield 42